diff options
| author | Michael Kruse <llvm-project@meinersbur.de> | 2025-01-03 10:22:51 +0100 |
|---|---|---|
| committer | Michael Kruse <llvm-project@meinersbur.de> | 2025-01-03 10:22:51 +0100 |
| commit | 38500d63e14ce340236840f60d356cdefb56a52c (patch) | |
| tree | 17edbec446ce9b50d2f215a483b83afb293a635d /llvm/test/CodeGen/X86 | |
| parent | 1a3d5daaef7a6a63448a497da3eff7fc9e23df26 (diff) | |
| parent | 27f30029741ecf023baece7b3dde1ff9011ffefc (diff) | |
Merge branch 'main' into users/meinersbur/flang_runtime_split-headersusers/meinersbur/flang_runtime_split-headers
Diffstat (limited to 'llvm/test/CodeGen/X86')
80 files changed, 5954 insertions, 4891 deletions
diff --git a/llvm/test/CodeGen/X86/2006-10-02-BoolRetCrash.ll b/llvm/test/CodeGen/X86/2006-10-02-BoolRetCrash.ll index 3b59c11c0668..7f938029dd83 100644 --- a/llvm/test/CodeGen/X86/2006-10-02-BoolRetCrash.ll +++ b/llvm/test/CodeGen/X86/2006-10-02-BoolRetCrash.ll @@ -1,4 +1,4 @@ -; RUN: llc -march=x86 < %s +; RUN: llc -mtriple=i686 < %s ; PR933 define fastcc i1 @test() { diff --git a/llvm/test/CodeGen/X86/GC/ocaml-gc.ll b/llvm/test/CodeGen/X86/GC/ocaml-gc.ll index 4e4e2e952f73..9e05f1ceaa0c 100644 --- a/llvm/test/CodeGen/X86/GC/ocaml-gc.ll +++ b/llvm/test/CodeGen/X86/GC/ocaml-gc.ll @@ -1,7 +1,6 @@ ; RUN: llc < %s -mtriple=x86_64-linux-gnu | FileCheck %s -; CHECK: .text -; CHECK-NEXT: .file "<stdin>" +; CHECK: .file "<stdin>" define i32 @main(i32 %x) nounwind gc "ocaml" { ; CHECK: .globl "caml<stdin>__code_begin" diff --git a/llvm/test/CodeGen/X86/GlobalISel/select-intrinsic-x86-flags-read-u32.mir b/llvm/test/CodeGen/X86/GlobalISel/select-intrinsic-x86-flags-read-u32.mir index 332ec2240c5b..3d1857a274b4 100644 --- a/llvm/test/CodeGen/X86/GlobalISel/select-intrinsic-x86-flags-read-u32.mir +++ b/llvm/test/CodeGen/X86/GlobalISel/select-intrinsic-x86-flags-read-u32.mir @@ -9,7 +9,7 @@ define void @read_flags() { ret void } ; CHECK-LABEL: name: read_flags ; CHECK: bb.0: - ; CHECK: [[RDFLAGS32_:%[0-9]+]]:gr32 = RDFLAGS32 implicit-def $esp, implicit $esp + ; CHECK: [[RDFLAGS32_:%[0-9]+]]:gr32 = RDFLAGS32 implicit-def dead $esp, implicit $esp ; CHECK: $eax = COPY [[RDFLAGS32_]] ... diff --git a/llvm/test/CodeGen/X86/anyregcc-crash.ll b/llvm/test/CodeGen/X86/anyregcc-crash.ll index d5d6ebe751c0..3b65babea23e 100644 --- a/llvm/test/CodeGen/X86/anyregcc-crash.ll +++ b/llvm/test/CodeGen/X86/anyregcc-crash.ll @@ -2,7 +2,7 @@ ; ; Check that misuse of anyregcc results in a compile time error. -; CHECK: error: ran out of registers during register allocation +; CHECK: error: <unknown>:0:0: ran out of registers during register allocation define i64 @anyreglimit(i64 %v1, i64 %v2, i64 %v3, i64 %v4, i64 %v5, i64 %v6, i64 %v7, i64 %v8, i64 %v9, i64 %v10, i64 %v11, i64 %v12, i64 %v13, i64 %v14, i64 %v15, i64 %v16) { diff --git a/llvm/test/CodeGen/X86/apx/cfcmov.ll b/llvm/test/CodeGen/X86/apx/cfcmov.ll index f643120c9b50..37ba3d451c2b 100644 --- a/llvm/test/CodeGen/X86/apx/cfcmov.ll +++ b/llvm/test/CodeGen/X86/apx/cfcmov.ll @@ -93,3 +93,91 @@ define i64 @cfcmov64rr_inv(i64 %0) { %3 = select i1 %2, i64 0, i64 %0 ret i64 %3 } + +define void @cfcmov16mr(ptr %p, i16 %0) { +; CHECK-LABEL: cfcmov16mr: +; CHECK: # %bb.0: +; CHECK-NEXT: movzwl (%rdi), %eax +; CHECK-NEXT: cmpw %ax, %si +; CHECK-NEXT: cfcmovlew %si, (%rdi) +; CHECK-NEXT: retq + %2 = load i16, ptr %p, align 2 + %3 = icmp sgt i16 %0, %2 + %4 = select i1 %3, i16 %2, i16 %0 + store i16 %4, ptr %p, align 2 + ret void +} + +define void @cfcmov32mr(ptr %p, i32 %0) { +; CHECK-LABEL: cfcmov32mr: +; CHECK: # %bb.0: +; CHECK-NEXT: cmpl (%rdi), %esi +; CHECK-NEXT: cfcmovgl %esi, (%rdi) +; CHECK-NEXT: retq + %2 = load i32, ptr %p, align 4 + %3 = call i32 @llvm.smax.i32(i32 %0, i32 %2) + store i32 %3, ptr %p, align 4 + ret void +} + +define void @cfcmov64mr(ptr %p, i64 %0) { +; CHECK-LABEL: cfcmov64mr: +; CHECK: # %bb.0: +; CHECK-NEXT: cmpq (%rdi), %rsi +; CHECK-NEXT: cfcmovgq %rsi, (%rdi) +; CHECK-NEXT: retq + %2 = load i64, ptr %p, align 8 + %3 = icmp sgt i64 %0, %2 + %4 = select i1 %3, i64 %0, i64 %2 + store i64 %4, ptr %p, align 8 + ret void +} + +define void @volatileload(ptr %p, i32 %0) { +; CHECK-LABEL: volatileload: +; CHECK: # %bb.0: +; CHECK-NEXT: movl (%rdi), %eax +; CHECK-NEXT: cmpl %eax, %esi +; CHECK-NEXT: cmovbl %esi, %eax +; CHECK-NEXT: movl %eax, (%rdi) +; CHECK-NEXT: retq + %2 = load volatile i32, ptr %p, align 4 + %3 = call i32 @llvm.umin.i32(i32 %0, i32 %2) + store i32 %3, ptr %p, align 4 + ret void +} + +define void @atomicstore(ptr %p, i64 %0) { +; CHECK-LABEL: atomicstore: +; CHECK: # %bb.0: +; CHECK-NEXT: movq (%rdi), %rax +; CHECK-NEXT: cmpq %rax, %rsi +; CHECK-NEXT: cmovaq %rsi, %rax +; CHECK-NEXT: movq %rax, (%rdi) +; CHECK-NEXT: retq + %2 = load i64, ptr %p, align 8 + %3 = icmp ugt i64 %0, %2 + %4 = select i1 %3, i64 %0, i64 %2 + store atomic i64 %4, ptr %p unordered, align 8 + ret void +} + +define void @loadstorediffptr(ptr %p, i32 %0) { +; CHECK-LABEL: loadstorediffptr: +; CHECK: # %bb.0: +; CHECK-NEXT: movl (%rdi), %eax +; CHECK-NEXT: cmpl %eax, %esi +; CHECK-NEXT: cmovbel %eax, %esi +; CHECK-NEXT: movl %esi, 4(%rdi) +; CHECK-NEXT: retq + %2 = getelementptr [2 x i32], ptr %p, i32 0, i32 0 + %3 = load i32, ptr %2, align 4 + %4 = icmp ule i32 %0, %3 + %5 = select i1 %4, i32 %3, i32 %0 + %6 = getelementptr [2 x i32], ptr %p, i32 0, i32 1 + store i32 %5, ptr %6, align 4 + ret void +} + +declare i32 @llvm.smax.i32(i32, i32) +declare i32 @llvm.umin.i32(i32, i32) diff --git a/llvm/test/CodeGen/X86/apx/mul-i1024.ll b/llvm/test/CodeGen/X86/apx/mul-i1024.ll index a4d15a1b21d6..a29a92176f43 100644 --- a/llvm/test/CodeGen/X86/apx/mul-i1024.ll +++ b/llvm/test/CodeGen/X86/apx/mul-i1024.ll @@ -13,104 +13,104 @@ define void @test_1024(ptr %a, ptr %b, ptr %out) nounwind { ; EGPR-NEXT: pushq %rbx ; EGPR-NEXT: subq $104, %rsp ; EGPR-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill -; EGPR-NEXT: movq %rdi, %r24 +; EGPR-NEXT: movq %rdi, %r26 ; EGPR-NEXT: movq (%rdi), %r13 ; EGPR-NEXT: movq 8(%rdi), %r18 -; EGPR-NEXT: movq 24(%rdi), %r29 +; EGPR-NEXT: movq 24(%rdi), %r21 ; EGPR-NEXT: movq 16(%rdi), %r17 ; EGPR-NEXT: movq 40(%rdi), %rdi -; EGPR-NEXT: movq 32(%r24), %r10 -; EGPR-NEXT: movq 56(%r24), %r15 -; EGPR-NEXT: movq 48(%r24), %r12 +; EGPR-NEXT: movq 32(%r26), %r10 +; EGPR-NEXT: movq 56(%r26), %r15 +; EGPR-NEXT: movq 48(%r26), %r12 ; EGPR-NEXT: movq %rsi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill -; EGPR-NEXT: movq 24(%rsi), %r23 +; EGPR-NEXT: movq 24(%rsi), %r25 ; EGPR-NEXT: movq 16(%rsi), %r11 -; EGPR-NEXT: movq (%rsi), %r27 +; EGPR-NEXT: movq (%rsi), %r31 ; EGPR-NEXT: movq 8(%rsi), %r14 ; EGPR-NEXT: movq %r12, %rax -; EGPR-NEXT: mulq %r27 +; EGPR-NEXT: mulq %r31 ; EGPR-NEXT: movq %rdx, %r8 ; EGPR-NEXT: movq %rax, %r19 ; EGPR-NEXT: movq %r15, %rax -; EGPR-NEXT: mulq %r27 +; EGPR-NEXT: mulq %r31 ; EGPR-NEXT: movq %rdx, %r9 ; EGPR-NEXT: movq %rax, %r16 ; EGPR-NEXT: addq %r8, %r16 ; EGPR-NEXT: adcq $0, %r9 ; EGPR-NEXT: movq %r12, %rax ; EGPR-NEXT: mulq %r14 -; EGPR-NEXT: movq %rdx, %r20 +; EGPR-NEXT: movq %rdx, %r22 ; EGPR-NEXT: movq %rax, %r8 ; EGPR-NEXT: addq %r16, %r8 -; EGPR-NEXT: adcq %r9, %r20 +; EGPR-NEXT: adcq %r9, %r22 ; EGPR-NEXT: setb %al ; EGPR-NEXT: movzbl %al, %ecx ; EGPR-NEXT: movq %r15, %rax ; EGPR-NEXT: mulq %r14 ; EGPR-NEXT: movq %rdx, %r9 ; EGPR-NEXT: movq %rax, %r16 -; EGPR-NEXT: addq %r20, %r16 +; EGPR-NEXT: addq %r22, %r16 ; EGPR-NEXT: adcq %rcx, %r9 ; EGPR-NEXT: movq %r10, %rax -; EGPR-NEXT: mulq %r27 -; EGPR-NEXT: movq %rdx, %r20 -; EGPR-NEXT: movq %rax, %r25 +; EGPR-NEXT: mulq %r31 +; EGPR-NEXT: movq %rdx, %r22 +; EGPR-NEXT: movq %rax, %r27 ; EGPR-NEXT: movq %rdi, %rax -; EGPR-NEXT: mulq %r27 -; EGPR-NEXT: movq %rdx, %r21 -; EGPR-NEXT: movq %rax, %r22 -; EGPR-NEXT: addq %r20, %r22 -; EGPR-NEXT: adcq $0, %r21 +; EGPR-NEXT: mulq %r31 +; EGPR-NEXT: movq %rdx, %r23 +; EGPR-NEXT: movq %rax, %r24 +; EGPR-NEXT: addq %r22, %r24 +; EGPR-NEXT: adcq $0, %r23 ; EGPR-NEXT: movq %r10, %rax ; EGPR-NEXT: mulq %r14 -; EGPR-NEXT: movq %rdx, %r20 -; EGPR-NEXT: movq %rax, %r28 -; EGPR-NEXT: addq %r22, %r28 -; EGPR-NEXT: adcq %r21, %r20 +; EGPR-NEXT: movq %rdx, %r22 +; EGPR-NEXT: movq %rax, %r20 +; EGPR-NEXT: addq %r24, %r20 +; EGPR-NEXT: adcq %r23, %r22 ; EGPR-NEXT: setb %al ; EGPR-NEXT: movzbl %al, %ecx ; EGPR-NEXT: movq %rdi, %rax ; EGPR-NEXT: mulq %r14 -; EGPR-NEXT: movq %rdx, %r21 -; EGPR-NEXT: movq %rax, %r22 -; EGPR-NEXT: addq %r20, %r22 -; EGPR-NEXT: adcq %rcx, %r21 -; EGPR-NEXT: addq %r19, %r22 -; EGPR-NEXT: adcq %r8, %r21 +; EGPR-NEXT: movq %rdx, %r23 +; EGPR-NEXT: movq %rax, %r24 +; EGPR-NEXT: addq %r22, %r24 +; EGPR-NEXT: adcq %rcx, %r23 +; EGPR-NEXT: addq %r19, %r24 +; EGPR-NEXT: adcq %r8, %r23 ; EGPR-NEXT: adcq $0, %r16 ; EGPR-NEXT: adcq $0, %r9 ; EGPR-NEXT: movq %r10, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; EGPR-NEXT: movq %r10, %rax ; EGPR-NEXT: mulq %r11 ; EGPR-NEXT: movq %rdx, %r8 -; EGPR-NEXT: movq %rax, %r30 +; EGPR-NEXT: movq %rax, %r28 ; EGPR-NEXT: movq %rdi, %rax ; EGPR-NEXT: movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; EGPR-NEXT: mulq %r11 ; EGPR-NEXT: movq %rdx, %r19 -; EGPR-NEXT: movq %rax, %r20 -; EGPR-NEXT: addq %r8, %r20 +; EGPR-NEXT: movq %rax, %r22 +; EGPR-NEXT: addq %r8, %r22 ; EGPR-NEXT: adcq $0, %r19 ; EGPR-NEXT: movq %r10, %rax -; EGPR-NEXT: mulq %r23 +; EGPR-NEXT: mulq %r25 ; EGPR-NEXT: movq %rdx, %rbx -; EGPR-NEXT: movq %rax, %r31 -; EGPR-NEXT: addq %r20, %r31 +; EGPR-NEXT: movq %rax, %r29 +; EGPR-NEXT: addq %r22, %r29 ; EGPR-NEXT: adcq %r19, %rbx ; EGPR-NEXT: setb %al ; EGPR-NEXT: movzbl %al, %ecx ; EGPR-NEXT: movq %rdi, %rax -; EGPR-NEXT: mulq %r23 -; EGPR-NEXT: movq %rdx, %r26 +; EGPR-NEXT: mulq %r25 +; EGPR-NEXT: movq %rdx, %r30 ; EGPR-NEXT: movq %rax, %r8 ; EGPR-NEXT: addq %rbx, %r8 -; EGPR-NEXT: adcq %rcx, %r26 -; EGPR-NEXT: addq %r22, %r30 -; EGPR-NEXT: adcq %r21, %r31 +; EGPR-NEXT: adcq %rcx, %r30 +; EGPR-NEXT: addq %r24, %r28 +; EGPR-NEXT: adcq %r23, %r29 ; EGPR-NEXT: adcq $0, %r8 -; EGPR-NEXT: adcq $0, %r26 +; EGPR-NEXT: adcq $0, %r30 ; EGPR-NEXT: addq %r16, %r8 -; EGPR-NEXT: adcq %r9, %r26 +; EGPR-NEXT: adcq %r9, %r30 ; EGPR-NEXT: setb %al ; EGPR-NEXT: movzbl %al, %ecx ; EGPR-NEXT: movq %r12, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill @@ -122,34 +122,34 @@ define void @test_1024(ptr %a, ptr %b, ptr %out) nounwind { ; EGPR-NEXT: movq %r15, %rax ; EGPR-NEXT: mulq %r11 ; EGPR-NEXT: movq %rdx, %r16 -; EGPR-NEXT: movq %rax, %r21 -; EGPR-NEXT: addq %r9, %r21 +; EGPR-NEXT: movq %rax, %r23 +; EGPR-NEXT: addq %r9, %r23 ; EGPR-NEXT: adcq $0, %r16 ; EGPR-NEXT: movq %r12, %rax -; EGPR-NEXT: mulq %r23 +; EGPR-NEXT: mulq %r25 ; EGPR-NEXT: movq %rdx, %r9 ; EGPR-NEXT: movq %rax, %rdi -; EGPR-NEXT: addq %r21, %rdi +; EGPR-NEXT: addq %r23, %rdi ; EGPR-NEXT: adcq %r16, %r9 ; EGPR-NEXT: setb %al ; EGPR-NEXT: movzbl %al, %r10d ; EGPR-NEXT: movq %r15, %rax -; EGPR-NEXT: mulq %r23 -; EGPR-NEXT: movq %rdx, %r21 -; EGPR-NEXT: movq %rax, %r22 -; EGPR-NEXT: addq %r9, %r22 -; EGPR-NEXT: adcq %r10, %r21 +; EGPR-NEXT: mulq %r25 +; EGPR-NEXT: movq %rdx, %r23 +; EGPR-NEXT: movq %rax, %r24 +; EGPR-NEXT: addq %r9, %r24 +; EGPR-NEXT: adcq %r10, %r23 ; EGPR-NEXT: addq %r8, %rsi ; EGPR-NEXT: movq %rsi, %r19 -; EGPR-NEXT: adcq %r26, %rdi -; EGPR-NEXT: adcq %rcx, %r22 -; EGPR-NEXT: adcq $0, %r21 +; EGPR-NEXT: adcq %r30, %rdi +; EGPR-NEXT: adcq %rcx, %r24 +; EGPR-NEXT: adcq $0, %r23 ; EGPR-NEXT: movq %r17, %rax -; EGPR-NEXT: mulq %r27 +; EGPR-NEXT: mulq %r31 ; EGPR-NEXT: movq %rdx, %r8 ; EGPR-NEXT: movq %rax, %rbx -; EGPR-NEXT: movq %r29, %rax -; EGPR-NEXT: mulq %r27 +; EGPR-NEXT: movq %r21, %rax +; EGPR-NEXT: mulq %r31 ; EGPR-NEXT: movq %rdx, %r9 ; EGPR-NEXT: movq %rax, %r16 ; EGPR-NEXT: addq %r8, %r16 @@ -157,12 +157,12 @@ define void @test_1024(ptr %a, ptr %b, ptr %out) nounwind { ; EGPR-NEXT: movq %r17, %rax ; EGPR-NEXT: mulq %r14 ; EGPR-NEXT: movq %rdx, %r8 -; EGPR-NEXT: movq %rax, %r26 -; EGPR-NEXT: addq %r16, %r26 +; EGPR-NEXT: movq %rax, %r30 +; EGPR-NEXT: addq %r16, %r30 ; EGPR-NEXT: adcq %r9, %r8 ; EGPR-NEXT: setb %al ; EGPR-NEXT: movzbl %al, %ecx -; EGPR-NEXT: movq %r29, %rax +; EGPR-NEXT: movq %r21, %rax ; EGPR-NEXT: mulq %r14 ; EGPR-NEXT: movq %r14, %rsi ; EGPR-NEXT: movq %rdx, %r9 @@ -170,11 +170,11 @@ define void @test_1024(ptr %a, ptr %b, ptr %out) nounwind { ; EGPR-NEXT: addq %r8, %r16 ; EGPR-NEXT: adcq %rcx, %r9 ; EGPR-NEXT: movq %r13, %rax -; EGPR-NEXT: mulq %r27 +; EGPR-NEXT: mulq %r31 ; EGPR-NEXT: movq %rdx, %r8 ; EGPR-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; EGPR-NEXT: movq %r18, %rax -; EGPR-NEXT: mulq %r27 +; EGPR-NEXT: mulq %r31 ; EGPR-NEXT: movq %rdx, %r14 ; EGPR-NEXT: movq %rax, %r15 ; EGPR-NEXT: addq %r8, %r15 @@ -195,40 +195,40 @@ define void @test_1024(ptr %a, ptr %b, ptr %out) nounwind { ; EGPR-NEXT: movzbl %cl, %eax ; EGPR-NEXT: adcq %rax, %r8 ; EGPR-NEXT: addq %rbx, %r15 -; EGPR-NEXT: adcq %r26, %r8 +; EGPR-NEXT: adcq %r30, %r8 ; EGPR-NEXT: adcq $0, %r16 ; EGPR-NEXT: adcq $0, %r9 ; EGPR-NEXT: movq %r13, %rax ; EGPR-NEXT: mulq %r11 -; EGPR-NEXT: movq %rdx, %r26 +; EGPR-NEXT: movq %rdx, %r30 ; EGPR-NEXT: movq %rax, %rsi ; EGPR-NEXT: movq %r18, %rax ; EGPR-NEXT: mulq %r11 ; EGPR-NEXT: movq %rdx, %rbx ; EGPR-NEXT: movq %rax, %r14 -; EGPR-NEXT: addq %r26, %r14 +; EGPR-NEXT: addq %r30, %r14 ; EGPR-NEXT: adcq $0, %rbx ; EGPR-NEXT: movq %r13, %rax -; EGPR-NEXT: mulq %r23 +; EGPR-NEXT: mulq %r25 ; EGPR-NEXT: movq %rdx, %r12 ; EGPR-NEXT: addq %r14, %rax ; EGPR-NEXT: movq %rax, %r10 ; EGPR-NEXT: adcq %rbx, %r12 ; EGPR-NEXT: setb %cl ; EGPR-NEXT: movq %r18, %rax -; EGPR-NEXT: mulq %r23 +; EGPR-NEXT: mulq %r25 ; EGPR-NEXT: movq %rdx, %r14 -; EGPR-NEXT: movq %rax, %r26 -; EGPR-NEXT: addq %r12, %r26 +; EGPR-NEXT: movq %rax, %r30 +; EGPR-NEXT: addq %r12, %r30 ; EGPR-NEXT: movzbl %cl, %eax ; EGPR-NEXT: adcq %rax, %r14 ; EGPR-NEXT: addq %r15, %rsi ; EGPR-NEXT: movq %rsi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; EGPR-NEXT: adcq %r8, %r10 ; EGPR-NEXT: movq %r10, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill -; EGPR-NEXT: adcq $0, %r26 +; EGPR-NEXT: adcq $0, %r30 ; EGPR-NEXT: adcq $0, %r14 -; EGPR-NEXT: addq %r16, %r26 +; EGPR-NEXT: addq %r16, %r30 ; EGPR-NEXT: adcq %r9, %r14 ; EGPR-NEXT: setb %cl ; EGPR-NEXT: movq %r17, %rax @@ -236,48 +236,48 @@ define void @test_1024(ptr %a, ptr %b, ptr %out) nounwind { ; EGPR-NEXT: mulq %r11 ; EGPR-NEXT: movq %rdx, %r8 ; EGPR-NEXT: movq %rax, %rbx -; EGPR-NEXT: movq %r29, %rax +; EGPR-NEXT: movq %r21, %rax ; EGPR-NEXT: mulq %r11 ; EGPR-NEXT: movq %rdx, %r9 ; EGPR-NEXT: movq %rax, %r16 ; EGPR-NEXT: addq %r8, %r16 ; EGPR-NEXT: adcq $0, %r9 ; EGPR-NEXT: movq %r17, %rax -; EGPR-NEXT: mulq %r23 +; EGPR-NEXT: mulq %r25 ; EGPR-NEXT: movq %rdx, %r8 ; EGPR-NEXT: movq %rax, %r15 ; EGPR-NEXT: addq %r16, %r15 ; EGPR-NEXT: adcq %r9, %r8 ; EGPR-NEXT: setb %r9b -; EGPR-NEXT: movq %r29, %rax -; EGPR-NEXT: mulq %r23 +; EGPR-NEXT: movq %r21, %rax +; EGPR-NEXT: mulq %r25 ; EGPR-NEXT: movq %rdx, %r12 ; EGPR-NEXT: movq %rax, %rbp ; EGPR-NEXT: addq %r8, %rbp ; EGPR-NEXT: movzbl %r9b, %eax ; EGPR-NEXT: adcq %rax, %r12 -; EGPR-NEXT: addq %r26, %rbx +; EGPR-NEXT: addq %r30, %rbx ; EGPR-NEXT: adcq %r14, %r15 ; EGPR-NEXT: movzbl %cl, %eax ; EGPR-NEXT: adcq %rax, %rbp ; EGPR-NEXT: adcq $0, %r12 -; EGPR-NEXT: addq %r25, %rbx +; EGPR-NEXT: addq %r27, %rbx ; EGPR-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload -; EGPR-NEXT: movq 32(%rsi), %r25 -; EGPR-NEXT: adcq %r28, %r15 -; EGPR-NEXT: adcq %r30, %rbp -; EGPR-NEXT: adcq %r31, %r12 +; EGPR-NEXT: movq 32(%rsi), %r27 +; EGPR-NEXT: adcq %r20, %r15 +; EGPR-NEXT: adcq %r28, %rbp +; EGPR-NEXT: adcq %r29, %r12 ; EGPR-NEXT: adcq $0, %r19 ; EGPR-NEXT: movq %r19, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; EGPR-NEXT: adcq $0, %rdi -; EGPR-NEXT: adcq $0, %r22 -; EGPR-NEXT: adcq $0, %r21 +; EGPR-NEXT: adcq $0, %r24 +; EGPR-NEXT: adcq $0, %r23 ; EGPR-NEXT: movq %r17, %rax -; EGPR-NEXT: mulq %r25 +; EGPR-NEXT: mulq %r27 ; EGPR-NEXT: movq %rdx, %r8 -; EGPR-NEXT: movq %rax, %r28 -; EGPR-NEXT: movq %r29, %rax -; EGPR-NEXT: mulq %r25 +; EGPR-NEXT: movq %rax, %r20 +; EGPR-NEXT: movq %r21, %rax +; EGPR-NEXT: mulq %r27 ; EGPR-NEXT: movq %rdx, %r9 ; EGPR-NEXT: movq %rax, %r16 ; EGPR-NEXT: addq %r8, %r16 @@ -286,11 +286,11 @@ define void @test_1024(ptr %a, ptr %b, ptr %out) nounwind { ; EGPR-NEXT: movq %r17, %rax ; EGPR-NEXT: mulq %rcx ; EGPR-NEXT: movq %rdx, %r8 -; EGPR-NEXT: movq %rax, %r26 -; EGPR-NEXT: addq %r16, %r26 +; EGPR-NEXT: movq %rax, %r30 +; EGPR-NEXT: addq %r16, %r30 ; EGPR-NEXT: adcq %r9, %r8 ; EGPR-NEXT: setb %r10b -; EGPR-NEXT: movq %r29, %rax +; EGPR-NEXT: movq %r21, %rax ; EGPR-NEXT: mulq %rcx ; EGPR-NEXT: movq %rdx, %r9 ; EGPR-NEXT: movq %rax, %r16 @@ -298,138 +298,138 @@ define void @test_1024(ptr %a, ptr %b, ptr %out) nounwind { ; EGPR-NEXT: movzbl %r10b, %eax ; EGPR-NEXT: adcq %rax, %r9 ; EGPR-NEXT: movq %r13, %rax -; EGPR-NEXT: mulq %r25 +; EGPR-NEXT: mulq %r27 ; EGPR-NEXT: movq %rdx, %r8 ; EGPR-NEXT: movq %rax, %r19 ; EGPR-NEXT: movq %r18, %rax -; EGPR-NEXT: mulq %r25 -; EGPR-NEXT: movq %rdx, %r30 -; EGPR-NEXT: movq %rax, %r31 -; EGPR-NEXT: addq %r8, %r31 -; EGPR-NEXT: adcq $0, %r30 +; EGPR-NEXT: mulq %r27 +; EGPR-NEXT: movq %rdx, %r28 +; EGPR-NEXT: movq %rax, %r29 +; EGPR-NEXT: addq %r8, %r29 +; EGPR-NEXT: adcq $0, %r28 ; EGPR-NEXT: movq %r13, %rax ; EGPR-NEXT: mulq %rcx ; EGPR-NEXT: movq %rdx, %r8 -; EGPR-NEXT: movq %rax, %r20 -; EGPR-NEXT: addq %r31, %r20 -; EGPR-NEXT: adcq %r30, %r8 +; EGPR-NEXT: movq %rax, %r22 +; EGPR-NEXT: addq %r29, %r22 +; EGPR-NEXT: adcq %r28, %r8 ; EGPR-NEXT: setb %r10b ; EGPR-NEXT: movq %r18, %rax ; EGPR-NEXT: mulq %rcx -; EGPR-NEXT: movq %rdx, %r30 -; EGPR-NEXT: movq %rax, %r31 -; EGPR-NEXT: addq %r8, %r31 +; EGPR-NEXT: movq %rdx, %r28 +; EGPR-NEXT: movq %rax, %r29 +; EGPR-NEXT: addq %r8, %r29 ; EGPR-NEXT: movzbl %r10b, %eax -; EGPR-NEXT: adcq %rax, %r30 -; EGPR-NEXT: addq %r28, %r31 -; EGPR-NEXT: adcq %r26, %r30 +; EGPR-NEXT: adcq %rax, %r28 +; EGPR-NEXT: addq %r20, %r29 +; EGPR-NEXT: adcq %r30, %r28 ; EGPR-NEXT: adcq $0, %r16 ; EGPR-NEXT: adcq $0, %r9 -; EGPR-NEXT: movq 48(%rsi), %r28 +; EGPR-NEXT: movq 48(%rsi), %r20 ; EGPR-NEXT: movq %r13, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; EGPR-NEXT: movq %r13, %rax -; EGPR-NEXT: mulq %r28 +; EGPR-NEXT: mulq %r20 ; EGPR-NEXT: movq %rdx, %r8 ; EGPR-NEXT: movq %rax, %r11 ; EGPR-NEXT: movq %r18, %rax ; EGPR-NEXT: movq %r18, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill -; EGPR-NEXT: mulq %r28 -; EGPR-NEXT: movq %rdx, %r26 +; EGPR-NEXT: mulq %r20 +; EGPR-NEXT: movq %rdx, %r30 ; EGPR-NEXT: movq %rax, %r14 ; EGPR-NEXT: addq %r8, %r14 -; EGPR-NEXT: adcq $0, %r26 +; EGPR-NEXT: adcq $0, %r30 ; EGPR-NEXT: movq 56(%rsi), %r10 ; EGPR-NEXT: movq %r13, %rax ; EGPR-NEXT: mulq %r10 ; EGPR-NEXT: movq %rdx, %r13 ; EGPR-NEXT: addq %r14, %rax ; EGPR-NEXT: movq %rax, %r14 -; EGPR-NEXT: adcq %r26, %r13 +; EGPR-NEXT: adcq %r30, %r13 ; EGPR-NEXT: setb %sil ; EGPR-NEXT: movq %r18, %rax ; EGPR-NEXT: mulq %r10 -; EGPR-NEXT: movq %rdx, %r26 +; EGPR-NEXT: movq %rdx, %r30 ; EGPR-NEXT: movq %rax, %r8 ; EGPR-NEXT: addq %r13, %r8 ; EGPR-NEXT: movzbl %sil, %eax -; EGPR-NEXT: adcq %rax, %r26 -; EGPR-NEXT: addq %r31, %r11 -; EGPR-NEXT: adcq %r30, %r14 +; EGPR-NEXT: adcq %rax, %r30 +; EGPR-NEXT: addq %r29, %r11 +; EGPR-NEXT: adcq %r28, %r14 ; EGPR-NEXT: adcq $0, %r8 -; EGPR-NEXT: adcq $0, %r26 +; EGPR-NEXT: adcq $0, %r30 ; EGPR-NEXT: addq %r16, %r8 -; EGPR-NEXT: adcq %r9, %r26 +; EGPR-NEXT: adcq %r9, %r30 ; EGPR-NEXT: setb %r18b ; EGPR-NEXT: movq %r17, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; EGPR-NEXT: movq %r17, %rax -; EGPR-NEXT: mulq %r28 +; EGPR-NEXT: mulq %r20 ; EGPR-NEXT: movq %rdx, %r9 -; EGPR-NEXT: movq %rax, %r30 -; EGPR-NEXT: movq %r29, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill -; EGPR-NEXT: movq %r29, %rax -; EGPR-NEXT: mulq %r28 +; EGPR-NEXT: movq %rax, %r28 +; EGPR-NEXT: movq %r21, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill +; EGPR-NEXT: movq %r21, %rax +; EGPR-NEXT: mulq %r20 ; EGPR-NEXT: movq %rdx, %r16 -; EGPR-NEXT: movq %rax, %r31 -; EGPR-NEXT: addq %r9, %r31 +; EGPR-NEXT: movq %rax, %r29 +; EGPR-NEXT: addq %r9, %r29 ; EGPR-NEXT: adcq $0, %r16 ; EGPR-NEXT: movq %r17, %rax ; EGPR-NEXT: mulq %r10 ; EGPR-NEXT: movq %rdx, %r9 ; EGPR-NEXT: movq %rax, %r17 -; EGPR-NEXT: addq %r31, %r17 +; EGPR-NEXT: addq %r29, %r17 ; EGPR-NEXT: adcq %r16, %r9 ; EGPR-NEXT: setb %r16b -; EGPR-NEXT: movq %r29, %rax +; EGPR-NEXT: movq %r21, %rax ; EGPR-NEXT: mulq %r10 ; EGPR-NEXT: movq %rdx, %r13 -; EGPR-NEXT: movq %rax, %r31 -; EGPR-NEXT: addq %r9, %r31 +; EGPR-NEXT: movq %rax, %r29 +; EGPR-NEXT: addq %r9, %r29 ; EGPR-NEXT: movzbl %r16b, %eax ; EGPR-NEXT: adcq %rax, %r13 -; EGPR-NEXT: addq %r8, %r30 -; EGPR-NEXT: adcq %r26, %r17 +; EGPR-NEXT: addq %r8, %r28 +; EGPR-NEXT: adcq %r30, %r17 ; EGPR-NEXT: movzbl %r18b, %eax -; EGPR-NEXT: adcq %rax, %r31 +; EGPR-NEXT: adcq %rax, %r29 ; EGPR-NEXT: adcq $0, %r13 ; EGPR-NEXT: addq %rbx, %r19 ; EGPR-NEXT: movq %r19, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill -; EGPR-NEXT: adcq %r15, %r20 -; EGPR-NEXT: movq %r20, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill +; EGPR-NEXT: adcq %r15, %r22 +; EGPR-NEXT: movq %r22, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; EGPR-NEXT: adcq %rbp, %r11 ; EGPR-NEXT: movq %r11, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; EGPR-NEXT: adcq %r12, %r14 ; EGPR-NEXT: movq %r14, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill -; EGPR-NEXT: adcq $0, %r30 +; EGPR-NEXT: adcq $0, %r28 ; EGPR-NEXT: adcq $0, %r17 -; EGPR-NEXT: adcq $0, %r31 +; EGPR-NEXT: adcq $0, %r29 ; EGPR-NEXT: adcq $0, %r13 -; EGPR-NEXT: addq {{[-0-9]+}}(%r{{[sb]}}p), %r30 # 8-byte Folded Reload +; EGPR-NEXT: addq {{[-0-9]+}}(%r{{[sb]}}p), %r28 # 8-byte Folded Reload ; EGPR-NEXT: adcq %rdi, %r17 -; EGPR-NEXT: adcq %r22, %r31 -; EGPR-NEXT: adcq %r21, %r13 +; EGPR-NEXT: adcq %r24, %r29 +; EGPR-NEXT: adcq %r23, %r13 ; EGPR-NEXT: setb %r15b ; EGPR-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload ; EGPR-NEXT: movq %rsi, %rax -; EGPR-NEXT: mulq %r25 +; EGPR-NEXT: mulq %r27 ; EGPR-NEXT: movq %rdx, %r8 ; EGPR-NEXT: movq %rax, %r19 -; EGPR-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r21 # 8-byte Reload -; EGPR-NEXT: movq %r21, %rax -; EGPR-NEXT: mulq %r25 +; EGPR-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r23 # 8-byte Reload +; EGPR-NEXT: movq %r23, %rax +; EGPR-NEXT: mulq %r27 ; EGPR-NEXT: movq %rdx, %r9 ; EGPR-NEXT: movq %rax, %r16 ; EGPR-NEXT: addq %r8, %r16 ; EGPR-NEXT: adcq $0, %r9 ; EGPR-NEXT: movq %rsi, %rax -; EGPR-NEXT: movq %rsi, %r29 +; EGPR-NEXT: movq %rsi, %r21 ; EGPR-NEXT: mulq %rcx ; EGPR-NEXT: movq %rdx, %r8 -; EGPR-NEXT: movq %rax, %r20 -; EGPR-NEXT: addq %r16, %r20 +; EGPR-NEXT: movq %rax, %r22 +; EGPR-NEXT: addq %r16, %r22 ; EGPR-NEXT: adcq %r9, %r8 ; EGPR-NEXT: setb %r18b -; EGPR-NEXT: movq %r21, %rax -; EGPR-NEXT: movq %r21, %r14 +; EGPR-NEXT: movq %r23, %rax +; EGPR-NEXT: movq %r23, %r14 ; EGPR-NEXT: mulq %rcx ; EGPR-NEXT: movq %rdx, %r9 ; EGPR-NEXT: movq %rax, %r16 @@ -438,77 +438,77 @@ define void @test_1024(ptr %a, ptr %b, ptr %out) nounwind { ; EGPR-NEXT: adcq %rax, %r9 ; EGPR-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rbx # 8-byte Reload ; EGPR-NEXT: movq %rbx, %rax -; EGPR-NEXT: mulq %r25 +; EGPR-NEXT: mulq %r27 ; EGPR-NEXT: movq %rdx, %r8 ; EGPR-NEXT: movq %rax, %rdi ; EGPR-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload ; EGPR-NEXT: movq %rsi, %rax -; EGPR-NEXT: mulq %r25 -; EGPR-NEXT: movq %rdx, %r21 -; EGPR-NEXT: movq %rax, %r22 -; EGPR-NEXT: addq %r8, %r22 -; EGPR-NEXT: adcq $0, %r21 +; EGPR-NEXT: mulq %r27 +; EGPR-NEXT: movq %rdx, %r23 +; EGPR-NEXT: movq %rax, %r24 +; EGPR-NEXT: addq %r8, %r24 +; EGPR-NEXT: adcq $0, %r23 ; EGPR-NEXT: movq %rbx, %rax ; EGPR-NEXT: mulq %rcx ; EGPR-NEXT: movq %rdx, %r8 -; EGPR-NEXT: addq %r22, %rax +; EGPR-NEXT: addq %r24, %rax ; EGPR-NEXT: movq %rax, %r11 -; EGPR-NEXT: adcq %r21, %r8 +; EGPR-NEXT: adcq %r23, %r8 ; EGPR-NEXT: setb %r18b ; EGPR-NEXT: movq %rsi, %rax -; EGPR-NEXT: movq %rsi, %r21 +; EGPR-NEXT: movq %rsi, %r23 ; EGPR-NEXT: mulq %rcx -; EGPR-NEXT: movq %rdx, %r22 -; EGPR-NEXT: movq %rax, %r26 -; EGPR-NEXT: addq %r8, %r26 +; EGPR-NEXT: movq %rdx, %r24 +; EGPR-NEXT: movq %rax, %r30 +; EGPR-NEXT: addq %r8, %r30 ; EGPR-NEXT: movzbl %r18b, %eax -; EGPR-NEXT: adcq %rax, %r22 -; EGPR-NEXT: addq %r19, %r26 -; EGPR-NEXT: adcq %r20, %r22 +; EGPR-NEXT: adcq %rax, %r24 +; EGPR-NEXT: addq %r19, %r30 +; EGPR-NEXT: adcq %r22, %r24 ; EGPR-NEXT: adcq $0, %r16 ; EGPR-NEXT: adcq $0, %r9 ; EGPR-NEXT: movq %rbx, %rax -; EGPR-NEXT: mulq %r28 +; EGPR-NEXT: mulq %r20 ; EGPR-NEXT: movq %rdx, %r8 ; EGPR-NEXT: movq %rax, %rsi -; EGPR-NEXT: movq %r21, %rax -; EGPR-NEXT: mulq %r28 +; EGPR-NEXT: movq %r23, %rax +; EGPR-NEXT: mulq %r20 ; EGPR-NEXT: movq %rdx, %r19 -; EGPR-NEXT: movq %rax, %r20 -; EGPR-NEXT: addq %r8, %r20 +; EGPR-NEXT: movq %rax, %r22 +; EGPR-NEXT: addq %r8, %r22 ; EGPR-NEXT: adcq $0, %r19 ; EGPR-NEXT: movq %rbx, %rax ; EGPR-NEXT: mulq %r10 ; EGPR-NEXT: movq %rdx, %rbx -; EGPR-NEXT: addq %r20, %rax -; EGPR-NEXT: movq %rax, %r20 +; EGPR-NEXT: addq %r22, %rax +; EGPR-NEXT: movq %rax, %r22 ; EGPR-NEXT: adcq %r19, %rbx ; EGPR-NEXT: setb %r18b -; EGPR-NEXT: movq %r21, %rax +; EGPR-NEXT: movq %r23, %rax ; EGPR-NEXT: mulq %r10 -; EGPR-NEXT: movq %rdx, %r21 +; EGPR-NEXT: movq %rdx, %r23 ; EGPR-NEXT: movq %rax, %r8 ; EGPR-NEXT: addq %rbx, %r8 ; EGPR-NEXT: movzbl %r18b, %eax -; EGPR-NEXT: adcq %rax, %r21 -; EGPR-NEXT: addq %r26, %rsi -; EGPR-NEXT: adcq %r22, %r20 +; EGPR-NEXT: adcq %rax, %r23 +; EGPR-NEXT: addq %r30, %rsi +; EGPR-NEXT: adcq %r24, %r22 ; EGPR-NEXT: adcq $0, %r8 -; EGPR-NEXT: adcq $0, %r21 +; EGPR-NEXT: adcq $0, %r23 ; EGPR-NEXT: addq %r16, %r8 -; EGPR-NEXT: adcq %r9, %r21 +; EGPR-NEXT: adcq %r9, %r23 ; EGPR-NEXT: setb %r18b -; EGPR-NEXT: movq %r29, %rax -; EGPR-NEXT: mulq %r28 +; EGPR-NEXT: movq %r21, %rax +; EGPR-NEXT: mulq %r20 ; EGPR-NEXT: movq %rdx, %r9 -; EGPR-NEXT: movq %rax, %r22 +; EGPR-NEXT: movq %rax, %r24 ; EGPR-NEXT: movq %r14, %rax -; EGPR-NEXT: mulq %r28 +; EGPR-NEXT: mulq %r20 ; EGPR-NEXT: movq %rdx, %r16 ; EGPR-NEXT: movq %rax, %r19 ; EGPR-NEXT: addq %r9, %r19 ; EGPR-NEXT: adcq $0, %r16 -; EGPR-NEXT: movq %r29, %rax +; EGPR-NEXT: movq %r21, %rax ; EGPR-NEXT: mulq %r10 ; EGPR-NEXT: movq %rdx, %r9 ; EGPR-NEXT: addq %r19, %rax @@ -522,121 +522,121 @@ define void @test_1024(ptr %a, ptr %b, ptr %out) nounwind { ; EGPR-NEXT: addq %r9, %r12 ; EGPR-NEXT: movzbl %r16b, %eax ; EGPR-NEXT: adcq %rax, %rbp -; EGPR-NEXT: addq %r8, %r22 -; EGPR-NEXT: adcq %r21, %r19 +; EGPR-NEXT: addq %r8, %r24 +; EGPR-NEXT: adcq %r23, %r19 ; EGPR-NEXT: movzbl %r18b, %eax ; EGPR-NEXT: adcq %rax, %r12 ; EGPR-NEXT: adcq $0, %rbp -; EGPR-NEXT: addq %r30, %rdi +; EGPR-NEXT: addq %r28, %rdi ; EGPR-NEXT: movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; EGPR-NEXT: adcq %r17, %r11 ; EGPR-NEXT: movq %r11, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill -; EGPR-NEXT: adcq %r31, %rsi +; EGPR-NEXT: adcq %r29, %rsi ; EGPR-NEXT: movq %rsi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill -; EGPR-NEXT: adcq %r13, %r20 -; EGPR-NEXT: movq %r20, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill +; EGPR-NEXT: adcq %r13, %r22 +; EGPR-NEXT: movq %r22, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; EGPR-NEXT: movzbl %r15b, %eax -; EGPR-NEXT: adcq %rax, %r22 -; EGPR-NEXT: movq %r22, (%rsp) # 8-byte Spill +; EGPR-NEXT: adcq %rax, %r24 +; EGPR-NEXT: movq %r24, (%rsp) # 8-byte Spill ; EGPR-NEXT: adcq $0, %r19 ; EGPR-NEXT: movq %r19, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; EGPR-NEXT: adcq $0, %r12 ; EGPR-NEXT: adcq $0, %rbp -; EGPR-NEXT: movq 64(%r24), %r21 +; EGPR-NEXT: movq 64(%r26), %r23 ; EGPR-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload ; EGPR-NEXT: movq %rdi, %rax -; EGPR-NEXT: mulq %r21 +; EGPR-NEXT: mulq %r23 ; EGPR-NEXT: movq %rdx, %r8 -; EGPR-NEXT: movq %rax, %r22 -; EGPR-NEXT: movq %r23, %rax -; EGPR-NEXT: mulq %r21 +; EGPR-NEXT: movq %rax, %r24 +; EGPR-NEXT: movq %r25, %rax +; EGPR-NEXT: mulq %r23 ; EGPR-NEXT: movq %rdx, %r9 ; EGPR-NEXT: movq %rax, %r16 ; EGPR-NEXT: addq %r8, %r16 ; EGPR-NEXT: adcq $0, %r9 -; EGPR-NEXT: movq 72(%r24), %r30 +; EGPR-NEXT: movq 72(%r26), %r28 ; EGPR-NEXT: movq %rdi, %rax -; EGPR-NEXT: mulq %r30 +; EGPR-NEXT: mulq %r28 ; EGPR-NEXT: movq %rdx, %r8 -; EGPR-NEXT: movq %rax, %r26 -; EGPR-NEXT: addq %r16, %r26 +; EGPR-NEXT: movq %rax, %r30 +; EGPR-NEXT: addq %r16, %r30 ; EGPR-NEXT: adcq %r9, %r8 ; EGPR-NEXT: setb %r18b -; EGPR-NEXT: movq %r23, %rax -; EGPR-NEXT: mulq %r30 +; EGPR-NEXT: movq %r25, %rax +; EGPR-NEXT: mulq %r28 ; EGPR-NEXT: movq %rdx, %r9 ; EGPR-NEXT: movq %rax, %r16 ; EGPR-NEXT: addq %r8, %r16 ; EGPR-NEXT: movzbl %r18b, %eax ; EGPR-NEXT: adcq %rax, %r9 -; EGPR-NEXT: movq %r27, %rax -; EGPR-NEXT: mulq %r21 +; EGPR-NEXT: movq %r31, %rax +; EGPR-NEXT: mulq %r23 ; EGPR-NEXT: movq %rdx, %r8 ; EGPR-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; EGPR-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r11 # 8-byte Reload ; EGPR-NEXT: movq %r11, %rax -; EGPR-NEXT: mulq %r21 -; EGPR-NEXT: movq %rdx, %r31 +; EGPR-NEXT: mulq %r23 +; EGPR-NEXT: movq %rdx, %r29 ; EGPR-NEXT: movq %rax, %rbx ; EGPR-NEXT: addq %r8, %rbx -; EGPR-NEXT: adcq $0, %r31 -; EGPR-NEXT: movq %r27, %rax -; EGPR-NEXT: mulq %r30 +; EGPR-NEXT: adcq $0, %r29 +; EGPR-NEXT: movq %r31, %rax +; EGPR-NEXT: mulq %r28 ; EGPR-NEXT: movq %rdx, %r8 ; EGPR-NEXT: addq %rbx, %rax ; EGPR-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill -; EGPR-NEXT: adcq %r31, %r8 +; EGPR-NEXT: adcq %r29, %r8 ; EGPR-NEXT: setb %r18b ; EGPR-NEXT: movq %r11, %rax -; EGPR-NEXT: mulq %r30 -; EGPR-NEXT: movq %rdx, %r31 +; EGPR-NEXT: mulq %r28 +; EGPR-NEXT: movq %rdx, %r29 ; EGPR-NEXT: movq %rax, %rbx ; EGPR-NEXT: addq %r8, %rbx ; EGPR-NEXT: movzbl %r18b, %eax -; EGPR-NEXT: adcq %rax, %r31 -; EGPR-NEXT: addq %r22, %rbx -; EGPR-NEXT: adcq %r26, %r31 +; EGPR-NEXT: adcq %rax, %r29 +; EGPR-NEXT: addq %r24, %rbx +; EGPR-NEXT: adcq %r30, %r29 ; EGPR-NEXT: adcq $0, %r16 ; EGPR-NEXT: adcq $0, %r9 -; EGPR-NEXT: movq 80(%r24), %r13 -; EGPR-NEXT: movq %r27, %rax +; EGPR-NEXT: movq 80(%r26), %r13 +; EGPR-NEXT: movq %r31, %rax ; EGPR-NEXT: mulq %r13 ; EGPR-NEXT: movq %rdx, %r8 ; EGPR-NEXT: movq %rax, %rsi ; EGPR-NEXT: movq %r11, %rax ; EGPR-NEXT: mulq %r13 -; EGPR-NEXT: movq %rdx, %r26 +; EGPR-NEXT: movq %rdx, %r30 ; EGPR-NEXT: movq %rax, %r14 ; EGPR-NEXT: addq %r8, %r14 -; EGPR-NEXT: adcq $0, %r26 -; EGPR-NEXT: movq 88(%r24), %r18 -; EGPR-NEXT: movq %r27, %rax +; EGPR-NEXT: adcq $0, %r30 +; EGPR-NEXT: movq 88(%r26), %r18 +; EGPR-NEXT: movq %r31, %rax ; EGPR-NEXT: mulq %r18 ; EGPR-NEXT: movq %rdx, %r15 -; EGPR-NEXT: movq %rax, %r22 -; EGPR-NEXT: addq %r14, %r22 -; EGPR-NEXT: adcq %r26, %r15 +; EGPR-NEXT: movq %rax, %r24 +; EGPR-NEXT: addq %r14, %r24 +; EGPR-NEXT: adcq %r30, %r15 ; EGPR-NEXT: setb %r14b ; EGPR-NEXT: movq %r11, %rax ; EGPR-NEXT: mulq %r18 -; EGPR-NEXT: movq %rdx, %r26 +; EGPR-NEXT: movq %rdx, %r30 ; EGPR-NEXT: movq %rax, %r8 ; EGPR-NEXT: addq %r15, %r8 ; EGPR-NEXT: movzbl %r14b, %eax -; EGPR-NEXT: adcq %rax, %r26 +; EGPR-NEXT: adcq %rax, %r30 ; EGPR-NEXT: addq %rbx, %rsi ; EGPR-NEXT: movq %rsi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill -; EGPR-NEXT: adcq %r31, %r22 +; EGPR-NEXT: adcq %r29, %r24 ; EGPR-NEXT: adcq $0, %r8 -; EGPR-NEXT: adcq $0, %r26 +; EGPR-NEXT: adcq $0, %r30 ; EGPR-NEXT: addq %r16, %r8 -; EGPR-NEXT: adcq %r9, %r26 -; EGPR-NEXT: setb %r31b +; EGPR-NEXT: adcq %r9, %r30 +; EGPR-NEXT: setb %r29b ; EGPR-NEXT: movq %rdi, %rax ; EGPR-NEXT: mulq %r13 ; EGPR-NEXT: movq %rdx, %r9 ; EGPR-NEXT: movq %rax, %rsi -; EGPR-NEXT: movq %r23, %rax +; EGPR-NEXT: movq %r25, %rax ; EGPR-NEXT: mulq %r13 ; EGPR-NEXT: movq %rdx, %r16 ; EGPR-NEXT: movq %rax, %r14 @@ -649,7 +649,7 @@ define void @test_1024(ptr %a, ptr %b, ptr %out) nounwind { ; EGPR-NEXT: addq %r14, %rbx ; EGPR-NEXT: adcq %r16, %r9 ; EGPR-NEXT: setb %r16b -; EGPR-NEXT: movq %r23, %rax +; EGPR-NEXT: movq %r25, %rax ; EGPR-NEXT: mulq %r18 ; EGPR-NEXT: movq %rdx, %r14 ; EGPR-NEXT: movq %rax, %r15 @@ -657,116 +657,116 @@ define void @test_1024(ptr %a, ptr %b, ptr %out) nounwind { ; EGPR-NEXT: movzbl %r16b, %eax ; EGPR-NEXT: adcq %rax, %r14 ; EGPR-NEXT: addq %r8, %rsi -; EGPR-NEXT: adcq %r26, %rbx -; EGPR-NEXT: movzbl %r31b, %eax +; EGPR-NEXT: adcq %r30, %rbx +; EGPR-NEXT: movzbl %r29b, %eax ; EGPR-NEXT: adcq %rax, %r15 ; EGPR-NEXT: adcq $0, %r14 -; EGPR-NEXT: imulq %r25, %r18 -; EGPR-NEXT: movq %r25, %rax +; EGPR-NEXT: imulq %r27, %r18 +; EGPR-NEXT: movq %r27, %rax ; EGPR-NEXT: mulq %r13 ; EGPR-NEXT: movq %rax, %r8 ; EGPR-NEXT: addq %r18, %rdx ; EGPR-NEXT: imulq %rcx, %r13 ; EGPR-NEXT: addq %rdx, %r13 -; EGPR-NEXT: movq %r28, %r9 -; EGPR-NEXT: imulq %r30, %r9 -; EGPR-NEXT: movq %r28, %rax -; EGPR-NEXT: mulq %r21 -; EGPR-NEXT: movq %rax, %r26 +; EGPR-NEXT: movq %r20, %r9 +; EGPR-NEXT: imulq %r28, %r9 +; EGPR-NEXT: movq %r20, %rax +; EGPR-NEXT: mulq %r23 +; EGPR-NEXT: movq %rax, %r30 ; EGPR-NEXT: addq %r9, %rdx -; EGPR-NEXT: imulq %r21, %r10 +; EGPR-NEXT: imulq %r23, %r10 ; EGPR-NEXT: addq %rdx, %r10 -; EGPR-NEXT: addq %r8, %r26 +; EGPR-NEXT: addq %r8, %r30 ; EGPR-NEXT: adcq %r13, %r10 -; EGPR-NEXT: movq %r21, %rax -; EGPR-NEXT: mulq %r25 +; EGPR-NEXT: movq %r23, %rax +; EGPR-NEXT: mulq %r27 ; EGPR-NEXT: movq %rdx, %r8 ; EGPR-NEXT: movq %rax, %r9 -; EGPR-NEXT: movq %r30, %rax -; EGPR-NEXT: mulq %r25 -; EGPR-NEXT: movq %rdx, %r25 -; EGPR-NEXT: movq %rax, %r28 -; EGPR-NEXT: addq %r8, %r28 -; EGPR-NEXT: adcq $0, %r25 -; EGPR-NEXT: movq %r21, %rax +; EGPR-NEXT: movq %r28, %rax +; EGPR-NEXT: mulq %r27 +; EGPR-NEXT: movq %rdx, %r27 +; EGPR-NEXT: movq %rax, %r20 +; EGPR-NEXT: addq %r8, %r20 +; EGPR-NEXT: adcq $0, %r27 +; EGPR-NEXT: movq %r23, %rax ; EGPR-NEXT: mulq %rcx ; EGPR-NEXT: movq %rdx, %r8 ; EGPR-NEXT: movq %rax, %r16 -; EGPR-NEXT: addq %r28, %r16 -; EGPR-NEXT: adcq %r25, %r8 +; EGPR-NEXT: addq %r20, %r16 +; EGPR-NEXT: adcq %r27, %r8 ; EGPR-NEXT: setb %r18b -; EGPR-NEXT: movq %r30, %rax +; EGPR-NEXT: movq %r28, %rax ; EGPR-NEXT: mulq %rcx -; EGPR-NEXT: movq %rdx, %r21 -; EGPR-NEXT: movq %rax, %r28 -; EGPR-NEXT: addq %r8, %r28 +; EGPR-NEXT: movq %rdx, %r23 +; EGPR-NEXT: movq %rax, %r20 +; EGPR-NEXT: addq %r8, %r20 ; EGPR-NEXT: movzbl %r18b, %eax -; EGPR-NEXT: adcq %rax, %r21 -; EGPR-NEXT: addq %r26, %r28 -; EGPR-NEXT: adcq %r10, %r21 -; EGPR-NEXT: movq 112(%r24), %rcx -; EGPR-NEXT: movq %r27, %rax +; EGPR-NEXT: adcq %rax, %r23 +; EGPR-NEXT: addq %r30, %r20 +; EGPR-NEXT: adcq %r10, %r23 +; EGPR-NEXT: movq 112(%r26), %rcx +; EGPR-NEXT: movq %r31, %rax ; EGPR-NEXT: mulq %rcx ; EGPR-NEXT: movq %rax, %r8 ; EGPR-NEXT: imulq %r11, %rcx ; EGPR-NEXT: addq %rdx, %rcx -; EGPR-NEXT: movq 120(%r24), %rax -; EGPR-NEXT: imulq %r27, %rax +; EGPR-NEXT: movq 120(%r26), %rax +; EGPR-NEXT: imulq %r31, %rax ; EGPR-NEXT: addq %rax, %rcx -; EGPR-NEXT: movq 96(%r24), %r25 -; EGPR-NEXT: movq 104(%r24), %r26 +; EGPR-NEXT: movq 96(%r26), %r27 +; EGPR-NEXT: movq 104(%r26), %r30 ; EGPR-NEXT: movq %rdi, %rax -; EGPR-NEXT: imulq %r26, %rdi -; EGPR-NEXT: mulq %r25 -; EGPR-NEXT: movq %rax, %r29 -; EGPR-NEXT: addq %rdi, %rdx -; EGPR-NEXT: imulq %r25, %r23 -; EGPR-NEXT: addq %rdx, %r23 -; EGPR-NEXT: addq %r8, %r29 -; EGPR-NEXT: adcq %rcx, %r23 -; EGPR-NEXT: movq %r25, %rax +; EGPR-NEXT: imulq %r30, %rdi ; EGPR-NEXT: mulq %r27 +; EGPR-NEXT: movq %rax, %r21 +; EGPR-NEXT: addq %rdi, %rdx +; EGPR-NEXT: imulq %r27, %r25 +; EGPR-NEXT: addq %rdx, %r25 +; EGPR-NEXT: addq %r8, %r21 +; EGPR-NEXT: adcq %rcx, %r25 +; EGPR-NEXT: movq %r27, %rax +; EGPR-NEXT: mulq %r31 ; EGPR-NEXT: movq %rdx, %r8 -; EGPR-NEXT: movq %rax, %r20 -; EGPR-NEXT: movq %r26, %rax -; EGPR-NEXT: mulq %r27 -; EGPR-NEXT: movq %rdx, %r27 -; EGPR-NEXT: movq %rax, %r30 -; EGPR-NEXT: addq %r8, %r30 -; EGPR-NEXT: adcq $0, %r27 -; EGPR-NEXT: movq %r25, %rax +; EGPR-NEXT: movq %rax, %r22 +; EGPR-NEXT: movq %r30, %rax +; EGPR-NEXT: mulq %r31 +; EGPR-NEXT: movq %rdx, %r31 +; EGPR-NEXT: movq %rax, %r28 +; EGPR-NEXT: addq %r8, %r28 +; EGPR-NEXT: adcq $0, %r31 +; EGPR-NEXT: movq %r27, %rax ; EGPR-NEXT: mulq %r11 ; EGPR-NEXT: movq %rdx, %r8 -; EGPR-NEXT: movq %rax, %r25 -; EGPR-NEXT: addq %r30, %r25 -; EGPR-NEXT: adcq %r27, %r8 +; EGPR-NEXT: movq %rax, %r27 +; EGPR-NEXT: addq %r28, %r27 +; EGPR-NEXT: adcq %r31, %r8 ; EGPR-NEXT: setb %cl -; EGPR-NEXT: movq %r26, %rax +; EGPR-NEXT: movq %r30, %rax ; EGPR-NEXT: mulq %r11 -; EGPR-NEXT: movq %rdx, %r24 -; EGPR-NEXT: movq %rax, %r27 -; EGPR-NEXT: addq %r8, %r27 +; EGPR-NEXT: movq %rdx, %r26 +; EGPR-NEXT: movq %rax, %r31 +; EGPR-NEXT: addq %r8, %r31 ; EGPR-NEXT: movzbl %cl, %eax -; EGPR-NEXT: adcq %rax, %r24 -; EGPR-NEXT: addq %r29, %r27 -; EGPR-NEXT: adcq %r23, %r24 -; EGPR-NEXT: addq %r9, %r20 -; EGPR-NEXT: adcq %r16, %r25 -; EGPR-NEXT: adcq %r28, %r27 -; EGPR-NEXT: adcq %r21, %r24 -; EGPR-NEXT: addq %rsi, %r20 -; EGPR-NEXT: adcq %rbx, %r25 -; EGPR-NEXT: adcq %r15, %r27 -; EGPR-NEXT: adcq %r14, %r24 +; EGPR-NEXT: adcq %rax, %r26 +; EGPR-NEXT: addq %r21, %r31 +; EGPR-NEXT: adcq %r25, %r26 +; EGPR-NEXT: addq %r9, %r22 +; EGPR-NEXT: adcq %r16, %r27 +; EGPR-NEXT: adcq %r20, %r31 +; EGPR-NEXT: adcq %r23, %r26 +; EGPR-NEXT: addq %rsi, %r22 +; EGPR-NEXT: adcq %rbx, %r27 +; EGPR-NEXT: adcq %r15, %r31 +; EGPR-NEXT: adcq %r14, %r26 ; EGPR-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r11 # 8-byte Reload ; EGPR-NEXT: movq 80(%r11), %rbx ; EGPR-NEXT: movq %rbx, %rax ; EGPR-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r19 # 8-byte Reload ; EGPR-NEXT: mulq %r19 -; EGPR-NEXT: movq %rax, %r21 +; EGPR-NEXT: movq %rax, %r23 ; EGPR-NEXT: movq %rdx, %r8 -; EGPR-NEXT: movq 88(%r11), %r28 -; EGPR-NEXT: movq %r28, %rax +; EGPR-NEXT: movq 88(%r11), %r20 +; EGPR-NEXT: movq %r20, %rax ; EGPR-NEXT: mulq %r19 ; EGPR-NEXT: movq %rdx, %r9 ; EGPR-NEXT: movq %rax, %r16 @@ -776,11 +776,11 @@ define void @test_1024(ptr %a, ptr %b, ptr %out) nounwind { ; EGPR-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r17 # 8-byte Reload ; EGPR-NEXT: mulq %r17 ; EGPR-NEXT: movq %rdx, %r8 -; EGPR-NEXT: movq %rax, %r26 -; EGPR-NEXT: addq %r16, %r26 +; EGPR-NEXT: movq %rax, %r30 +; EGPR-NEXT: addq %r16, %r30 ; EGPR-NEXT: adcq %r9, %r8 ; EGPR-NEXT: setb %cl -; EGPR-NEXT: movq %r28, %rax +; EGPR-NEXT: movq %r20, %rax ; EGPR-NEXT: mulq %r17 ; EGPR-NEXT: movq %rdx, %r9 ; EGPR-NEXT: movq %rax, %r16 @@ -790,71 +790,71 @@ define void @test_1024(ptr %a, ptr %b, ptr %out) nounwind { ; EGPR-NEXT: movq 64(%r11), %r15 ; EGPR-NEXT: movq %r15, %rax ; EGPR-NEXT: mulq %r19 -; EGPR-NEXT: movq %rax, %r23 +; EGPR-NEXT: movq %rax, %r25 ; EGPR-NEXT: movq %rdx, %r8 ; EGPR-NEXT: movq 72(%r11), %r14 ; EGPR-NEXT: movq %r14, %rax ; EGPR-NEXT: mulq %r19 -; EGPR-NEXT: movq %rdx, %r30 -; EGPR-NEXT: movq %rax, %r31 -; EGPR-NEXT: addq %r8, %r31 -; EGPR-NEXT: adcq $0, %r30 +; EGPR-NEXT: movq %rdx, %r28 +; EGPR-NEXT: movq %rax, %r29 +; EGPR-NEXT: addq %r8, %r29 +; EGPR-NEXT: adcq $0, %r28 ; EGPR-NEXT: movq %r15, %rax ; EGPR-NEXT: mulq %r17 ; EGPR-NEXT: movq %rdx, %r8 -; EGPR-NEXT: movq %rax, %r29 -; EGPR-NEXT: addq %r31, %r29 -; EGPR-NEXT: adcq %r30, %r8 +; EGPR-NEXT: movq %rax, %r21 +; EGPR-NEXT: addq %r29, %r21 +; EGPR-NEXT: adcq %r28, %r8 ; EGPR-NEXT: setb %cl ; EGPR-NEXT: movq %r14, %rax ; EGPR-NEXT: mulq %r17 -; EGPR-NEXT: movq %rdx, %r31 +; EGPR-NEXT: movq %rdx, %r29 ; EGPR-NEXT: movq %rax, %r13 ; EGPR-NEXT: addq %r8, %r13 ; EGPR-NEXT: movzbl %cl, %eax -; EGPR-NEXT: adcq %rax, %r31 -; EGPR-NEXT: addq %r21, %r13 -; EGPR-NEXT: adcq %r26, %r31 +; EGPR-NEXT: adcq %rax, %r29 +; EGPR-NEXT: addq %r23, %r13 +; EGPR-NEXT: adcq %r30, %r29 ; EGPR-NEXT: adcq $0, %r16 ; EGPR-NEXT: adcq $0, %r9 ; EGPR-NEXT: movq %r15, %rax ; EGPR-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload ; EGPR-NEXT: mulq %rdi ; EGPR-NEXT: movq %rdx, %r8 -; EGPR-NEXT: movq %rax, %r30 +; EGPR-NEXT: movq %rax, %r28 ; EGPR-NEXT: movq %r14, %rax ; EGPR-NEXT: mulq %rdi -; EGPR-NEXT: movq %rdx, %r26 +; EGPR-NEXT: movq %rdx, %r30 ; EGPR-NEXT: movq %rax, %rcx ; EGPR-NEXT: addq %r8, %rcx -; EGPR-NEXT: adcq $0, %r26 +; EGPR-NEXT: adcq $0, %r30 ; EGPR-NEXT: movq %r15, %rax ; EGPR-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r18 # 8-byte Reload ; EGPR-NEXT: mulq %r18 ; EGPR-NEXT: movq %rdx, %r10 -; EGPR-NEXT: movq %rax, %r21 -; EGPR-NEXT: addq %rcx, %r21 -; EGPR-NEXT: adcq %r26, %r10 +; EGPR-NEXT: movq %rax, %r23 +; EGPR-NEXT: addq %rcx, %r23 +; EGPR-NEXT: adcq %r30, %r10 ; EGPR-NEXT: setb %cl ; EGPR-NEXT: movq %r14, %rax ; EGPR-NEXT: mulq %r18 -; EGPR-NEXT: movq %rdx, %r26 +; EGPR-NEXT: movq %rdx, %r30 ; EGPR-NEXT: movq %rax, %r8 ; EGPR-NEXT: addq %r10, %r8 ; EGPR-NEXT: movzbl %cl, %eax -; EGPR-NEXT: adcq %rax, %r26 -; EGPR-NEXT: addq %r13, %r30 -; EGPR-NEXT: adcq %r31, %r21 +; EGPR-NEXT: adcq %rax, %r30 +; EGPR-NEXT: addq %r13, %r28 +; EGPR-NEXT: adcq %r29, %r23 ; EGPR-NEXT: adcq $0, %r8 -; EGPR-NEXT: adcq $0, %r26 +; EGPR-NEXT: adcq $0, %r30 ; EGPR-NEXT: addq %r16, %r8 -; EGPR-NEXT: adcq %r9, %r26 +; EGPR-NEXT: adcq %r9, %r30 ; EGPR-NEXT: setb %sil ; EGPR-NEXT: movq %rbx, %rax ; EGPR-NEXT: mulq %rdi ; EGPR-NEXT: movq %rdx, %rcx -; EGPR-NEXT: movq %rax, %r31 -; EGPR-NEXT: movq %r28, %rax +; EGPR-NEXT: movq %rax, %r29 +; EGPR-NEXT: movq %r20, %rax ; EGPR-NEXT: mulq %rdi ; EGPR-NEXT: movq %rdx, %r9 ; EGPR-NEXT: movq %rax, %r10 @@ -867,15 +867,15 @@ define void @test_1024(ptr %a, ptr %b, ptr %out) nounwind { ; EGPR-NEXT: addq %r10, %r13 ; EGPR-NEXT: adcq %r9, %rcx ; EGPR-NEXT: setb %r10b -; EGPR-NEXT: movq %r28, %rax +; EGPR-NEXT: movq %r20, %rax ; EGPR-NEXT: mulq %r18 ; EGPR-NEXT: movq %rdx, %r16 ; EGPR-NEXT: movq %rax, %r9 ; EGPR-NEXT: addq %rcx, %r9 ; EGPR-NEXT: movzbl %r10b, %eax ; EGPR-NEXT: adcq %rax, %r16 -; EGPR-NEXT: addq %r8, %r31 -; EGPR-NEXT: adcq %r26, %r13 +; EGPR-NEXT: addq %r8, %r29 +; EGPR-NEXT: adcq %r30, %r13 ; EGPR-NEXT: movzbl %sil, %eax ; EGPR-NEXT: adcq %rax, %r9 ; EGPR-NEXT: adcq $0, %r16 @@ -885,9 +885,9 @@ define void @test_1024(ptr %a, ptr %b, ptr %out) nounwind { ; EGPR-NEXT: mulq %rdi ; EGPR-NEXT: movq %rax, %r8 ; EGPR-NEXT: addq %r18, %rdx -; EGPR-NEXT: movq 104(%r11), %r26 +; EGPR-NEXT: movq 104(%r11), %r30 ; EGPR-NEXT: movq %rdi, %rax -; EGPR-NEXT: imulq %r26, %rax +; EGPR-NEXT: imulq %r30, %rax ; EGPR-NEXT: addq %rdx, %rax ; EGPR-NEXT: movq %rax, %r10 ; EGPR-NEXT: movq 112(%r11), %rax @@ -912,14 +912,14 @@ define void @test_1024(ptr %a, ptr %b, ptr %out) nounwind { ; EGPR-NEXT: addq %r8, %r10 ; EGPR-NEXT: adcq $0, %rcx ; EGPR-NEXT: movq %r19, %rax -; EGPR-NEXT: mulq %r26 +; EGPR-NEXT: mulq %r30 ; EGPR-NEXT: movq %rdx, %r8 ; EGPR-NEXT: movq %rax, %r11 ; EGPR-NEXT: addq %r10, %r11 ; EGPR-NEXT: adcq %rcx, %r8 ; EGPR-NEXT: setb %cl ; EGPR-NEXT: movq %r17, %rax -; EGPR-NEXT: mulq %r26 +; EGPR-NEXT: mulq %r30 ; EGPR-NEXT: movq %rdx, %r10 ; EGPR-NEXT: movq %rax, %r17 ; EGPR-NEXT: addq %r8, %r17 @@ -944,12 +944,12 @@ define void @test_1024(ptr %a, ptr %b, ptr %out) nounwind { ; EGPR-NEXT: movq %rbx, %rax ; EGPR-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r8 # 8-byte Reload ; EGPR-NEXT: mulq %r8 -; EGPR-NEXT: movq %rax, %r26 +; EGPR-NEXT: movq %rax, %r30 ; EGPR-NEXT: addq %rdi, %rdx -; EGPR-NEXT: imulq %r8, %r28 -; EGPR-NEXT: addq %rdx, %r28 -; EGPR-NEXT: addq %rcx, %r26 -; EGPR-NEXT: adcq %r18, %r28 +; EGPR-NEXT: imulq %r8, %r20 +; EGPR-NEXT: addq %rdx, %r20 +; EGPR-NEXT: addq %rcx, %r30 +; EGPR-NEXT: adcq %r18, %r20 ; EGPR-NEXT: movq %r8, %rax ; EGPR-NEXT: movq %r8, %rdi ; EGPR-NEXT: mulq %r15 @@ -973,28 +973,28 @@ define void @test_1024(ptr %a, ptr %b, ptr %out) nounwind { ; EGPR-NEXT: addq %rcx, %rax ; EGPR-NEXT: movzbl %dil, %ecx ; EGPR-NEXT: adcq %rcx, %rdx -; EGPR-NEXT: addq %r26, %rax -; EGPR-NEXT: adcq %r28, %rdx +; EGPR-NEXT: addq %r30, %rax +; EGPR-NEXT: adcq %r20, %rdx ; EGPR-NEXT: addq %rsi, %r8 ; EGPR-NEXT: adcq %r11, %r18 ; EGPR-NEXT: adcq %r17, %rax ; EGPR-NEXT: adcq %r10, %rdx -; EGPR-NEXT: addq %r31, %r8 +; EGPR-NEXT: addq %r29, %r8 ; EGPR-NEXT: adcq %r13, %r18 ; EGPR-NEXT: adcq %r9, %rax ; EGPR-NEXT: adcq %r16, %rdx -; EGPR-NEXT: addq {{[-0-9]+}}(%r{{[sb]}}p), %r23 # 8-byte Folded Reload -; EGPR-NEXT: adcq {{[-0-9]+}}(%r{{[sb]}}p), %r29 # 8-byte Folded Reload -; EGPR-NEXT: adcq {{[-0-9]+}}(%r{{[sb]}}p), %r30 # 8-byte Folded Reload -; EGPR-NEXT: adcq %r22, %r21 -; EGPR-NEXT: adcq %r20, %r8 -; EGPR-NEXT: adcq %r25, %r18 -; EGPR-NEXT: adcq %r27, %rax -; EGPR-NEXT: adcq %r24, %rdx -; EGPR-NEXT: addq {{[-0-9]+}}(%r{{[sb]}}p), %r23 # 8-byte Folded Reload -; EGPR-NEXT: adcq {{[-0-9]+}}(%r{{[sb]}}p), %r29 # 8-byte Folded Reload -; EGPR-NEXT: adcq {{[-0-9]+}}(%r{{[sb]}}p), %r30 # 8-byte Folded Reload +; EGPR-NEXT: addq {{[-0-9]+}}(%r{{[sb]}}p), %r25 # 8-byte Folded Reload ; EGPR-NEXT: adcq {{[-0-9]+}}(%r{{[sb]}}p), %r21 # 8-byte Folded Reload +; EGPR-NEXT: adcq {{[-0-9]+}}(%r{{[sb]}}p), %r28 # 8-byte Folded Reload +; EGPR-NEXT: adcq %r24, %r23 +; EGPR-NEXT: adcq %r22, %r8 +; EGPR-NEXT: adcq %r27, %r18 +; EGPR-NEXT: adcq %r31, %rax +; EGPR-NEXT: adcq %r26, %rdx +; EGPR-NEXT: addq {{[-0-9]+}}(%r{{[sb]}}p), %r25 # 8-byte Folded Reload +; EGPR-NEXT: adcq {{[-0-9]+}}(%r{{[sb]}}p), %r21 # 8-byte Folded Reload +; EGPR-NEXT: adcq {{[-0-9]+}}(%r{{[sb]}}p), %r28 # 8-byte Folded Reload +; EGPR-NEXT: adcq {{[-0-9]+}}(%r{{[sb]}}p), %r23 # 8-byte Folded Reload ; EGPR-NEXT: adcq (%rsp), %r8 # 8-byte Folded Reload ; EGPR-NEXT: adcq {{[-0-9]+}}(%r{{[sb]}}p), %r18 # 8-byte Folded Reload ; EGPR-NEXT: adcq %r12, %rax @@ -1016,10 +1016,10 @@ define void @test_1024(ptr %a, ptr %b, ptr %out) nounwind { ; EGPR-NEXT: movq %rsi, 48(%rcx) ; EGPR-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload ; EGPR-NEXT: movq %rsi, 56(%rcx) -; EGPR-NEXT: movq %r23, 64(%rcx) -; EGPR-NEXT: movq %r29, 72(%rcx) -; EGPR-NEXT: movq %r30, 80(%rcx) -; EGPR-NEXT: movq %r21, 88(%rcx) +; EGPR-NEXT: movq %r25, 64(%rcx) +; EGPR-NEXT: movq %r21, 72(%rcx) +; EGPR-NEXT: movq %r28, 80(%rcx) +; EGPR-NEXT: movq %r23, 88(%rcx) ; EGPR-NEXT: movq %r8, 96(%rcx) ; EGPR-NEXT: movq %r18, 104(%rcx) ; EGPR-NEXT: movq %rax, 112(%rcx) @@ -1044,67 +1044,67 @@ define void @test_1024(ptr %a, ptr %b, ptr %out) nounwind { ; EGPR-NDD-NEXT: subq $96, %rsp ; EGPR-NDD-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; EGPR-NDD-NEXT: movq %rsi, %r15 -; EGPR-NDD-NEXT: movq %rdi, %r20 +; EGPR-NDD-NEXT: movq %rdi, %r22 ; EGPR-NDD-NEXT: movq (%rdi), %r17 ; EGPR-NDD-NEXT: movq 8(%rdi), %r11 ; EGPR-NDD-NEXT: movq 24(%rdi), %r9 ; EGPR-NDD-NEXT: movq 16(%rdi), %r10 ; EGPR-NDD-NEXT: movq 40(%rdi), %rdi -; EGPR-NDD-NEXT: movq 32(%r20), %r16 -; EGPR-NDD-NEXT: movq 56(%r20), %r18 -; EGPR-NDD-NEXT: movq 48(%r20), %r23 +; EGPR-NDD-NEXT: movq 32(%r22), %r16 +; EGPR-NDD-NEXT: movq 56(%r22), %r18 +; EGPR-NDD-NEXT: movq 48(%r22), %r25 ; EGPR-NDD-NEXT: movq 24(%rsi), %r14 -; EGPR-NDD-NEXT: movq 16(%rsi), %r24 -; EGPR-NDD-NEXT: movq (%rsi), %r22 -; EGPR-NDD-NEXT: movq 8(%rsi), %r21 -; EGPR-NDD-NEXT: movq %r23, %rax -; EGPR-NDD-NEXT: mulq %r22 -; EGPR-NDD-NEXT: movq %rdx, %r25 +; EGPR-NDD-NEXT: movq 16(%rsi), %r26 +; EGPR-NDD-NEXT: movq (%rsi), %r24 +; EGPR-NDD-NEXT: movq 8(%rsi), %r23 +; EGPR-NDD-NEXT: movq %r25, %rax +; EGPR-NDD-NEXT: mulq %r24 +; EGPR-NDD-NEXT: movq %rdx, %r27 ; EGPR-NDD-NEXT: movq %rax, %r19 ; EGPR-NDD-NEXT: movq %r18, %rax -; EGPR-NDD-NEXT: mulq %r22 -; EGPR-NDD-NEXT: addq %rax, %r25 +; EGPR-NDD-NEXT: mulq %r24 +; EGPR-NDD-NEXT: addq %rax, %r27 ; EGPR-NDD-NEXT: adcq $0, %rdx, %rcx -; EGPR-NDD-NEXT: movq %r23, %rax -; EGPR-NDD-NEXT: mulq %r21 -; EGPR-NDD-NEXT: addq %r25, %rax, %rsi +; EGPR-NDD-NEXT: movq %r25, %rax +; EGPR-NDD-NEXT: mulq %r23 +; EGPR-NDD-NEXT: addq %r27, %rax, %rsi ; EGPR-NDD-NEXT: adcq %rdx, %rcx ; EGPR-NDD-NEXT: setb %al ; EGPR-NDD-NEXT: movzbl %al, %r8d ; EGPR-NDD-NEXT: movq %r18, %rax -; EGPR-NDD-NEXT: mulq %r21 -; EGPR-NDD-NEXT: addq %rcx, %rax, %r27 +; EGPR-NDD-NEXT: mulq %r23 +; EGPR-NDD-NEXT: addq %rcx, %rax, %r31 ; EGPR-NDD-NEXT: adcq %rdx, %r8 ; EGPR-NDD-NEXT: movq %r16, %rax -; EGPR-NDD-NEXT: mulq %r22 -; EGPR-NDD-NEXT: movq %rdx, %r26 -; EGPR-NDD-NEXT: movq %rax, %r25 +; EGPR-NDD-NEXT: mulq %r24 +; EGPR-NDD-NEXT: movq %rdx, %r30 +; EGPR-NDD-NEXT: movq %rax, %r27 ; EGPR-NDD-NEXT: movq %rdi, %rax -; EGPR-NDD-NEXT: mulq %r22 -; EGPR-NDD-NEXT: addq %r26, %rax, %rcx -; EGPR-NDD-NEXT: adcq $0, %rdx, %r26 +; EGPR-NDD-NEXT: mulq %r24 +; EGPR-NDD-NEXT: addq %r30, %rax, %rcx +; EGPR-NDD-NEXT: adcq $0, %rdx, %r30 ; EGPR-NDD-NEXT: movq %r16, %rax -; EGPR-NDD-NEXT: mulq %r21 +; EGPR-NDD-NEXT: mulq %r23 ; EGPR-NDD-NEXT: addq %rax, %rcx -; EGPR-NDD-NEXT: adcq %rdx, %r26 +; EGPR-NDD-NEXT: adcq %rdx, %r30 ; EGPR-NDD-NEXT: setb %al -; EGPR-NDD-NEXT: movzbl %al, %r28d +; EGPR-NDD-NEXT: movzbl %al, %r20d ; EGPR-NDD-NEXT: movq %rdi, %rax -; EGPR-NDD-NEXT: mulq %r21 -; EGPR-NDD-NEXT: addq %r26, %rax -; EGPR-NDD-NEXT: adcq %r28, %rdx -; EGPR-NDD-NEXT: addq %rax, %r19, %r28 -; EGPR-NDD-NEXT: adcq %rdx, %rsi, %r29 -; EGPR-NDD-NEXT: adcq $0, %r27 +; EGPR-NDD-NEXT: mulq %r23 +; EGPR-NDD-NEXT: addq %r30, %rax +; EGPR-NDD-NEXT: adcq %r20, %rdx +; EGPR-NDD-NEXT: addq %rax, %r19, %r20 +; EGPR-NDD-NEXT: adcq %rdx, %rsi, %r21 +; EGPR-NDD-NEXT: adcq $0, %r31 ; EGPR-NDD-NEXT: adcq $0, %r8 ; EGPR-NDD-NEXT: movq %r16, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; EGPR-NDD-NEXT: movq %r16, %rax -; EGPR-NDD-NEXT: mulq %r24 +; EGPR-NDD-NEXT: mulq %r26 ; EGPR-NDD-NEXT: movq %rdx, %r19 -; EGPR-NDD-NEXT: movq %rax, %r26 +; EGPR-NDD-NEXT: movq %rax, %r30 ; EGPR-NDD-NEXT: movq %rdi, %rax ; EGPR-NDD-NEXT: movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill -; EGPR-NDD-NEXT: mulq %r24 +; EGPR-NDD-NEXT: mulq %r26 ; EGPR-NDD-NEXT: addq %rax, %r19 ; EGPR-NDD-NEXT: adcq $0, %rdx, %rsi ; EGPR-NDD-NEXT: movq %r16, %rax @@ -1112,95 +1112,95 @@ define void @test_1024(ptr %a, ptr %b, ptr %out) nounwind { ; EGPR-NDD-NEXT: addq %rax, %r19 ; EGPR-NDD-NEXT: adcq %rdx, %rsi ; EGPR-NDD-NEXT: setb %al -; EGPR-NDD-NEXT: movzbl %al, %r30d +; EGPR-NDD-NEXT: movzbl %al, %r28d ; EGPR-NDD-NEXT: movq %rdi, %rax ; EGPR-NDD-NEXT: mulq %r14 ; EGPR-NDD-NEXT: addq %rsi, %rax -; EGPR-NDD-NEXT: adcq %r30, %rdx -; EGPR-NDD-NEXT: addq %r28, %r26, %rsi -; EGPR-NDD-NEXT: adcq %r29, %r19, %r28 +; EGPR-NDD-NEXT: adcq %r28, %rdx +; EGPR-NDD-NEXT: addq %r20, %r30, %rsi +; EGPR-NDD-NEXT: adcq %r21, %r19, %r20 ; EGPR-NDD-NEXT: adcq $0, %rax ; EGPR-NDD-NEXT: adcq $0, %rdx -; EGPR-NDD-NEXT: addq %rax, %r27 +; EGPR-NDD-NEXT: addq %rax, %r31 ; EGPR-NDD-NEXT: adcq %rdx, %r8 ; EGPR-NDD-NEXT: setb %al -; EGPR-NDD-NEXT: movzbl %al, %r31d -; EGPR-NDD-NEXT: movq %r23, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill -; EGPR-NDD-NEXT: movq %r23, %rax -; EGPR-NDD-NEXT: mulq %r24 +; EGPR-NDD-NEXT: movzbl %al, %r29d +; EGPR-NDD-NEXT: movq %r25, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill +; EGPR-NDD-NEXT: movq %r25, %rax +; EGPR-NDD-NEXT: mulq %r26 ; EGPR-NDD-NEXT: movq %rdx, %r19 -; EGPR-NDD-NEXT: movq %rax, %r26 +; EGPR-NDD-NEXT: movq %rax, %r30 ; EGPR-NDD-NEXT: movq %r18, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; EGPR-NDD-NEXT: movq %r18, %rax -; EGPR-NDD-NEXT: mulq %r24 +; EGPR-NDD-NEXT: mulq %r26 ; EGPR-NDD-NEXT: addq %rax, %r19 -; EGPR-NDD-NEXT: adcq $0, %rdx, %r29 -; EGPR-NDD-NEXT: movq %r23, %rax +; EGPR-NDD-NEXT: adcq $0, %rdx, %r21 +; EGPR-NDD-NEXT: movq %r25, %rax ; EGPR-NDD-NEXT: mulq %r14 ; EGPR-NDD-NEXT: addq %rax, %r19 -; EGPR-NDD-NEXT: adcq %rdx, %r29 +; EGPR-NDD-NEXT: adcq %rdx, %r21 ; EGPR-NDD-NEXT: setb %al -; EGPR-NDD-NEXT: movzbl %al, %r30d +; EGPR-NDD-NEXT: movzbl %al, %r28d ; EGPR-NDD-NEXT: movq %r18, %rax ; EGPR-NDD-NEXT: mulq %r14 -; EGPR-NDD-NEXT: addq %r29, %rax -; EGPR-NDD-NEXT: adcq %r30, %rdx -; EGPR-NDD-NEXT: addq %r27, %r26, %r29 -; EGPR-NDD-NEXT: adcq %r8, %r19, %r30 -; EGPR-NDD-NEXT: adcq %rax, %r31 +; EGPR-NDD-NEXT: addq %r21, %rax +; EGPR-NDD-NEXT: adcq %r28, %rdx +; EGPR-NDD-NEXT: addq %r31, %r30, %r21 +; EGPR-NDD-NEXT: adcq %r8, %r19, %r28 +; EGPR-NDD-NEXT: adcq %rax, %r29 ; EGPR-NDD-NEXT: adcq $0, %rdx, %rdi ; EGPR-NDD-NEXT: movq %r10, %rax -; EGPR-NDD-NEXT: mulq %r22 +; EGPR-NDD-NEXT: mulq %r24 ; EGPR-NDD-NEXT: movq %rdx, %r19 -; EGPR-NDD-NEXT: movq %rax, %r26 +; EGPR-NDD-NEXT: movq %rax, %r30 ; EGPR-NDD-NEXT: movq %r9, %rax -; EGPR-NDD-NEXT: mulq %r22 +; EGPR-NDD-NEXT: mulq %r24 ; EGPR-NDD-NEXT: addq %rax, %r19 ; EGPR-NDD-NEXT: adcq $0, %rdx, %r8 ; EGPR-NDD-NEXT: movq %r10, %rax -; EGPR-NDD-NEXT: mulq %r21 +; EGPR-NDD-NEXT: mulq %r23 ; EGPR-NDD-NEXT: addq %rax, %r19 ; EGPR-NDD-NEXT: adcq %rdx, %r8 ; EGPR-NDD-NEXT: setb %al -; EGPR-NDD-NEXT: movzbl %al, %r27d +; EGPR-NDD-NEXT: movzbl %al, %r31d ; EGPR-NDD-NEXT: movq %r9, %rax -; EGPR-NDD-NEXT: mulq %r21 +; EGPR-NDD-NEXT: mulq %r23 ; EGPR-NDD-NEXT: addq %rax, %r8 -; EGPR-NDD-NEXT: adcq %r27, %rdx, %rbx +; EGPR-NDD-NEXT: adcq %r31, %rdx, %rbx ; EGPR-NDD-NEXT: movq %r17, %rax -; EGPR-NDD-NEXT: mulq %r22 -; EGPR-NDD-NEXT: movq %rdx, %r27 +; EGPR-NDD-NEXT: mulq %r24 +; EGPR-NDD-NEXT: movq %rdx, %r31 ; EGPR-NDD-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; EGPR-NDD-NEXT: movq %r11, %rax -; EGPR-NDD-NEXT: mulq %r22 -; EGPR-NDD-NEXT: addq %rax, %r27 +; EGPR-NDD-NEXT: mulq %r24 +; EGPR-NDD-NEXT: addq %rax, %r31 ; EGPR-NDD-NEXT: adcq $0, %rdx, %r12 ; EGPR-NDD-NEXT: movq %r17, %rax -; EGPR-NDD-NEXT: mulq %r21 -; EGPR-NDD-NEXT: addq %r27, %rax +; EGPR-NDD-NEXT: mulq %r23 +; EGPR-NDD-NEXT: addq %r31, %rax ; EGPR-NDD-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; EGPR-NDD-NEXT: adcq %rdx, %r12 -; EGPR-NDD-NEXT: setb %r27b +; EGPR-NDD-NEXT: setb %r31b ; EGPR-NDD-NEXT: movq %r11, %rax -; EGPR-NDD-NEXT: mulq %r21 +; EGPR-NDD-NEXT: mulq %r23 ; EGPR-NDD-NEXT: addq %r12, %rax -; EGPR-NDD-NEXT: movzbl %r27b, %r27d -; EGPR-NDD-NEXT: adcq %r27, %rdx -; EGPR-NDD-NEXT: addq %rax, %r26, %r12 +; EGPR-NDD-NEXT: movzbl %r31b, %r31d +; EGPR-NDD-NEXT: adcq %r31, %rdx +; EGPR-NDD-NEXT: addq %rax, %r30, %r12 ; EGPR-NDD-NEXT: adcq %rdx, %r19 ; EGPR-NDD-NEXT: adcq $0, %r8 ; EGPR-NDD-NEXT: adcq $0, %rbx ; EGPR-NDD-NEXT: movq %r17, %rax -; EGPR-NDD-NEXT: mulq %r24 -; EGPR-NDD-NEXT: movq %rdx, %r26 -; EGPR-NDD-NEXT: movq %rax, %r27 +; EGPR-NDD-NEXT: mulq %r26 +; EGPR-NDD-NEXT: movq %rdx, %r30 +; EGPR-NDD-NEXT: movq %rax, %r31 ; EGPR-NDD-NEXT: movq %r11, %rax -; EGPR-NDD-NEXT: mulq %r24 -; EGPR-NDD-NEXT: addq %rax, %r26 +; EGPR-NDD-NEXT: mulq %r26 +; EGPR-NDD-NEXT: addq %rax, %r30 ; EGPR-NDD-NEXT: adcq $0, %rdx, %r13 ; EGPR-NDD-NEXT: movq %r17, %rax ; EGPR-NDD-NEXT: mulq %r14 -; EGPR-NDD-NEXT: addq %rax, %r26 +; EGPR-NDD-NEXT: addq %rax, %r30 ; EGPR-NDD-NEXT: adcq %rdx, %r13 ; EGPR-NDD-NEXT: setb %bpl ; EGPR-NDD-NEXT: movq %r11, %rax @@ -1208,9 +1208,9 @@ define void @test_1024(ptr %a, ptr %b, ptr %out) nounwind { ; EGPR-NDD-NEXT: addq %r13, %rax ; EGPR-NDD-NEXT: movzbl %bpl, %r13d ; EGPR-NDD-NEXT: adcq %r13, %rdx -; EGPR-NDD-NEXT: addq %r12, %r27 -; EGPR-NDD-NEXT: movq %r27, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill -; EGPR-NDD-NEXT: adcq %r26, %r19 +; EGPR-NDD-NEXT: addq %r12, %r31 +; EGPR-NDD-NEXT: movq %r31, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill +; EGPR-NDD-NEXT: adcq %r30, %r19 ; EGPR-NDD-NEXT: movq %r19, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; EGPR-NDD-NEXT: adcq $0, %rax ; EGPR-NDD-NEXT: adcq $0, %rdx @@ -1219,16 +1219,16 @@ define void @test_1024(ptr %a, ptr %b, ptr %out) nounwind { ; EGPR-NDD-NEXT: setb %r19b ; EGPR-NDD-NEXT: movq %r10, %r16 ; EGPR-NDD-NEXT: movq %r10, %rax -; EGPR-NDD-NEXT: mulq %r24 -; EGPR-NDD-NEXT: movq %rdx, %r26 -; EGPR-NDD-NEXT: movq %rax, %r27 +; EGPR-NDD-NEXT: mulq %r26 +; EGPR-NDD-NEXT: movq %rdx, %r30 +; EGPR-NDD-NEXT: movq %rax, %r31 ; EGPR-NDD-NEXT: movq %r9, %rax -; EGPR-NDD-NEXT: mulq %r24 -; EGPR-NDD-NEXT: addq %rax, %r26 +; EGPR-NDD-NEXT: mulq %r26 +; EGPR-NDD-NEXT: addq %rax, %r30 ; EGPR-NDD-NEXT: adcq $0, %rdx, %r12 ; EGPR-NDD-NEXT: movq %r10, %rax ; EGPR-NDD-NEXT: mulq %r14 -; EGPR-NDD-NEXT: addq %rax, %r26 +; EGPR-NDD-NEXT: addq %rax, %r30 ; EGPR-NDD-NEXT: adcq %rdx, %r12 ; EGPR-NDD-NEXT: setb %bpl ; EGPR-NDD-NEXT: movq %r9, %rax @@ -1236,35 +1236,35 @@ define void @test_1024(ptr %a, ptr %b, ptr %out) nounwind { ; EGPR-NDD-NEXT: addq %r12, %rax ; EGPR-NDD-NEXT: movzbl %bpl, %r12d ; EGPR-NDD-NEXT: adcq %r12, %rdx -; EGPR-NDD-NEXT: addq %r27, %r8 -; EGPR-NDD-NEXT: adcq %r26, %rbx +; EGPR-NDD-NEXT: addq %r31, %r8 +; EGPR-NDD-NEXT: adcq %r30, %rbx ; EGPR-NDD-NEXT: movzbl %r19b, %r19d ; EGPR-NDD-NEXT: adcq %r19, %rax ; EGPR-NDD-NEXT: adcq $0, %rdx -; EGPR-NDD-NEXT: addq %r8, %r25, %r12 -; EGPR-NDD-NEXT: movq 32(%r15), %r26 +; EGPR-NDD-NEXT: addq %r8, %r27, %r12 +; EGPR-NDD-NEXT: movq 32(%r15), %r30 ; EGPR-NDD-NEXT: adcq %rbx, %rcx, %r13 ; EGPR-NDD-NEXT: adcq %rax, %rsi, %rbp -; EGPR-NDD-NEXT: adcq %rdx, %r28, %rbx +; EGPR-NDD-NEXT: adcq %rdx, %r20, %rbx +; EGPR-NDD-NEXT: adcq $0, %r21 +; EGPR-NDD-NEXT: movq %r21, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill +; EGPR-NDD-NEXT: adcq $0, %r28 ; EGPR-NDD-NEXT: adcq $0, %r29 -; EGPR-NDD-NEXT: movq %r29, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill -; EGPR-NDD-NEXT: adcq $0, %r30 -; EGPR-NDD-NEXT: adcq $0, %r31 ; EGPR-NDD-NEXT: adcq $0, %rdi ; EGPR-NDD-NEXT: movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; EGPR-NDD-NEXT: movq %r10, %rax -; EGPR-NDD-NEXT: mulq %r26 -; EGPR-NDD-NEXT: movq %rdx, %r25 -; EGPR-NDD-NEXT: movq %rax, %r27 +; EGPR-NDD-NEXT: mulq %r30 +; EGPR-NDD-NEXT: movq %rdx, %r27 +; EGPR-NDD-NEXT: movq %rax, %r31 ; EGPR-NDD-NEXT: movq %r9, %r19 ; EGPR-NDD-NEXT: movq %r9, %rax -; EGPR-NDD-NEXT: mulq %r26 -; EGPR-NDD-NEXT: addq %rax, %r25 +; EGPR-NDD-NEXT: mulq %r30 +; EGPR-NDD-NEXT: addq %rax, %r27 ; EGPR-NDD-NEXT: adcq $0, %rdx, %rcx ; EGPR-NDD-NEXT: movq 40(%r15), %r18 ; EGPR-NDD-NEXT: movq %r10, %rax ; EGPR-NDD-NEXT: mulq %r18 -; EGPR-NDD-NEXT: addq %r25, %rax, %r29 +; EGPR-NDD-NEXT: addq %r27, %rax, %r21 ; EGPR-NDD-NEXT: adcq %rdx, %rcx ; EGPR-NDD-NEXT: setb %r8b ; EGPR-NDD-NEXT: movq %r9, %rax @@ -1273,26 +1273,26 @@ define void @test_1024(ptr %a, ptr %b, ptr %out) nounwind { ; EGPR-NDD-NEXT: movzbl %r8b, %eax ; EGPR-NDD-NEXT: adcq %rax, %rdx, %rsi ; EGPR-NDD-NEXT: movq %r17, %rax -; EGPR-NDD-NEXT: mulq %r26 -; EGPR-NDD-NEXT: movq %rdx, %r28 -; EGPR-NDD-NEXT: movq %rax, %r25 +; EGPR-NDD-NEXT: mulq %r30 +; EGPR-NDD-NEXT: movq %rdx, %r20 +; EGPR-NDD-NEXT: movq %rax, %r27 ; EGPR-NDD-NEXT: movq %r11, %r10 ; EGPR-NDD-NEXT: movq %r11, %rax -; EGPR-NDD-NEXT: mulq %r26 -; EGPR-NDD-NEXT: addq %r28, %rax, %r8 -; EGPR-NDD-NEXT: adcq $0, %rdx, %r28 +; EGPR-NDD-NEXT: mulq %r30 +; EGPR-NDD-NEXT: addq %r20, %rax, %r8 +; EGPR-NDD-NEXT: adcq $0, %rdx, %r20 ; EGPR-NDD-NEXT: movq %r17, %rax ; EGPR-NDD-NEXT: mulq %r18 -; EGPR-NDD-NEXT: addq %r8, %rax, %r23 -; EGPR-NDD-NEXT: adcq %rdx, %r28 +; EGPR-NDD-NEXT: addq %r8, %rax, %r25 +; EGPR-NDD-NEXT: adcq %rdx, %r20 ; EGPR-NDD-NEXT: setb %cl ; EGPR-NDD-NEXT: movq %r11, %rax ; EGPR-NDD-NEXT: mulq %r18 -; EGPR-NDD-NEXT: addq %r28, %rax +; EGPR-NDD-NEXT: addq %r20, %rax ; EGPR-NDD-NEXT: movzbl %cl, %ecx ; EGPR-NDD-NEXT: adcq %rdx, %rcx -; EGPR-NDD-NEXT: addq %rax, %r27 -; EGPR-NDD-NEXT: adcq %rcx, %r29, %r8 +; EGPR-NDD-NEXT: addq %rax, %r31 +; EGPR-NDD-NEXT: adcq %rcx, %r21, %r8 ; EGPR-NDD-NEXT: adcq $0, %rdi ; EGPR-NDD-NEXT: adcq $0, %rsi, %r9 ; EGPR-NDD-NEXT: movq 48(%r15), %r11 @@ -1300,17 +1300,17 @@ define void @test_1024(ptr %a, ptr %b, ptr %out) nounwind { ; EGPR-NDD-NEXT: movq %r17, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; EGPR-NDD-NEXT: movq %r17, %rax ; EGPR-NDD-NEXT: mulq %r11 -; EGPR-NDD-NEXT: movq %rdx, %r28 -; EGPR-NDD-NEXT: movq %rax, %r29 +; EGPR-NDD-NEXT: movq %rdx, %r20 +; EGPR-NDD-NEXT: movq %rax, %r21 ; EGPR-NDD-NEXT: movq %r10, %rax ; EGPR-NDD-NEXT: movq %r10, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; EGPR-NDD-NEXT: mulq %r11 -; EGPR-NDD-NEXT: addq %rax, %r28 +; EGPR-NDD-NEXT: addq %rax, %r20 ; EGPR-NDD-NEXT: adcq $0, %rdx, %rcx ; EGPR-NDD-NEXT: movq 56(%r15), %r17 ; EGPR-NDD-NEXT: movq %rsi, %rax ; EGPR-NDD-NEXT: mulq %r17 -; EGPR-NDD-NEXT: addq %rax, %r28 +; EGPR-NDD-NEXT: addq %rax, %r20 ; EGPR-NDD-NEXT: adcq %rdx, %rcx ; EGPR-NDD-NEXT: setb %sil ; EGPR-NDD-NEXT: movq %r10, %rax @@ -1318,8 +1318,8 @@ define void @test_1024(ptr %a, ptr %b, ptr %out) nounwind { ; EGPR-NDD-NEXT: addq %rcx, %rax ; EGPR-NDD-NEXT: movzbl %sil, %ecx ; EGPR-NDD-NEXT: adcq %rdx, %rcx -; EGPR-NDD-NEXT: addq %r29, %r27 -; EGPR-NDD-NEXT: adcq %r8, %r28, %r10 +; EGPR-NDD-NEXT: addq %r21, %r31 +; EGPR-NDD-NEXT: adcq %r8, %r20, %r10 ; EGPR-NDD-NEXT: adcq $0, %rax ; EGPR-NDD-NEXT: adcq $0, %rcx ; EGPR-NDD-NEXT: addq %rax, %rdi @@ -1328,16 +1328,16 @@ define void @test_1024(ptr %a, ptr %b, ptr %out) nounwind { ; EGPR-NDD-NEXT: movq %r16, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; EGPR-NDD-NEXT: movq %r16, %rax ; EGPR-NDD-NEXT: mulq %r11 -; EGPR-NDD-NEXT: movq %rdx, %r28 -; EGPR-NDD-NEXT: movq %rax, %r29 +; EGPR-NDD-NEXT: movq %rdx, %r20 +; EGPR-NDD-NEXT: movq %rax, %r21 ; EGPR-NDD-NEXT: movq %r19, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; EGPR-NDD-NEXT: movq %r19, %rax ; EGPR-NDD-NEXT: mulq %r11 -; EGPR-NDD-NEXT: addq %rax, %r28 +; EGPR-NDD-NEXT: addq %rax, %r20 ; EGPR-NDD-NEXT: adcq $0, %rdx, %r9 ; EGPR-NDD-NEXT: movq %r16, %rax ; EGPR-NDD-NEXT: mulq %r17 -; EGPR-NDD-NEXT: addq %rax, %r28 +; EGPR-NDD-NEXT: addq %rax, %r20 ; EGPR-NDD-NEXT: adcq %rdx, %r9 ; EGPR-NDD-NEXT: setb %cl ; EGPR-NDD-NEXT: movq %r19, %rax @@ -1345,17 +1345,17 @@ define void @test_1024(ptr %a, ptr %b, ptr %out) nounwind { ; EGPR-NDD-NEXT: addq %r9, %rax ; EGPR-NDD-NEXT: movzbl %cl, %ecx ; EGPR-NDD-NEXT: adcq %rdx, %rcx -; EGPR-NDD-NEXT: addq %r29, %rdi -; EGPR-NDD-NEXT: adcq %r28, %r8 +; EGPR-NDD-NEXT: addq %r21, %rdi +; EGPR-NDD-NEXT: adcq %r20, %r8 ; EGPR-NDD-NEXT: movzbl %sil, %edx ; EGPR-NDD-NEXT: adcq %rdx, %rax ; EGPR-NDD-NEXT: adcq $0, %rcx -; EGPR-NDD-NEXT: addq %r12, %r25 -; EGPR-NDD-NEXT: movq %r25, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill -; EGPR-NDD-NEXT: adcq %r13, %r23, %r19 -; EGPR-NDD-NEXT: movq %r19, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill -; EGPR-NDD-NEXT: adcq %rbp, %r27 +; EGPR-NDD-NEXT: addq %r12, %r27 ; EGPR-NDD-NEXT: movq %r27, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill +; EGPR-NDD-NEXT: adcq %r13, %r25, %r19 +; EGPR-NDD-NEXT: movq %r19, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill +; EGPR-NDD-NEXT: adcq %rbp, %r31 +; EGPR-NDD-NEXT: movq %r31, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; EGPR-NDD-NEXT: adcq %rbx, %r10 ; EGPR-NDD-NEXT: movq %r10, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; EGPR-NDD-NEXT: adcq $0, %rdi @@ -1363,23 +1363,23 @@ define void @test_1024(ptr %a, ptr %b, ptr %out) nounwind { ; EGPR-NDD-NEXT: adcq $0, %rax ; EGPR-NDD-NEXT: adcq $0, %rcx ; EGPR-NDD-NEXT: addq %rdi, {{[-0-9]+}}(%r{{[sb]}}p), %r19 # 8-byte Folded Reload -; EGPR-NDD-NEXT: adcq %r8, %r30 -; EGPR-NDD-NEXT: adcq %rax, %r31 +; EGPR-NDD-NEXT: adcq %r8, %r28 +; EGPR-NDD-NEXT: adcq %rax, %r29 ; EGPR-NDD-NEXT: adcq %rcx, {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Folded Reload ; EGPR-NDD-NEXT: setb %r8b ; EGPR-NDD-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r13 # 8-byte Reload ; EGPR-NDD-NEXT: movq %r13, %rax -; EGPR-NDD-NEXT: mulq %r26 -; EGPR-NDD-NEXT: movq %rdx, %r25 -; EGPR-NDD-NEXT: movq %rax, %r28 +; EGPR-NDD-NEXT: mulq %r30 +; EGPR-NDD-NEXT: movq %rdx, %r27 +; EGPR-NDD-NEXT: movq %rax, %r20 ; EGPR-NDD-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r10 # 8-byte Reload ; EGPR-NDD-NEXT: movq %r10, %rax -; EGPR-NDD-NEXT: mulq %r26 -; EGPR-NDD-NEXT: addq %rax, %r25 +; EGPR-NDD-NEXT: mulq %r30 +; EGPR-NDD-NEXT: addq %rax, %r27 ; EGPR-NDD-NEXT: adcq $0, %rdx, %rsi ; EGPR-NDD-NEXT: movq %r13, %rax ; EGPR-NDD-NEXT: mulq %r18 -; EGPR-NDD-NEXT: addq %r25, %rax, %rdi +; EGPR-NDD-NEXT: addq %r27, %rax, %rdi ; EGPR-NDD-NEXT: adcq %rdx, %rsi ; EGPR-NDD-NEXT: setb %r9b ; EGPR-NDD-NEXT: movq %r10, %rax @@ -1388,66 +1388,66 @@ define void @test_1024(ptr %a, ptr %b, ptr %out) nounwind { ; EGPR-NDD-NEXT: addq %rax, %rsi ; EGPR-NDD-NEXT: movzbl %r9b, %eax ; EGPR-NDD-NEXT: adcq %rax, %rdx, %r9 -; EGPR-NDD-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r23 # 8-byte Reload -; EGPR-NDD-NEXT: movq %r23, %rax -; EGPR-NDD-NEXT: mulq %r26 -; EGPR-NDD-NEXT: movq %rdx, %r29 -; EGPR-NDD-NEXT: movq %rax, %r25 +; EGPR-NDD-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r25 # 8-byte Reload +; EGPR-NDD-NEXT: movq %r25, %rax +; EGPR-NDD-NEXT: mulq %r30 +; EGPR-NDD-NEXT: movq %rdx, %r21 +; EGPR-NDD-NEXT: movq %rax, %r27 ; EGPR-NDD-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r12 # 8-byte Reload ; EGPR-NDD-NEXT: movq %r12, %rax -; EGPR-NDD-NEXT: mulq %r26 -; EGPR-NDD-NEXT: addq %rax, %r29 +; EGPR-NDD-NEXT: mulq %r30 +; EGPR-NDD-NEXT: addq %rax, %r21 ; EGPR-NDD-NEXT: adcq $0, %rdx, %r10 -; EGPR-NDD-NEXT: movq %r23, %rax +; EGPR-NDD-NEXT: movq %r25, %rax ; EGPR-NDD-NEXT: mulq %r18 -; EGPR-NDD-NEXT: addq %r29, %rax, %rbx +; EGPR-NDD-NEXT: addq %r21, %rax, %rbx ; EGPR-NDD-NEXT: adcq %rdx, %r10 -; EGPR-NDD-NEXT: setb %r27b +; EGPR-NDD-NEXT: setb %r31b ; EGPR-NDD-NEXT: movq %r12, %rax ; EGPR-NDD-NEXT: mulq %r18 ; EGPR-NDD-NEXT: addq %r10, %rax -; EGPR-NDD-NEXT: movzbl %r27b, %r10d +; EGPR-NDD-NEXT: movzbl %r31b, %r10d ; EGPR-NDD-NEXT: adcq %r10, %rdx -; EGPR-NDD-NEXT: addq %rax, %r28, %r10 +; EGPR-NDD-NEXT: addq %rax, %r20, %r10 ; EGPR-NDD-NEXT: adcq %rdx, %rdi ; EGPR-NDD-NEXT: adcq $0, %rsi ; EGPR-NDD-NEXT: adcq $0, %r9 -; EGPR-NDD-NEXT: movq %r23, %rax +; EGPR-NDD-NEXT: movq %r25, %rax ; EGPR-NDD-NEXT: mulq %r11 -; EGPR-NDD-NEXT: movq %rdx, %r28 -; EGPR-NDD-NEXT: movq %rax, %r29 +; EGPR-NDD-NEXT: movq %rdx, %r20 +; EGPR-NDD-NEXT: movq %rax, %r21 ; EGPR-NDD-NEXT: movq %r12, %rax ; EGPR-NDD-NEXT: mulq %r11 -; EGPR-NDD-NEXT: addq %rax, %r28 -; EGPR-NDD-NEXT: adcq $0, %rdx, %r27 -; EGPR-NDD-NEXT: movq %r23, %rax +; EGPR-NDD-NEXT: addq %rax, %r20 +; EGPR-NDD-NEXT: adcq $0, %rdx, %r31 +; EGPR-NDD-NEXT: movq %r25, %rax ; EGPR-NDD-NEXT: mulq %r17 -; EGPR-NDD-NEXT: addq %rax, %r28 -; EGPR-NDD-NEXT: adcq %rdx, %r27 +; EGPR-NDD-NEXT: addq %rax, %r20 +; EGPR-NDD-NEXT: adcq %rdx, %r31 ; EGPR-NDD-NEXT: setb %bpl ; EGPR-NDD-NEXT: movq %r12, %rax ; EGPR-NDD-NEXT: mulq %r17 -; EGPR-NDD-NEXT: addq %r27, %rax -; EGPR-NDD-NEXT: movzbl %bpl, %r27d -; EGPR-NDD-NEXT: adcq %r27, %rdx -; EGPR-NDD-NEXT: addq %r29, %r10 -; EGPR-NDD-NEXT: adcq %r28, %rdi +; EGPR-NDD-NEXT: addq %r31, %rax +; EGPR-NDD-NEXT: movzbl %bpl, %r31d +; EGPR-NDD-NEXT: adcq %r31, %rdx +; EGPR-NDD-NEXT: addq %r21, %r10 +; EGPR-NDD-NEXT: adcq %r20, %rdi ; EGPR-NDD-NEXT: adcq $0, %rax ; EGPR-NDD-NEXT: adcq $0, %rdx ; EGPR-NDD-NEXT: addq %rax, %rsi ; EGPR-NDD-NEXT: adcq %rdx, %r9 -; EGPR-NDD-NEXT: setb %r27b +; EGPR-NDD-NEXT: setb %r31b ; EGPR-NDD-NEXT: movq %r13, %rax ; EGPR-NDD-NEXT: mulq %r11 -; EGPR-NDD-NEXT: movq %rdx, %r28 -; EGPR-NDD-NEXT: movq %rax, %r29 +; EGPR-NDD-NEXT: movq %rdx, %r20 +; EGPR-NDD-NEXT: movq %rax, %r21 ; EGPR-NDD-NEXT: movq %r16, %rax ; EGPR-NDD-NEXT: mulq %r11 -; EGPR-NDD-NEXT: addq %rax, %r28 +; EGPR-NDD-NEXT: addq %rax, %r20 ; EGPR-NDD-NEXT: adcq $0, %rdx, %r12 ; EGPR-NDD-NEXT: movq %r13, %rax ; EGPR-NDD-NEXT: mulq %r17 -; EGPR-NDD-NEXT: addq %rax, %r28 +; EGPR-NDD-NEXT: addq %rax, %r20 ; EGPR-NDD-NEXT: adcq %rdx, %r12 ; EGPR-NDD-NEXT: setb %bpl ; EGPR-NDD-NEXT: movq %r16, %rax @@ -1455,16 +1455,16 @@ define void @test_1024(ptr %a, ptr %b, ptr %out) nounwind { ; EGPR-NDD-NEXT: addq %r12, %rax ; EGPR-NDD-NEXT: movzbl %bpl, %r12d ; EGPR-NDD-NEXT: adcq %r12, %rdx -; EGPR-NDD-NEXT: addq %r29, %rsi -; EGPR-NDD-NEXT: adcq %r28, %r9 -; EGPR-NDD-NEXT: movzbl %r27b, %r27d -; EGPR-NDD-NEXT: adcq %r27, %rax +; EGPR-NDD-NEXT: addq %r21, %rsi +; EGPR-NDD-NEXT: adcq %r20, %r9 +; EGPR-NDD-NEXT: movzbl %r31b, %r31d +; EGPR-NDD-NEXT: adcq %r31, %rax ; EGPR-NDD-NEXT: adcq $0, %rdx -; EGPR-NDD-NEXT: addq %r25, %r19 +; EGPR-NDD-NEXT: addq %r27, %r19 ; EGPR-NDD-NEXT: movq %r19, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill -; EGPR-NDD-NEXT: adcq %rbx, %r30 -; EGPR-NDD-NEXT: movq %r30, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill -; EGPR-NDD-NEXT: adcq %r31, %r10 +; EGPR-NDD-NEXT: adcq %rbx, %r28 +; EGPR-NDD-NEXT: movq %r28, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill +; EGPR-NDD-NEXT: adcq %r29, %r10 ; EGPR-NDD-NEXT: movq %r10, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; EGPR-NDD-NEXT: adcq %rdi, %rcx ; EGPR-NDD-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill @@ -1477,88 +1477,88 @@ define void @test_1024(ptr %a, ptr %b, ptr %out) nounwind { ; EGPR-NDD-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; EGPR-NDD-NEXT: adcq $0, %rdx ; EGPR-NDD-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill -; EGPR-NDD-NEXT: movq 64(%r20), %r28 -; EGPR-NDD-NEXT: movq %r24, %rax -; EGPR-NDD-NEXT: mulq %r28 -; EGPR-NDD-NEXT: movq %rdx, %r25 -; EGPR-NDD-NEXT: movq %rax, %r30 +; EGPR-NDD-NEXT: movq 64(%r22), %r20 +; EGPR-NDD-NEXT: movq %r26, %rax +; EGPR-NDD-NEXT: mulq %r20 +; EGPR-NDD-NEXT: movq %rdx, %r27 +; EGPR-NDD-NEXT: movq %rax, %r28 ; EGPR-NDD-NEXT: movq %r14, %rax -; EGPR-NDD-NEXT: mulq %r28 -; EGPR-NDD-NEXT: addq %rax, %r25 +; EGPR-NDD-NEXT: mulq %r20 +; EGPR-NDD-NEXT: addq %rax, %r27 ; EGPR-NDD-NEXT: adcq $0, %rdx, %rcx -; EGPR-NDD-NEXT: movq 72(%r20), %r29 -; EGPR-NDD-NEXT: movq %r24, %rax -; EGPR-NDD-NEXT: mulq %r29 -; EGPR-NDD-NEXT: addq %rax, %r25 +; EGPR-NDD-NEXT: movq 72(%r22), %r21 +; EGPR-NDD-NEXT: movq %r26, %rax +; EGPR-NDD-NEXT: mulq %r21 +; EGPR-NDD-NEXT: addq %rax, %r27 ; EGPR-NDD-NEXT: adcq %rdx, %rcx ; EGPR-NDD-NEXT: setb %sil ; EGPR-NDD-NEXT: movq %r14, %rax -; EGPR-NDD-NEXT: mulq %r29 +; EGPR-NDD-NEXT: mulq %r21 ; EGPR-NDD-NEXT: addq %rax, %rcx ; EGPR-NDD-NEXT: movzbl %sil, %eax ; EGPR-NDD-NEXT: adcq %rax, %rdx, %rsi -; EGPR-NDD-NEXT: movq %r22, %rax -; EGPR-NDD-NEXT: mulq %r28 -; EGPR-NDD-NEXT: movq %rdx, %r31 +; EGPR-NDD-NEXT: movq %r24, %rax +; EGPR-NDD-NEXT: mulq %r20 +; EGPR-NDD-NEXT: movq %rdx, %r29 ; EGPR-NDD-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill -; EGPR-NDD-NEXT: movq %r21, %rax -; EGPR-NDD-NEXT: mulq %r28 -; EGPR-NDD-NEXT: addq %rax, %r31 +; EGPR-NDD-NEXT: movq %r23, %rax +; EGPR-NDD-NEXT: mulq %r20 +; EGPR-NDD-NEXT: addq %rax, %r29 ; EGPR-NDD-NEXT: adcq $0, %rdx, %rdi -; EGPR-NDD-NEXT: movq %r22, %rax -; EGPR-NDD-NEXT: mulq %r29 -; EGPR-NDD-NEXT: addq %r31, %rax +; EGPR-NDD-NEXT: movq %r24, %rax +; EGPR-NDD-NEXT: mulq %r21 +; EGPR-NDD-NEXT: addq %r29, %rax ; EGPR-NDD-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill ; EGPR-NDD-NEXT: adcq %rdx, %rdi ; EGPR-NDD-NEXT: setb %r8b -; EGPR-NDD-NEXT: movq %r21, %rax -; EGPR-NDD-NEXT: mulq %r29 +; EGPR-NDD-NEXT: movq %r23, %rax +; EGPR-NDD-NEXT: mulq %r21 ; EGPR-NDD-NEXT: addq %rdi, %rax ; EGPR-NDD-NEXT: movzbl %r8b, %edi ; EGPR-NDD-NEXT: adcq %rdi, %rdx -; EGPR-NDD-NEXT: addq %rax, %r30, %rdi -; EGPR-NDD-NEXT: adcq %rdx, %r25 +; EGPR-NDD-NEXT: addq %rax, %r28, %rdi +; EGPR-NDD-NEXT: adcq %rdx, %r27 ; EGPR-NDD-NEXT: adcq $0, %rcx ; EGPR-NDD-NEXT: adcq $0, %rsi -; EGPR-NDD-NEXT: movq 80(%r20), %r8 -; EGPR-NDD-NEXT: movq %r22, %rax +; EGPR-NDD-NEXT: movq 80(%r22), %r8 +; EGPR-NDD-NEXT: movq %r24, %rax ; EGPR-NDD-NEXT: mulq %r8 -; EGPR-NDD-NEXT: movq %rdx, %r30 -; EGPR-NDD-NEXT: movq %rax, %r31 -; EGPR-NDD-NEXT: movq %r21, %rax +; EGPR-NDD-NEXT: movq %rdx, %r28 +; EGPR-NDD-NEXT: movq %rax, %r29 +; EGPR-NDD-NEXT: movq %r23, %rax ; EGPR-NDD-NEXT: mulq %r8 -; EGPR-NDD-NEXT: addq %rax, %r30 +; EGPR-NDD-NEXT: addq %rax, %r28 ; EGPR-NDD-NEXT: adcq $0, %rdx, %r9 -; EGPR-NDD-NEXT: movq 88(%r20), %rbx -; EGPR-NDD-NEXT: movq %r22, %rax +; EGPR-NDD-NEXT: movq 88(%r22), %rbx +; EGPR-NDD-NEXT: movq %r24, %rax ; EGPR-NDD-NEXT: mulq %rbx -; EGPR-NDD-NEXT: addq %rax, %r30 +; EGPR-NDD-NEXT: addq %rax, %r28 ; EGPR-NDD-NEXT: adcq %rdx, %r9 ; EGPR-NDD-NEXT: setb %r10b -; EGPR-NDD-NEXT: movq %r21, %rax +; EGPR-NDD-NEXT: movq %r23, %rax ; EGPR-NDD-NEXT: mulq %rbx ; EGPR-NDD-NEXT: addq %r9, %rax ; EGPR-NDD-NEXT: movzbl %r10b, %r9d ; EGPR-NDD-NEXT: adcq %r9, %rdx -; EGPR-NDD-NEXT: addq %r31, %rdi +; EGPR-NDD-NEXT: addq %r29, %rdi ; EGPR-NDD-NEXT: movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill -; EGPR-NDD-NEXT: adcq %r25, %r30, %rbp +; EGPR-NDD-NEXT: adcq %r27, %r28, %rbp ; EGPR-NDD-NEXT: adcq $0, %rax ; EGPR-NDD-NEXT: adcq $0, %rdx ; EGPR-NDD-NEXT: addq %rax, %rcx ; EGPR-NDD-NEXT: adcq %rdx, %rsi ; EGPR-NDD-NEXT: setb %dil -; EGPR-NDD-NEXT: movq %r24, %rax +; EGPR-NDD-NEXT: movq %r26, %rax ; EGPR-NDD-NEXT: mulq %r8 -; EGPR-NDD-NEXT: movq %rdx, %r30 -; EGPR-NDD-NEXT: movq %rax, %r31 +; EGPR-NDD-NEXT: movq %rdx, %r28 +; EGPR-NDD-NEXT: movq %rax, %r29 ; EGPR-NDD-NEXT: movq %r14, %rax ; EGPR-NDD-NEXT: mulq %r8 -; EGPR-NDD-NEXT: addq %rax, %r30 +; EGPR-NDD-NEXT: addq %rax, %r28 ; EGPR-NDD-NEXT: adcq $0, %rdx, %r9 -; EGPR-NDD-NEXT: movq %r24, %rax +; EGPR-NDD-NEXT: movq %r26, %rax ; EGPR-NDD-NEXT: mulq %rbx -; EGPR-NDD-NEXT: addq %rax, %r30 +; EGPR-NDD-NEXT: addq %rax, %r28 ; EGPR-NDD-NEXT: adcq %rdx, %r9 ; EGPR-NDD-NEXT: setb %r10b ; EGPR-NDD-NEXT: movq %r14, %rax @@ -1566,191 +1566,191 @@ define void @test_1024(ptr %a, ptr %b, ptr %out) nounwind { ; EGPR-NDD-NEXT: addq %r9, %rax ; EGPR-NDD-NEXT: movzbl %r10b, %r9d ; EGPR-NDD-NEXT: adcq %r9, %rdx -; EGPR-NDD-NEXT: addq %rcx, %r31, %r25 -; EGPR-NDD-NEXT: adcq %rsi, %r30, %r12 +; EGPR-NDD-NEXT: addq %rcx, %r29, %r27 +; EGPR-NDD-NEXT: adcq %rsi, %r28, %r12 ; EGPR-NDD-NEXT: movzbl %dil, %r19d ; EGPR-NDD-NEXT: adcq %rax, %r19 -; EGPR-NDD-NEXT: adcq $0, %rdx, %r31 -; EGPR-NDD-NEXT: imulq %r26, %rbx -; EGPR-NDD-NEXT: movq %r26, %rax +; EGPR-NDD-NEXT: adcq $0, %rdx, %r29 +; EGPR-NDD-NEXT: imulq %r30, %rbx +; EGPR-NDD-NEXT: movq %r30, %rax ; EGPR-NDD-NEXT: mulq %r8 -; EGPR-NDD-NEXT: movq %rax, %r30 +; EGPR-NDD-NEXT: movq %rax, %r28 ; EGPR-NDD-NEXT: addq %rbx, %rdx ; EGPR-NDD-NEXT: imulq %r18, %r8 ; EGPR-NDD-NEXT: addq %rdx, %r8 -; EGPR-NDD-NEXT: imulq %r29, %r11, %rcx +; EGPR-NDD-NEXT: imulq %r21, %r11, %rcx ; EGPR-NDD-NEXT: movq %r11, %rax -; EGPR-NDD-NEXT: mulq %r28 +; EGPR-NDD-NEXT: mulq %r20 ; EGPR-NDD-NEXT: addq %rdx, %rcx -; EGPR-NDD-NEXT: imulq %r28, %r17, %r16 +; EGPR-NDD-NEXT: imulq %r20, %r17, %r16 ; EGPR-NDD-NEXT: addq %r16, %rcx -; EGPR-NDD-NEXT: addq %r30, %rax, %rsi +; EGPR-NDD-NEXT: addq %r28, %rax, %rsi ; EGPR-NDD-NEXT: adcq %rcx, %r8 -; EGPR-NDD-NEXT: movq %r28, %rax -; EGPR-NDD-NEXT: mulq %r26 -; EGPR-NDD-NEXT: movq %rdx, %r30 -; EGPR-NDD-NEXT: movq %rax, %r27 -; EGPR-NDD-NEXT: movq %r29, %rax -; EGPR-NDD-NEXT: mulq %r26 -; EGPR-NDD-NEXT: addq %r30, %rax, %rcx +; EGPR-NDD-NEXT: movq %r20, %rax +; EGPR-NDD-NEXT: mulq %r30 +; EGPR-NDD-NEXT: movq %rdx, %r28 +; EGPR-NDD-NEXT: movq %rax, %r31 +; EGPR-NDD-NEXT: movq %r21, %rax +; EGPR-NDD-NEXT: mulq %r30 +; EGPR-NDD-NEXT: addq %r28, %rax, %rcx ; EGPR-NDD-NEXT: adcq $0, %rdx, %rdi -; EGPR-NDD-NEXT: movq %r28, %rax +; EGPR-NDD-NEXT: movq %r20, %rax ; EGPR-NDD-NEXT: mulq %r18 ; EGPR-NDD-NEXT: addq %rax, %rcx ; EGPR-NDD-NEXT: adcq %rdx, %rdi ; EGPR-NDD-NEXT: setb %r9b -; EGPR-NDD-NEXT: movq %r29, %rax +; EGPR-NDD-NEXT: movq %r21, %rax ; EGPR-NDD-NEXT: mulq %r18 ; EGPR-NDD-NEXT: addq %rdi, %rax ; EGPR-NDD-NEXT: movzbl %r9b, %edi ; EGPR-NDD-NEXT: adcq %rdi, %rdx ; EGPR-NDD-NEXT: addq %rax, %rsi ; EGPR-NDD-NEXT: adcq %rdx, %r8 -; EGPR-NDD-NEXT: movq 112(%r20), %rdi -; EGPR-NDD-NEXT: movq %r22, %rax +; EGPR-NDD-NEXT: movq 112(%r22), %rdi +; EGPR-NDD-NEXT: movq %r24, %rax ; EGPR-NDD-NEXT: mulq %rdi -; EGPR-NDD-NEXT: movq %rax, %r26 -; EGPR-NDD-NEXT: imulq %r21, %rdi +; EGPR-NDD-NEXT: movq %rax, %r30 +; EGPR-NDD-NEXT: imulq %r23, %rdi ; EGPR-NDD-NEXT: addq %rdi, %rdx -; EGPR-NDD-NEXT: imulq 120(%r20), %r22, %rax +; EGPR-NDD-NEXT: imulq 120(%r22), %r24, %rax ; EGPR-NDD-NEXT: addq %rax, %rdx, %r9 -; EGPR-NDD-NEXT: movq 96(%r20), %r28 -; EGPR-NDD-NEXT: movq 104(%r20), %rdi -; EGPR-NDD-NEXT: imulq %rdi, %r24, %r10 -; EGPR-NDD-NEXT: movq %r24, %rax -; EGPR-NDD-NEXT: mulq %r28 +; EGPR-NDD-NEXT: movq 96(%r22), %r20 +; EGPR-NDD-NEXT: movq 104(%r22), %rdi +; EGPR-NDD-NEXT: imulq %rdi, %r26, %r10 +; EGPR-NDD-NEXT: movq %r26, %rax +; EGPR-NDD-NEXT: mulq %r20 ; EGPR-NDD-NEXT: addq %r10, %rdx -; EGPR-NDD-NEXT: imulq %r28, %r14, %r23 -; EGPR-NDD-NEXT: addq %r23, %rdx -; EGPR-NDD-NEXT: addq %rax, %r26 +; EGPR-NDD-NEXT: imulq %r20, %r14, %r25 +; EGPR-NDD-NEXT: addq %r25, %rdx +; EGPR-NDD-NEXT: addq %rax, %r30 ; EGPR-NDD-NEXT: adcq %rdx, %r9 -; EGPR-NDD-NEXT: movq %r28, %rax -; EGPR-NDD-NEXT: mulq %r22 -; EGPR-NDD-NEXT: movq %rdx, %r23 -; EGPR-NDD-NEXT: movq %rax, %r24 +; EGPR-NDD-NEXT: movq %r20, %rax +; EGPR-NDD-NEXT: mulq %r24 +; EGPR-NDD-NEXT: movq %rdx, %r25 +; EGPR-NDD-NEXT: movq %rax, %r26 ; EGPR-NDD-NEXT: movq %rdi, %rax -; EGPR-NDD-NEXT: mulq %r22 -; EGPR-NDD-NEXT: addq %rax, %r23 +; EGPR-NDD-NEXT: mulq %r24 +; EGPR-NDD-NEXT: addq %rax, %r25 ; EGPR-NDD-NEXT: adcq $0, %rdx, %r10 -; EGPR-NDD-NEXT: movq %r28, %rax -; EGPR-NDD-NEXT: mulq %r21 -; EGPR-NDD-NEXT: addq %rax, %r23 +; EGPR-NDD-NEXT: movq %r20, %rax +; EGPR-NDD-NEXT: mulq %r23 +; EGPR-NDD-NEXT: addq %rax, %r25 ; EGPR-NDD-NEXT: adcq %rdx, %r10 ; EGPR-NDD-NEXT: setb %r11b ; EGPR-NDD-NEXT: movq %rdi, %rax -; EGPR-NDD-NEXT: mulq %r21 +; EGPR-NDD-NEXT: mulq %r23 ; EGPR-NDD-NEXT: addq %r10, %rax ; EGPR-NDD-NEXT: movzbl %r11b, %edi ; EGPR-NDD-NEXT: adcq %rdi, %rdx -; EGPR-NDD-NEXT: addq %r26, %rax +; EGPR-NDD-NEXT: addq %r30, %rax ; EGPR-NDD-NEXT: adcq %r9, %rdx -; EGPR-NDD-NEXT: addq %r27, %r24 -; EGPR-NDD-NEXT: adcq %r23, %rcx +; EGPR-NDD-NEXT: addq %r31, %r26 +; EGPR-NDD-NEXT: adcq %r25, %rcx ; EGPR-NDD-NEXT: adcq %rsi, %rax ; EGPR-NDD-NEXT: adcq %r8, %rdx -; EGPR-NDD-NEXT: addq %r24, %r25, %rbx +; EGPR-NDD-NEXT: addq %r26, %r27, %rbx ; EGPR-NDD-NEXT: adcq %rcx, %r12 ; EGPR-NDD-NEXT: adcq %rax, %r19, %r13 -; EGPR-NDD-NEXT: adcq %rdx, %r31, %r30 -; EGPR-NDD-NEXT: movq 80(%r15), %r22 -; EGPR-NDD-NEXT: movq %r22, %rax +; EGPR-NDD-NEXT: adcq %rdx, %r29, %r28 +; EGPR-NDD-NEXT: movq 80(%r15), %r24 +; EGPR-NDD-NEXT: movq %r24, %rax ; EGPR-NDD-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r16 # 8-byte Reload ; EGPR-NDD-NEXT: mulq %r16 -; EGPR-NDD-NEXT: movq %rax, %r26 +; EGPR-NDD-NEXT: movq %rax, %r30 ; EGPR-NDD-NEXT: movq %rdx, %rdi -; EGPR-NDD-NEXT: movq 88(%r15), %r20 -; EGPR-NDD-NEXT: movq %r20, %rax +; EGPR-NDD-NEXT: movq 88(%r15), %r22 +; EGPR-NDD-NEXT: movq %r22, %rax ; EGPR-NDD-NEXT: mulq %r16 ; EGPR-NDD-NEXT: addq %rax, %rdi ; EGPR-NDD-NEXT: adcq $0, %rdx, %rcx -; EGPR-NDD-NEXT: movq %r22, %rax -; EGPR-NDD-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r21 # 8-byte Reload -; EGPR-NDD-NEXT: mulq %r21 +; EGPR-NDD-NEXT: movq %r24, %rax +; EGPR-NDD-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r23 # 8-byte Reload +; EGPR-NDD-NEXT: mulq %r23 ; EGPR-NDD-NEXT: addq %rax, %rdi ; EGPR-NDD-NEXT: adcq %rdx, %rcx ; EGPR-NDD-NEXT: setb %sil -; EGPR-NDD-NEXT: movq %r20, %rax -; EGPR-NDD-NEXT: mulq %r21 +; EGPR-NDD-NEXT: movq %r22, %rax +; EGPR-NDD-NEXT: mulq %r23 ; EGPR-NDD-NEXT: addq %rax, %rcx ; EGPR-NDD-NEXT: movzbl %sil, %eax ; EGPR-NDD-NEXT: adcq %rax, %rdx, %rsi -; EGPR-NDD-NEXT: movq 64(%r15), %r24 -; EGPR-NDD-NEXT: movq %r24, %rax +; EGPR-NDD-NEXT: movq 64(%r15), %r26 +; EGPR-NDD-NEXT: movq %r26, %rax ; EGPR-NDD-NEXT: mulq %r16 -; EGPR-NDD-NEXT: movq %rax, %r29 -; EGPR-NDD-NEXT: movq %rdx, %r27 -; EGPR-NDD-NEXT: movq 72(%r15), %r23 -; EGPR-NDD-NEXT: movq %r23, %rax +; EGPR-NDD-NEXT: movq %rax, %r21 +; EGPR-NDD-NEXT: movq %rdx, %r31 +; EGPR-NDD-NEXT: movq 72(%r15), %r25 +; EGPR-NDD-NEXT: movq %r25, %rax ; EGPR-NDD-NEXT: mulq %r16 -; EGPR-NDD-NEXT: addq %rax, %r27 +; EGPR-NDD-NEXT: addq %rax, %r31 ; EGPR-NDD-NEXT: adcq $0, %rdx, %r8 -; EGPR-NDD-NEXT: movq %r24, %rax -; EGPR-NDD-NEXT: mulq %r21 -; EGPR-NDD-NEXT: addq %r27, %rax, %r31 +; EGPR-NDD-NEXT: movq %r26, %rax +; EGPR-NDD-NEXT: mulq %r23 +; EGPR-NDD-NEXT: addq %r31, %rax, %r29 ; EGPR-NDD-NEXT: adcq %rdx, %r8 ; EGPR-NDD-NEXT: setb %r9b -; EGPR-NDD-NEXT: movq %r23, %rax -; EGPR-NDD-NEXT: mulq %r21 +; EGPR-NDD-NEXT: movq %r25, %rax +; EGPR-NDD-NEXT: mulq %r23 ; EGPR-NDD-NEXT: addq %r8, %rax ; EGPR-NDD-NEXT: movzbl %r9b, %r8d ; EGPR-NDD-NEXT: adcq %r8, %rdx -; EGPR-NDD-NEXT: addq %rax, %r26, %r28 +; EGPR-NDD-NEXT: addq %rax, %r30, %r20 ; EGPR-NDD-NEXT: adcq %rdx, %rdi ; EGPR-NDD-NEXT: adcq $0, %rcx ; EGPR-NDD-NEXT: adcq $0, %rsi -; EGPR-NDD-NEXT: movq %r24, %rax +; EGPR-NDD-NEXT: movq %r26, %rax ; EGPR-NDD-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r10 # 8-byte Reload ; EGPR-NDD-NEXT: mulq %r10 -; EGPR-NDD-NEXT: movq %rdx, %r26 -; EGPR-NDD-NEXT: movq %rax, %r27 -; EGPR-NDD-NEXT: movq %r23, %rax +; EGPR-NDD-NEXT: movq %rdx, %r30 +; EGPR-NDD-NEXT: movq %rax, %r31 +; EGPR-NDD-NEXT: movq %r25, %rax ; EGPR-NDD-NEXT: mulq %r10 -; EGPR-NDD-NEXT: addq %rax, %r26 +; EGPR-NDD-NEXT: addq %rax, %r30 ; EGPR-NDD-NEXT: adcq $0, %rdx, %r8 -; EGPR-NDD-NEXT: movq %r24, %rax +; EGPR-NDD-NEXT: movq %r26, %rax ; EGPR-NDD-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r11 # 8-byte Reload ; EGPR-NDD-NEXT: mulq %r11 -; EGPR-NDD-NEXT: addq %r26, %rax, %r25 +; EGPR-NDD-NEXT: addq %r30, %rax, %r27 ; EGPR-NDD-NEXT: adcq %rdx, %r8 ; EGPR-NDD-NEXT: setb %r9b -; EGPR-NDD-NEXT: movq %r23, %rax +; EGPR-NDD-NEXT: movq %r25, %rax ; EGPR-NDD-NEXT: mulq %r11 ; EGPR-NDD-NEXT: addq %r8, %rax ; EGPR-NDD-NEXT: movzbl %r9b, %r8d ; EGPR-NDD-NEXT: adcq %r8, %rdx -; EGPR-NDD-NEXT: addq %r27, %r28 -; EGPR-NDD-NEXT: adcq %rdi, %r25 +; EGPR-NDD-NEXT: addq %r31, %r20 +; EGPR-NDD-NEXT: adcq %rdi, %r27 ; EGPR-NDD-NEXT: adcq $0, %rax ; EGPR-NDD-NEXT: adcq $0, %rdx ; EGPR-NDD-NEXT: addq %rax, %rcx ; EGPR-NDD-NEXT: adcq %rdx, %rsi ; EGPR-NDD-NEXT: setb %dil -; EGPR-NDD-NEXT: movq %r22, %rax +; EGPR-NDD-NEXT: movq %r24, %rax ; EGPR-NDD-NEXT: mulq %r10 -; EGPR-NDD-NEXT: movq %rdx, %r26 -; EGPR-NDD-NEXT: movq %rax, %r27 -; EGPR-NDD-NEXT: movq %r20, %rax +; EGPR-NDD-NEXT: movq %rdx, %r30 +; EGPR-NDD-NEXT: movq %rax, %r31 +; EGPR-NDD-NEXT: movq %r22, %rax ; EGPR-NDD-NEXT: mulq %r10 -; EGPR-NDD-NEXT: addq %rax, %r26 +; EGPR-NDD-NEXT: addq %rax, %r30 ; EGPR-NDD-NEXT: adcq $0, %rdx, %r8 -; EGPR-NDD-NEXT: movq %r22, %rax +; EGPR-NDD-NEXT: movq %r24, %rax ; EGPR-NDD-NEXT: mulq %r11 -; EGPR-NDD-NEXT: addq %r26, %rax, %r19 +; EGPR-NDD-NEXT: addq %r30, %rax, %r19 ; EGPR-NDD-NEXT: adcq %rdx, %r8 ; EGPR-NDD-NEXT: setb %r9b -; EGPR-NDD-NEXT: movq %r20, %rax +; EGPR-NDD-NEXT: movq %r22, %rax ; EGPR-NDD-NEXT: mulq %r11 ; EGPR-NDD-NEXT: addq %r8, %rax ; EGPR-NDD-NEXT: movzbl %r9b, %r8d ; EGPR-NDD-NEXT: adcq %r8, %rdx -; EGPR-NDD-NEXT: addq %rcx, %r27 +; EGPR-NDD-NEXT: addq %rcx, %r31 ; EGPR-NDD-NEXT: adcq %rsi, %r19 ; EGPR-NDD-NEXT: movzbl %dil, %ecx ; EGPR-NDD-NEXT: adcq %rax, %rcx ; EGPR-NDD-NEXT: adcq $0, %rdx, %rdi -; EGPR-NDD-NEXT: movq 96(%r15), %r26 -; EGPR-NDD-NEXT: imulq %r11, %r26, %rsi -; EGPR-NDD-NEXT: movq %r26, %rax +; EGPR-NDD-NEXT: movq 96(%r15), %r30 +; EGPR-NDD-NEXT: imulq %r11, %r30, %rsi +; EGPR-NDD-NEXT: movq %r30, %rax ; EGPR-NDD-NEXT: mulq %r10 ; EGPR-NDD-NEXT: movq %rax, %r18 ; EGPR-NDD-NEXT: addq %rsi, %rdx @@ -1758,7 +1758,7 @@ define void @test_1024(ptr %a, ptr %b, ptr %out) nounwind { ; EGPR-NDD-NEXT: imulq %r10, %r8, %rax ; EGPR-NDD-NEXT: addq %rax, %rdx, %rsi ; EGPR-NDD-NEXT: movq 112(%r15), %rax -; EGPR-NDD-NEXT: imulq %r21, %rax, %r9 +; EGPR-NDD-NEXT: imulq %r23, %rax, %r9 ; EGPR-NDD-NEXT: mulq %r16 ; EGPR-NDD-NEXT: addq %r9, %rdx ; EGPR-NDD-NEXT: imulq 120(%r15), %r16, %r9 @@ -1767,11 +1767,11 @@ define void @test_1024(ptr %a, ptr %b, ptr %out) nounwind { ; EGPR-NDD-NEXT: adcq %rsi, %rdx, %r9 ; EGPR-NDD-NEXT: movq %r16, %rax ; EGPR-NDD-NEXT: movq %r16, %r18 -; EGPR-NDD-NEXT: mulq %r26 +; EGPR-NDD-NEXT: mulq %r30 ; EGPR-NDD-NEXT: movq %rdx, %r17 ; EGPR-NDD-NEXT: movq %rax, %rsi -; EGPR-NDD-NEXT: movq %r21, %rax -; EGPR-NDD-NEXT: mulq %r26 +; EGPR-NDD-NEXT: movq %r23, %rax +; EGPR-NDD-NEXT: mulq %r30 ; EGPR-NDD-NEXT: addq %r17, %rax, %r11 ; EGPR-NDD-NEXT: adcq $0, %rdx, %r16 ; EGPR-NDD-NEXT: movq %r18, %rax @@ -1779,72 +1779,72 @@ define void @test_1024(ptr %a, ptr %b, ptr %out) nounwind { ; EGPR-NDD-NEXT: addq %rax, %r11 ; EGPR-NDD-NEXT: adcq %rdx, %r16 ; EGPR-NDD-NEXT: setb %r17b -; EGPR-NDD-NEXT: movq %r21, %rax +; EGPR-NDD-NEXT: movq %r23, %rax ; EGPR-NDD-NEXT: mulq %r8 ; EGPR-NDD-NEXT: addq %r16, %rax ; EGPR-NDD-NEXT: movzbl %r17b, %r8d ; EGPR-NDD-NEXT: adcq %r8, %rdx ; EGPR-NDD-NEXT: addq %rax, %r10 ; EGPR-NDD-NEXT: adcq %r9, %rdx, %r17 -; EGPR-NDD-NEXT: imulq {{[-0-9]+}}(%r{{[sb]}}p), %r24, %r8 # 8-byte Folded Reload -; EGPR-NDD-NEXT: movq %r24, %rax +; EGPR-NDD-NEXT: imulq {{[-0-9]+}}(%r{{[sb]}}p), %r26, %r8 # 8-byte Folded Reload +; EGPR-NDD-NEXT: movq %r26, %rax ; EGPR-NDD-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r16 # 8-byte Reload ; EGPR-NDD-NEXT: mulq %r16 ; EGPR-NDD-NEXT: movq %rax, %r9 ; EGPR-NDD-NEXT: addq %r8, %rdx -; EGPR-NDD-NEXT: imulq %r16, %r23, %rax +; EGPR-NDD-NEXT: imulq %r16, %r25, %rax ; EGPR-NDD-NEXT: addq %rax, %rdx, %r8 -; EGPR-NDD-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r21 # 8-byte Reload -; EGPR-NDD-NEXT: imulq %r21, %r22, %r16 -; EGPR-NDD-NEXT: movq %r22, %rax -; EGPR-NDD-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r26 # 8-byte Reload -; EGPR-NDD-NEXT: mulq %r26 +; EGPR-NDD-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r23 # 8-byte Reload +; EGPR-NDD-NEXT: imulq %r23, %r24, %r16 +; EGPR-NDD-NEXT: movq %r24, %rax +; EGPR-NDD-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r30 # 8-byte Reload +; EGPR-NDD-NEXT: mulq %r30 ; EGPR-NDD-NEXT: addq %r16, %rdx -; EGPR-NDD-NEXT: imulq %r26, %r20 -; EGPR-NDD-NEXT: addq %r20, %rdx +; EGPR-NDD-NEXT: imulq %r30, %r22 +; EGPR-NDD-NEXT: addq %r22, %rdx ; EGPR-NDD-NEXT: addq %r9, %rax, %r16 ; EGPR-NDD-NEXT: adcq %r8, %rdx, %r18 -; EGPR-NDD-NEXT: movq %r26, %rax -; EGPR-NDD-NEXT: mulq %r24 +; EGPR-NDD-NEXT: movq %r30, %rax +; EGPR-NDD-NEXT: mulq %r26 ; EGPR-NDD-NEXT: movq %rdx, %r8 ; EGPR-NDD-NEXT: movq %rax, %r9 -; EGPR-NDD-NEXT: movq %r21, %rax -; EGPR-NDD-NEXT: movq %r21, %r22 -; EGPR-NDD-NEXT: mulq %r24 +; EGPR-NDD-NEXT: movq %r23, %rax +; EGPR-NDD-NEXT: movq %r23, %r24 +; EGPR-NDD-NEXT: mulq %r26 ; EGPR-NDD-NEXT: addq %rax, %r8 -; EGPR-NDD-NEXT: adcq $0, %rdx, %r20 -; EGPR-NDD-NEXT: movq %r26, %rax -; EGPR-NDD-NEXT: mulq %r23 +; EGPR-NDD-NEXT: adcq $0, %rdx, %r22 +; EGPR-NDD-NEXT: movq %r30, %rax +; EGPR-NDD-NEXT: mulq %r25 ; EGPR-NDD-NEXT: addq %rax, %r8 -; EGPR-NDD-NEXT: adcq %rdx, %r20 -; EGPR-NDD-NEXT: setb %r21b -; EGPR-NDD-NEXT: movq %r22, %rax -; EGPR-NDD-NEXT: mulq %r23 -; EGPR-NDD-NEXT: addq %r20, %rax -; EGPR-NDD-NEXT: movzbl %r21b, %r20d -; EGPR-NDD-NEXT: adcq %r20, %rdx +; EGPR-NDD-NEXT: adcq %rdx, %r22 +; EGPR-NDD-NEXT: setb %r23b +; EGPR-NDD-NEXT: movq %r24, %rax +; EGPR-NDD-NEXT: mulq %r25 +; EGPR-NDD-NEXT: addq %r22, %rax +; EGPR-NDD-NEXT: movzbl %r23b, %r22d +; EGPR-NDD-NEXT: adcq %r22, %rdx ; EGPR-NDD-NEXT: addq %r16, %rax ; EGPR-NDD-NEXT: adcq %r18, %rdx ; EGPR-NDD-NEXT: addq %r9, %rsi ; EGPR-NDD-NEXT: adcq %r11, %r8 ; EGPR-NDD-NEXT: adcq %r10, %rax ; EGPR-NDD-NEXT: adcq %r17, %rdx -; EGPR-NDD-NEXT: addq %r27, %rsi +; EGPR-NDD-NEXT: addq %r31, %rsi ; EGPR-NDD-NEXT: adcq %r19, %r8 ; EGPR-NDD-NEXT: adcq %rcx, %rax ; EGPR-NDD-NEXT: adcq %rdi, %rdx -; EGPR-NDD-NEXT: addq {{[-0-9]+}}(%r{{[sb]}}p), %r29 # 8-byte Folded Reload -; EGPR-NDD-NEXT: adcq {{[-0-9]+}}(%r{{[sb]}}p), %r31 # 8-byte Folded Reload -; EGPR-NDD-NEXT: adcq {{[-0-9]+}}(%r{{[sb]}}p), %r28 # 8-byte Folded Reload -; EGPR-NDD-NEXT: adcq %rbp, %r25 +; EGPR-NDD-NEXT: addq {{[-0-9]+}}(%r{{[sb]}}p), %r21 # 8-byte Folded Reload +; EGPR-NDD-NEXT: adcq {{[-0-9]+}}(%r{{[sb]}}p), %r29 # 8-byte Folded Reload +; EGPR-NDD-NEXT: adcq {{[-0-9]+}}(%r{{[sb]}}p), %r20 # 8-byte Folded Reload +; EGPR-NDD-NEXT: adcq %rbp, %r27 ; EGPR-NDD-NEXT: adcq %rbx, %rsi ; EGPR-NDD-NEXT: adcq %r12, %r8 ; EGPR-NDD-NEXT: adcq %r13, %rax -; EGPR-NDD-NEXT: adcq %r30, %rdx -; EGPR-NDD-NEXT: addq %r29, {{[-0-9]+}}(%r{{[sb]}}p), %r29 # 8-byte Folded Reload -; EGPR-NDD-NEXT: adcq %r31, {{[-0-9]+}}(%r{{[sb]}}p), %r31 # 8-byte Folded Reload -; EGPR-NDD-NEXT: adcq %r28, {{[-0-9]+}}(%r{{[sb]}}p), %r28 # 8-byte Folded Reload -; EGPR-NDD-NEXT: adcq %r25, {{[-0-9]+}}(%r{{[sb]}}p), %r25 # 8-byte Folded Reload +; EGPR-NDD-NEXT: adcq %r28, %rdx +; EGPR-NDD-NEXT: addq %r21, {{[-0-9]+}}(%r{{[sb]}}p), %r21 # 8-byte Folded Reload +; EGPR-NDD-NEXT: adcq %r29, {{[-0-9]+}}(%r{{[sb]}}p), %r29 # 8-byte Folded Reload +; EGPR-NDD-NEXT: adcq %r20, {{[-0-9]+}}(%r{{[sb]}}p), %r20 # 8-byte Folded Reload +; EGPR-NDD-NEXT: adcq %r27, {{[-0-9]+}}(%r{{[sb]}}p), %r27 # 8-byte Folded Reload ; EGPR-NDD-NEXT: adcq %rsi, (%rsp), %rsi # 8-byte Folded Reload ; EGPR-NDD-NEXT: adcq %r8, {{[-0-9]+}}(%r{{[sb]}}p), %r8 # 8-byte Folded Reload ; EGPR-NDD-NEXT: adcq %rax, {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Folded Reload @@ -1866,10 +1866,10 @@ define void @test_1024(ptr %a, ptr %b, ptr %out) nounwind { ; EGPR-NDD-NEXT: movq %rdi, 48(%rcx) ; EGPR-NDD-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload ; EGPR-NDD-NEXT: movq %rdi, 56(%rcx) -; EGPR-NDD-NEXT: movq %r29, 64(%rcx) -; EGPR-NDD-NEXT: movq %r31, 72(%rcx) -; EGPR-NDD-NEXT: movq %r28, 80(%rcx) -; EGPR-NDD-NEXT: movq %r25, 88(%rcx) +; EGPR-NDD-NEXT: movq %r21, 64(%rcx) +; EGPR-NDD-NEXT: movq %r29, 72(%rcx) +; EGPR-NDD-NEXT: movq %r20, 80(%rcx) +; EGPR-NDD-NEXT: movq %r27, 88(%rcx) ; EGPR-NDD-NEXT: movq %rsi, 96(%rcx) ; EGPR-NDD-NEXT: movq %r8, 104(%rcx) ; EGPR-NDD-NEXT: movq %rax, 112(%rcx) diff --git a/llvm/test/CodeGen/X86/apx/or.ll b/llvm/test/CodeGen/X86/apx/or.ll index e51ba9d9bf03..995946c007dc 100644 --- a/llvm/test/CodeGen/X86/apx/or.ll +++ b/llvm/test/CodeGen/X86/apx/or.ll @@ -906,13 +906,13 @@ entry: define void @or64mi_legacy(ptr %a) { ; CHECK-LABEL: or64mi_legacy: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: orq $123456, (%rdi) # encoding: [0x48,0x81,0x0f,0x40,0xe2,0x01,0x00] +; CHECK-NEXT: orl $123456, (%rdi) # encoding: [0x81,0x0f,0x40,0xe2,0x01,0x00] ; CHECK-NEXT: # imm = 0x1E240 ; CHECK-NEXT: retq # encoding: [0xc3] ; ; NF-LABEL: or64mi_legacy: ; NF: # %bb.0: # %entry -; NF-NEXT: orq $123456, (%rdi) # encoding: [0x48,0x81,0x0f,0x40,0xe2,0x01,0x00] +; NF-NEXT: orl $123456, (%rdi) # encoding: [0x81,0x0f,0x40,0xe2,0x01,0x00] ; NF-NEXT: # imm = 0x1E240 ; NF-NEXT: retq # encoding: [0xc3] entry: diff --git a/llvm/test/CodeGen/X86/avx10_2-cmp.ll b/llvm/test/CodeGen/X86/avx10_2-cmp.ll index de0bec7ea269..140a20c17ea6 100644 --- a/llvm/test/CodeGen/X86/avx10_2-cmp.ll +++ b/llvm/test/CodeGen/X86/avx10_2-cmp.ll @@ -235,3 +235,44 @@ define i1 @dune_mem(ptr %xp, ptr %yp) { %1 = fcmp une double %x, %y ret i1 %1 } + +define i32 @PR118606(x86_fp80 %val1) #0 { +; X64-LABEL: PR118606: +; X64: # %bb.0: # %entry +; X64-NEXT: fldt {{[0-9]+}}(%rsp) +; X64-NEXT: fldz +; X64-NEXT: fucomi %st(1), %st +; X64-NEXT: fstp %st(1) +; X64-NEXT: fld1 +; X64-NEXT: fcmovne %st(1), %st +; X64-NEXT: fcmovu %st(1), %st +; X64-NEXT: fucompi %st(1), %st +; X64-NEXT: fstp %st(0) +; X64-NEXT: xorl %eax, %eax +; X64-NEXT: retq +; +; X86-LABEL: PR118606: +; X86: # %bb.0: # %entry +; X86-NEXT: fldt {{[0-9]+}}(%esp) +; X86-NEXT: fldz +; X86-NEXT: fucomi %st(1), %st +; X86-NEXT: fstp %st(1) +; X86-NEXT: fld1 +; X86-NEXT: fcmovne %st(1), %st +; X86-NEXT: fcmovu %st(1), %st +; X86-NEXT: fucompi %st(1), %st +; X86-NEXT: fstp %st(0) +; X86-NEXT: xorl %eax, %eax +; X86-NEXT: retl +entry: + %cmp8 = fcmp oeq x86_fp80 %val1, 0xK00000000000000000000 + %0 = select i1 %cmp8, x86_fp80 0xK3FFF8000000000000000, x86_fp80 0xK00000000000000000000 + %cmp64 = fcmp ogt x86_fp80 %0, 0xK00000000000000000000 + br i1 %cmp64, label %if.then66, label %if.end70 + +if.then66: ; preds = %entry + ret i32 0 + +if.end70: ; preds = %entry + ret i32 0 +} diff --git a/llvm/test/CodeGen/X86/avx512-bugfix-26264.ll b/llvm/test/CodeGen/X86/avx512-bugfix-26264.ll index 537f42dd9c2c..e0f3b6c4ec90 100644 --- a/llvm/test/CodeGen/X86/avx512-bugfix-26264.ll +++ b/llvm/test/CodeGen/X86/avx512-bugfix-26264.ll @@ -7,11 +7,11 @@ define <32 x double> @test_load_32f64(ptr %ptrs, <32 x i1> %mask, <32 x double> ; AVX512BW-NEXT: vpsllw $7, %ymm0, %ymm0 ; AVX512BW-NEXT: vpmovb2m %zmm0, %k1 ; AVX512BW-NEXT: vblendmpd (%rdi), %zmm1, %zmm0 {%k1} -; AVX512BW-NEXT: kshiftrw $8, %k1, %k2 +; AVX512BW-NEXT: kshiftrd $8, %k1, %k2 ; AVX512BW-NEXT: vblendmpd 64(%rdi), %zmm2, %zmm1 {%k2} -; AVX512BW-NEXT: kshiftrd $16, %k1, %k1 -; AVX512BW-NEXT: vblendmpd 128(%rdi), %zmm3, %zmm2 {%k1} -; AVX512BW-NEXT: kshiftrw $8, %k1, %k1 +; AVX512BW-NEXT: kshiftrd $16, %k1, %k2 +; AVX512BW-NEXT: vblendmpd 128(%rdi), %zmm3, %zmm2 {%k2} +; AVX512BW-NEXT: kshiftrd $24, %k1, %k1 ; AVX512BW-NEXT: vblendmpd 192(%rdi), %zmm4, %zmm3 {%k1} ; AVX512BW-NEXT: retq %res = call <32 x double> @llvm.masked.load.v32f64.p0(ptr %ptrs, i32 4, <32 x i1> %mask, <32 x double> %src0) @@ -24,11 +24,11 @@ define <32 x i64> @test_load_32i64(ptr %ptrs, <32 x i1> %mask, <32 x i64> %src0) ; AVX512BW-NEXT: vpsllw $7, %ymm0, %ymm0 ; AVX512BW-NEXT: vpmovb2m %zmm0, %k1 ; AVX512BW-NEXT: vpblendmq (%rdi), %zmm1, %zmm0 {%k1} -; AVX512BW-NEXT: kshiftrw $8, %k1, %k2 +; AVX512BW-NEXT: kshiftrd $8, %k1, %k2 ; AVX512BW-NEXT: vpblendmq 64(%rdi), %zmm2, %zmm1 {%k2} -; AVX512BW-NEXT: kshiftrd $16, %k1, %k1 -; AVX512BW-NEXT: vpblendmq 128(%rdi), %zmm3, %zmm2 {%k1} -; AVX512BW-NEXT: kshiftrw $8, %k1, %k1 +; AVX512BW-NEXT: kshiftrd $16, %k1, %k2 +; AVX512BW-NEXT: vpblendmq 128(%rdi), %zmm3, %zmm2 {%k2} +; AVX512BW-NEXT: kshiftrd $24, %k1, %k1 ; AVX512BW-NEXT: vpblendmq 192(%rdi), %zmm4, %zmm3 {%k1} ; AVX512BW-NEXT: retq %res = call <32 x i64> @llvm.masked.load.v32i64.p0(ptr %ptrs, i32 4, <32 x i1> %mask, <32 x i64> %src0) diff --git a/llvm/test/CodeGen/X86/avx512-insert-extract.ll b/llvm/test/CodeGen/X86/avx512-insert-extract.ll index 1c4bfa8422d8..7ce37c637a79 100644 --- a/llvm/test/CodeGen/X86/avx512-insert-extract.ll +++ b/llvm/test/CodeGen/X86/avx512-insert-extract.ll @@ -672,7 +672,7 @@ define <32 x i16> @insert_v32i16(<32 x i16> %x, i16 %y, ptr %ptr) nounwind { ; KNL-NEXT: vinserti32x4 $0, %xmm1, %zmm0, %zmm1 ; KNL-NEXT: vmovd %edi, %xmm0 ; KNL-NEXT: vpbroadcastw %xmm0, %ymm0 -; KNL-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm0 +; KNL-NEXT: vpternlogq {{.*#+}} zmm0 = zmm0 ^ (mem & (zmm0 ^ zmm1)) ; KNL-NEXT: retq ; ; SKX-LABEL: insert_v32i16: @@ -1466,7 +1466,7 @@ define zeroext i8 @test_extractelement_varible_v2i1(<2 x i64> %a, <2 x i64> %b, ; KNL-NEXT: ## kill: def $xmm1 killed $xmm1 def $zmm1 ; KNL-NEXT: ## kill: def $xmm0 killed $xmm0 def $zmm0 ; KNL-NEXT: vpcmpnleuq %zmm1, %zmm0, %k1 -; KNL-NEXT: vpternlogq $255, %zmm0, %zmm0, %zmm0 {%k1} {z} +; KNL-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1 ; KNL-NEXT: vmovdqa %xmm0, -{{[0-9]+}}(%rsp) ; KNL-NEXT: andl $1, %edi ; KNL-NEXT: movzbl -24(%rsp,%rdi,8), %eax @@ -1497,7 +1497,7 @@ define zeroext i8 @test_extractelement_varible_v4i1(<4 x i32> %a, <4 x i32> %b, ; KNL-NEXT: ## kill: def $xmm1 killed $xmm1 def $zmm1 ; KNL-NEXT: ## kill: def $xmm0 killed $xmm0 def $zmm0 ; KNL-NEXT: vpcmpnleud %zmm1, %zmm0, %k1 -; KNL-NEXT: vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z} +; KNL-NEXT: vpternlogd {{.*#+}} zmm0 {%k1} {z} = -1 ; KNL-NEXT: vmovdqa %xmm0, -{{[0-9]+}}(%rsp) ; KNL-NEXT: andl $3, %edi ; KNL-NEXT: movzbl -24(%rsp,%rdi,4), %eax @@ -1528,7 +1528,7 @@ define zeroext i8 @test_extractelement_varible_v8i1(<8 x i32> %a, <8 x i32> %b, ; KNL-NEXT: ## kill: def $ymm1 killed $ymm1 def $zmm1 ; KNL-NEXT: ## kill: def $ymm0 killed $ymm0 def $zmm0 ; KNL-NEXT: vpcmpnleud %zmm1, %zmm0, %k1 -; KNL-NEXT: vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z} +; KNL-NEXT: vpternlogd {{.*#+}} zmm0 {%k1} {z} = -1 ; KNL-NEXT: vpmovdw %zmm0, %ymm0 ; KNL-NEXT: vmovdqa %xmm0, -{{[0-9]+}}(%rsp) ; KNL-NEXT: andl $7, %edi @@ -1559,7 +1559,7 @@ define zeroext i8 @test_extractelement_varible_v16i1(<16 x i32> %a, <16 x i32> % ; KNL: ## %bb.0: ; KNL-NEXT: ## kill: def $edi killed $edi def $rdi ; KNL-NEXT: vpcmpnleud %zmm1, %zmm0, %k1 -; KNL-NEXT: vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z} +; KNL-NEXT: vpternlogd {{.*#+}} zmm0 {%k1} {z} = -1 ; KNL-NEXT: vpmovdb %zmm0, -{{[0-9]+}}(%rsp) ; KNL-NEXT: andl $15, %edi ; KNL-NEXT: movzbl -24(%rsp,%rdi), %eax @@ -1641,7 +1641,7 @@ define i32 @test_insertelement_variable_v32i1(<32 x i8> %a, i8 %b, i32 %index) n ; KNL-NEXT: ## kill: def $esi killed $esi def $rsi ; KNL-NEXT: vpxor %xmm1, %xmm1, %xmm1 ; KNL-NEXT: vpcmpeqb %ymm1, %ymm0, %ymm0 -; KNL-NEXT: vpternlogq $15, %zmm0, %zmm0, %zmm0 +; KNL-NEXT: vpternlogq {{.*#+}} zmm0 = ~zmm0 ; KNL-NEXT: andl $31, %esi ; KNL-NEXT: testb %dil, %dil ; KNL-NEXT: vmovdqa %ymm0, (%rsp) @@ -1695,7 +1695,7 @@ define i64 @test_insertelement_variable_v64i1(<64 x i8> %a, i8 %b, i32 %index) n ; KNL-NEXT: vpcmpeqb %ymm2, %ymm1, %ymm1 ; KNL-NEXT: vpcmpeqb %ymm2, %ymm0, %ymm0 ; KNL-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0 -; KNL-NEXT: vpternlogq $15, %zmm0, %zmm0, %zmm0 +; KNL-NEXT: vpternlogq {{.*#+}} zmm0 = ~zmm0 ; KNL-NEXT: andl $63, %esi ; KNL-NEXT: testb %dil, %dil ; KNL-NEXT: vmovdqa64 %zmm0, (%rsp) @@ -1827,7 +1827,7 @@ define i96 @test_insertelement_variable_v96i1(<96 x i8> %a, i8 %b, i32 %index) n ; KNL-NEXT: vinserti128 $1, %xmm3, %ymm2, %ymm2 ; KNL-NEXT: vpcmpeqb %ymm0, %ymm2, %ymm2 ; KNL-NEXT: vinserti64x4 $1, %ymm1, %zmm2, %zmm1 -; KNL-NEXT: vpternlogq $15, %zmm1, %zmm1, %zmm1 +; KNL-NEXT: vpternlogq {{.*#+}} zmm1 = ~zmm1 ; KNL-NEXT: vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero ; KNL-NEXT: vpinsrb $1, 488(%rbp), %xmm2, %xmm2 ; KNL-NEXT: vpinsrb $2, 496(%rbp), %xmm2, %xmm2 @@ -1862,7 +1862,7 @@ define i96 @test_insertelement_variable_v96i1(<96 x i8> %a, i8 %b, i32 %index) n ; KNL-NEXT: vpinsrb $15, 728(%rbp), %xmm3, %xmm3 ; KNL-NEXT: vinserti128 $1, %xmm3, %ymm2, %ymm2 ; KNL-NEXT: vpcmpeqb %ymm0, %ymm2, %ymm2 -; KNL-NEXT: vpternlogq $15, %zmm2, %zmm2, %zmm2 +; KNL-NEXT: vpternlogq {{.*#+}} zmm2 = ~zmm2 ; KNL-NEXT: cmpb $0, 736(%rbp) ; KNL-NEXT: vmovdqa %ymm0, {{[0-9]+}}(%rsp) ; KNL-NEXT: vmovdqa %ymm2, {{[0-9]+}}(%rsp) @@ -2063,12 +2063,12 @@ define i128 @test_insertelement_variable_v128i1(<128 x i8> %a, i8 %b, i32 %index ; KNL-NEXT: vpcmpeqb %ymm3, %ymm2, %ymm2 ; KNL-NEXT: vpcmpeqb %ymm3, %ymm0, %ymm0 ; KNL-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0 -; KNL-NEXT: vpternlogq $15, %zmm0, %zmm0, %zmm0 +; KNL-NEXT: vpternlogq {{.*#+}} zmm0 = ~zmm0 ; KNL-NEXT: vextracti64x4 $1, %zmm1, %ymm2 ; KNL-NEXT: vpcmpeqb %ymm3, %ymm2, %ymm2 ; KNL-NEXT: vpcmpeqb %ymm3, %ymm1, %ymm1 ; KNL-NEXT: vinserti64x4 $1, %ymm2, %zmm1, %zmm1 -; KNL-NEXT: vpternlogq $15, %zmm1, %zmm1, %zmm1 +; KNL-NEXT: vpternlogq {{.*#+}} zmm1 = ~zmm1 ; KNL-NEXT: andl $127, %esi ; KNL-NEXT: testb %dil, %dil ; KNL-NEXT: vmovdqa64 %zmm1, {{[0-9]+}}(%rsp) @@ -2159,14 +2159,33 @@ define i128 @test_insertelement_variable_v128i1(<128 x i8> %a, i8 %b, i32 %index define void @test_concat_v2i1(ptr %arg, ptr %arg1, ptr %arg2) nounwind { ; KNL-LABEL: test_concat_v2i1: ; KNL: ## %bb.0: -; KNL-NEXT: vmovsd {{.*#+}} xmm0 = mem[0],zero -; KNL-NEXT: vcvtph2ps %xmm0, %ymm0 -; KNL-NEXT: vcmpltps {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm0, %k0 -; KNL-NEXT: vxorps %xmm1, %xmm1, %xmm1 -; KNL-NEXT: vcmpltps %zmm0, %zmm1, %k1 +; KNL-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero +; KNL-NEXT: vcvtph2ps %xmm0, %xmm1 +; KNL-NEXT: vmovss {{.*#+}} xmm2 = [6.0E+0,0.0E+0,0.0E+0,0.0E+0] +; KNL-NEXT: vucomiss %xmm2, %xmm1 +; KNL-NEXT: setb %al +; KNL-NEXT: andl $1, %eax +; KNL-NEXT: kmovw %eax, %k0 +; KNL-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[1,1,1,1,4,5,6,7] +; KNL-NEXT: vcvtph2ps %xmm0, %xmm0 +; KNL-NEXT: vucomiss %xmm2, %xmm0 +; KNL-NEXT: setb %al +; KNL-NEXT: kmovw %eax, %k1 +; KNL-NEXT: kshiftlw $1, %k1, %k1 +; KNL-NEXT: korw %k1, %k0, %k0 +; KNL-NEXT: vxorps %xmm2, %xmm2, %xmm2 +; KNL-NEXT: vucomiss %xmm2, %xmm1 +; KNL-NEXT: seta %al +; KNL-NEXT: andl $1, %eax +; KNL-NEXT: kmovw %eax, %k1 +; KNL-NEXT: vucomiss %xmm2, %xmm0 +; KNL-NEXT: seta %al +; KNL-NEXT: kmovw %eax, %k2 +; KNL-NEXT: kshiftlw $1, %k2, %k2 +; KNL-NEXT: korw %k2, %k1, %k1 ; KNL-NEXT: kandw %k1, %k0, %k1 ; KNL-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero -; KNL-NEXT: vpternlogd $255, %zmm1, %zmm1, %zmm1 {%k1} {z} +; KNL-NEXT: vpternlogd {{.*#+}} zmm1 {%k1} {z} = -1 ; KNL-NEXT: vpmovdw %zmm1, %ymm1 ; KNL-NEXT: vpand %xmm0, %xmm1, %xmm0 ; KNL-NEXT: vmovd %xmm0, (%rdx) diff --git a/llvm/test/CodeGen/X86/avx512-masked-memop-64-32.ll b/llvm/test/CodeGen/X86/avx512-masked-memop-64-32.ll index bd52b9cd4158..f6e5986afac5 100644 --- a/llvm/test/CodeGen/X86/avx512-masked-memop-64-32.ll +++ b/llvm/test/CodeGen/X86/avx512-masked-memop-64-32.ll @@ -261,11 +261,11 @@ define <32 x double> @test_load_32f64(ptr %ptrs, <32 x i1> %mask, <32 x double> ; SKX-NEXT: vpsllw $7, %ymm0, %ymm0 ; SKX-NEXT: vpmovb2m %ymm0, %k1 ; SKX-NEXT: vblendmpd (%rdi), %zmm1, %zmm0 {%k1} -; SKX-NEXT: kshiftrw $8, %k1, %k2 +; SKX-NEXT: kshiftrd $8, %k1, %k2 ; SKX-NEXT: vblendmpd 64(%rdi), %zmm2, %zmm1 {%k2} -; SKX-NEXT: kshiftrd $16, %k1, %k1 -; SKX-NEXT: vblendmpd 128(%rdi), %zmm3, %zmm2 {%k1} -; SKX-NEXT: kshiftrw $8, %k1, %k1 +; SKX-NEXT: kshiftrd $16, %k1, %k2 +; SKX-NEXT: vblendmpd 128(%rdi), %zmm3, %zmm2 {%k2} +; SKX-NEXT: kshiftrd $24, %k1, %k1 ; SKX-NEXT: vblendmpd 192(%rdi), %zmm4, %zmm3 {%k1} ; SKX-NEXT: retq %res = call <32 x double> @llvm.masked.load.v32f64.p0(ptr %ptrs, i32 4, <32 x i1> %mask, <32 x double> %src0) diff --git a/llvm/test/CodeGen/X86/avx512-vec-cmp.ll b/llvm/test/CodeGen/X86/avx512-vec-cmp.ll index 24eb9b3715ed..5ce2b56cbd43 100644 --- a/llvm/test/CodeGen/X86/avx512-vec-cmp.ll +++ b/llvm/test/CodeGen/X86/avx512-vec-cmp.ll @@ -1441,30 +1441,56 @@ define <4 x i32> @zext_bool_logic(<4 x i64> %cond1, <4 x i64> %cond2, <4 x i32> define void @half_vec_compare(ptr %x, ptr %y) { ; KNL-LABEL: half_vec_compare: ; KNL: ## %bb.0: ## %entry -; KNL-NEXT: vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero -; KNL-NEXT: ## EVEX TO VEX Compression encoding: [0xc5,0xfa,0x10,0x07] -; KNL-NEXT: vcvtph2ps %xmm0, %ymm0 ## encoding: [0xc4,0xe2,0x7d,0x13,0xc0] -; KNL-NEXT: vxorps %xmm1, %xmm1, %xmm1 ## encoding: [0xc5,0xf0,0x57,0xc9] -; KNL-NEXT: vcmpneqps %ymm1, %ymm0, %ymm0 ## encoding: [0xc5,0xfc,0xc2,0xc1,0x04] -; KNL-NEXT: vpmovdb %zmm0, %xmm0 ## encoding: [0x62,0xf2,0x7e,0x48,0x31,0xc0] +; KNL-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero +; KNL-NEXT: ## EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6e,0x07] +; KNL-NEXT: vpshuflw $85, %xmm0, %xmm1 ## encoding: [0xc5,0xfb,0x70,0xc8,0x55] +; KNL-NEXT: ## xmm1 = xmm0[1,1,1,1,4,5,6,7] +; KNL-NEXT: vcvtph2ps %xmm1, %xmm1 ## encoding: [0xc4,0xe2,0x79,0x13,0xc9] +; KNL-NEXT: xorl %eax, %eax ## encoding: [0x31,0xc0] +; KNL-NEXT: vxorps %xmm2, %xmm2, %xmm2 ## encoding: [0xc5,0xe8,0x57,0xd2] +; KNL-NEXT: vucomiss %xmm2, %xmm1 ## EVEX TO VEX Compression encoding: [0xc5,0xf8,0x2e,0xca] +; KNL-NEXT: movl $65535, %ecx ## encoding: [0xb9,0xff,0xff,0x00,0x00] +; KNL-NEXT: ## imm = 0xFFFF +; KNL-NEXT: movl $0, %edx ## encoding: [0xba,0x00,0x00,0x00,0x00] +; KNL-NEXT: cmovnel %ecx, %edx ## encoding: [0x0f,0x45,0xd1] +; KNL-NEXT: cmovpl %ecx, %edx ## encoding: [0x0f,0x4a,0xd1] +; KNL-NEXT: vcvtph2ps %xmm0, %xmm0 ## encoding: [0xc4,0xe2,0x79,0x13,0xc0] +; KNL-NEXT: vucomiss %xmm2, %xmm0 ## EVEX TO VEX Compression encoding: [0xc5,0xf8,0x2e,0xc2] +; KNL-NEXT: cmovnel %ecx, %eax ## encoding: [0x0f,0x45,0xc1] +; KNL-NEXT: cmovpl %ecx, %eax ## encoding: [0x0f,0x4a,0xc1] +; KNL-NEXT: vmovd %eax, %xmm0 ## EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6e,0xc0] +; KNL-NEXT: vpinsrw $1, %edx, %xmm0, %xmm0 ## encoding: [0xc5,0xf9,0xc4,0xc2,0x01] +; KNL-NEXT: vpacksswb %xmm0, %xmm0, %xmm0 ## encoding: [0xc5,0xf9,0x63,0xc0] ; KNL-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 ## encoding: [0xc5,0xf9,0xdb,0x05,A,A,A,A] ; KNL-NEXT: ## fixup A - offset: 4, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte ; KNL-NEXT: vpextrw $0, %xmm0, (%rsi) ## encoding: [0xc4,0xe3,0x79,0x15,0x06,0x00] -; KNL-NEXT: vzeroupper ## encoding: [0xc5,0xf8,0x77] ; KNL-NEXT: retq ## encoding: [0xc3] ; ; AVX512BW-LABEL: half_vec_compare: ; AVX512BW: ## %bb.0: ## %entry -; AVX512BW-NEXT: vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero -; AVX512BW-NEXT: ## EVEX TO VEX Compression encoding: [0xc5,0xfa,0x10,0x07] -; AVX512BW-NEXT: vcvtph2ps %xmm0, %ymm0 ## encoding: [0xc4,0xe2,0x7d,0x13,0xc0] -; AVX512BW-NEXT: vxorps %xmm1, %xmm1, %xmm1 ## encoding: [0xc5,0xf0,0x57,0xc9] -; AVX512BW-NEXT: vcmpneqps %ymm1, %ymm0, %ymm0 ## encoding: [0xc5,0xfc,0xc2,0xc1,0x04] -; AVX512BW-NEXT: vpmovdb %zmm0, %xmm0 ## encoding: [0x62,0xf2,0x7e,0x48,0x31,0xc0] +; AVX512BW-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero +; AVX512BW-NEXT: ## EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6e,0x07] +; AVX512BW-NEXT: vpshuflw $85, %xmm0, %xmm1 ## encoding: [0xc5,0xfb,0x70,0xc8,0x55] +; AVX512BW-NEXT: ## xmm1 = xmm0[1,1,1,1,4,5,6,7] +; AVX512BW-NEXT: vcvtph2ps %xmm1, %xmm1 ## encoding: [0xc4,0xe2,0x79,0x13,0xc9] +; AVX512BW-NEXT: xorl %eax, %eax ## encoding: [0x31,0xc0] +; AVX512BW-NEXT: vxorps %xmm2, %xmm2, %xmm2 ## encoding: [0xc5,0xe8,0x57,0xd2] +; AVX512BW-NEXT: vucomiss %xmm2, %xmm1 ## EVEX TO VEX Compression encoding: [0xc5,0xf8,0x2e,0xca] +; AVX512BW-NEXT: movl $65535, %ecx ## encoding: [0xb9,0xff,0xff,0x00,0x00] +; AVX512BW-NEXT: ## imm = 0xFFFF +; AVX512BW-NEXT: movl $0, %edx ## encoding: [0xba,0x00,0x00,0x00,0x00] +; AVX512BW-NEXT: cmovnel %ecx, %edx ## encoding: [0x0f,0x45,0xd1] +; AVX512BW-NEXT: cmovpl %ecx, %edx ## encoding: [0x0f,0x4a,0xd1] +; AVX512BW-NEXT: vcvtph2ps %xmm0, %xmm0 ## encoding: [0xc4,0xe2,0x79,0x13,0xc0] +; AVX512BW-NEXT: vucomiss %xmm2, %xmm0 ## EVEX TO VEX Compression encoding: [0xc5,0xf8,0x2e,0xc2] +; AVX512BW-NEXT: cmovnel %ecx, %eax ## encoding: [0x0f,0x45,0xc1] +; AVX512BW-NEXT: cmovpl %ecx, %eax ## encoding: [0x0f,0x4a,0xc1] +; AVX512BW-NEXT: vmovd %eax, %xmm0 ## EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6e,0xc0] +; AVX512BW-NEXT: vpinsrw $1, %edx, %xmm0, %xmm0 ## EVEX TO VEX Compression encoding: [0xc5,0xf9,0xc4,0xc2,0x01] +; AVX512BW-NEXT: vpacksswb %xmm0, %xmm0, %xmm0 ## encoding: [0xc5,0xf9,0x63,0xc0] ; AVX512BW-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 ## encoding: [0xc5,0xf9,0xdb,0x05,A,A,A,A] ; AVX512BW-NEXT: ## fixup A - offset: 4, value: {{\.?LCPI[0-9]+_[0-9]+}}-4, kind: reloc_riprel_4byte ; AVX512BW-NEXT: vpextrw $0, %xmm0, (%rsi) ## EVEX TO VEX Compression encoding: [0xc4,0xe3,0x79,0x15,0x06,0x00] -; AVX512BW-NEXT: vzeroupper ## encoding: [0xc5,0xf8,0x77] ; AVX512BW-NEXT: retq ## encoding: [0xc3] ; ; SKX-LABEL: half_vec_compare: diff --git a/llvm/test/CodeGen/X86/avx512fp16-fminimum-fmaximum.ll b/llvm/test/CodeGen/X86/avx512fp16-fminimum-fmaximum.ll index 55b86cadfe30..9db57fe68bb4 100644 --- a/llvm/test/CodeGen/X86/avx512fp16-fminimum-fmaximum.ll +++ b/llvm/test/CodeGen/X86/avx512fp16-fminimum-fmaximum.ll @@ -5,6 +5,10 @@ declare half @llvm.minimum.f16(half, half) declare half @llvm.maximum.f16(half, half) declare <8 x half> @llvm.minimum.v8f16(<8 x half>, <8 x half>) declare <8 x half> @llvm.maximum.v8f16(<8 x half>, <8 x half>) +declare <16 x half> @llvm.minimum.v16f16(<16 x half>, <16 x half>) +declare <16 x half> @llvm.maximum.v16f16(<16 x half>, <16 x half>) +declare <32 x half> @llvm.minimum.v32f16(<32 x half>, <32 x half>) +declare <32 x half> @llvm.maximum.v32f16(<32 x half>, <32 x half>) define half @test_fminimum(half %x, half %y) { ; CHECK-LABEL: test_fminimum: @@ -25,20 +29,10 @@ define half @test_fminimum(half %x, half %y) { ret half %z } -define <8 x half> @test_fminimum_scalarize(<8 x half> %x, <8 x half> %y) "no-nans-fp-math"="true" "no-signed-zeros-fp-math"="true" { -; CHECK-LABEL: test_fminimum_scalarize: +define <8 x half> @test_fminimum_v8f16(<8 x half> %x, <8 x half> %y) "no-nans-fp-math"="true" "no-signed-zeros-fp-math"="true" { +; CHECK-LABEL: test_fminimum_v8f16: ; CHECK: # %bb.0: -; CHECK-NEXT: vcmpltph %xmm1, %xmm0, %k1 -; CHECK-NEXT: vpblendmw %xmm0, %xmm1, %xmm2 {%k1} -; CHECK-NEXT: vpbroadcastw {{.*#+}} xmm3 = [32768,32768,32768,32768,32768,32768,32768,32768] -; CHECK-NEXT: vpcmpeqw %xmm3, %xmm0, %k1 -; CHECK-NEXT: vpblendmw %xmm0, %xmm2, %xmm0 {%k1} -; CHECK-NEXT: vpcmpeqw %xmm3, %xmm1, %k1 -; CHECK-NEXT: vmovdqu16 %xmm1, %xmm0 {%k1} -; CHECK-NEXT: vxorps %xmm1, %xmm1, %xmm1 -; CHECK-NEXT: vcmpeqph %xmm1, %xmm2, %k1 -; CHECK-NEXT: vmovdqu16 %xmm0, %xmm2 {%k1} -; CHECK-NEXT: vmovdqa %xmm2, %xmm0 +; CHECK-NEXT: vminph %xmm1, %xmm0, %xmm0 ; CHECK-NEXT: retq %r = call <8 x half> @llvm.minimum.v8f16(<8 x half> %x, <8 x half> %y) ret <8 x half> %r @@ -113,19 +107,10 @@ define half @test_fmaximum(half %x, half %y) { ret half %r } -define <8 x half> @test_fmaximum_scalarize(<8 x half> %x, <8 x half> %y) "no-nans-fp-math"="true" "no-signed-zeros-fp-math"="true" { -; CHECK-LABEL: test_fmaximum_scalarize: +define <8 x half> @test_fmaximum_v8f16(<8 x half> %x, <8 x half> %y) "no-nans-fp-math"="true" "no-signed-zeros-fp-math"="true" { +; CHECK-LABEL: test_fmaximum_v8f16: ; CHECK: # %bb.0: -; CHECK-NEXT: vcmpltph %xmm0, %xmm1, %k1 -; CHECK-NEXT: vpblendmw %xmm0, %xmm1, %xmm2 {%k1} -; CHECK-NEXT: vptestnmw %xmm0, %xmm0, %k1 -; CHECK-NEXT: vpblendmw %xmm0, %xmm2, %xmm0 {%k1} -; CHECK-NEXT: vptestnmw %xmm1, %xmm1, %k1 -; CHECK-NEXT: vmovdqu16 %xmm1, %xmm0 {%k1} -; CHECK-NEXT: vxorps %xmm1, %xmm1, %xmm1 -; CHECK-NEXT: vcmpeqph %xmm1, %xmm2, %k1 -; CHECK-NEXT: vmovdqu16 %xmm0, %xmm2 {%k1} -; CHECK-NEXT: vmovdqa %xmm2, %xmm0 +; CHECK-NEXT: vmaxph %xmm1, %xmm0, %xmm0 ; CHECK-NEXT: retq %r = call <8 x half> @llvm.maximum.v8f16(<8 x half> %x, <8 x half> %y) ret <8 x half> %r @@ -186,3 +171,50 @@ define half @test_fmaximum_combine_cmps(half %x, half %y) { %2 = tail call half @llvm.maximum.f16(half %x, half %1) ret half %2 } + +define <16 x half> @test_fminimum_v16f16(<16 x half> %x, <16 x half> %y) "no-nans-fp-math"="true" "no-signed-zeros-fp-math"="true" { +; CHECK-LABEL: test_fminimum_v16f16: +; CHECK: # %bb.0: +; CHECK-NEXT: vminph %ymm1, %ymm0, %ymm0 +; CHECK-NEXT: retq + %r = call <16 x half> @llvm.minimum.v16f16(<16 x half> %x, <16 x half> %y) + ret <16 x half> %r +} + +define <16 x half> @test_fmaximum_v16f16_nans(<16 x half> %x, <16 x half> %y) "no-signed-zeros-fp-math"="true" { +; CHECK-LABEL: test_fmaximum_v16f16_nans: +; CHECK: # %bb.0: +; CHECK-NEXT: vmaxph %ymm1, %ymm0, %ymm1 +; CHECK-NEXT: vcmpunordph %ymm0, %ymm0, %k1 +; CHECK-NEXT: vmovdqu16 %ymm0, %ymm1 {%k1} +; CHECK-NEXT: vmovdqa %ymm1, %ymm0 +; CHECK-NEXT: retq + %r = call <16 x half> @llvm.maximum.v16f16(<16 x half> %x, <16 x half> %y) + ret <16 x half> %r +} + +define <32 x half> @test_fminimum_v32f16_szero(<32 x half> %x, <32 x half> %y) "no-nans-fp-math"="true" { +; CHECK-LABEL: test_fminimum_v32f16_szero: +; CHECK: # %bb.0: +; CHECK-NEXT: vpmovw2m %zmm0, %k1 +; CHECK-NEXT: vpblendmw %zmm0, %zmm1, %zmm2 {%k1} +; CHECK-NEXT: vmovdqu16 %zmm1, %zmm0 {%k1} +; CHECK-NEXT: vminph %zmm2, %zmm0, %zmm0 +; CHECK-NEXT: retq + %r = call <32 x half> @llvm.minimum.v32f16(<32 x half> %x, <32 x half> %y) + ret <32 x half> %r +} + +define <32 x half> @test_fmaximum_v32f16_nans_szero(<32 x half> %x, <32 x half> %y) { +; CHECK-LABEL: test_fmaximum_v32f16_nans_szero: +; CHECK: # %bb.0: +; CHECK-NEXT: vpmovw2m %zmm0, %k1 +; CHECK-NEXT: vpblendmw %zmm1, %zmm0, %zmm2 {%k1} +; CHECK-NEXT: vmovdqu16 %zmm0, %zmm1 {%k1} +; CHECK-NEXT: vmaxph %zmm2, %zmm1, %zmm0 +; CHECK-NEXT: vcmpunordph %zmm1, %zmm1, %k1 +; CHECK-NEXT: vmovdqu16 %zmm1, %zmm0 {%k1} +; CHECK-NEXT: retq + %r = call <32 x half> @llvm.maximum.v32f16(<32 x half> %x, <32 x half> %y) + ret <32 x half> %r +} diff --git a/llvm/test/CodeGen/X86/basic-block-sections-blockaddress-taken.ll b/llvm/test/CodeGen/X86/basic-block-sections-blockaddress-taken.ll index 2e562c350be6..efdbfef5b87f 100644 --- a/llvm/test/CodeGen/X86/basic-block-sections-blockaddress-taken.ll +++ b/llvm/test/CodeGen/X86/basic-block-sections-blockaddress-taken.ll @@ -6,7 +6,6 @@ entry: %1 = select i1 %0, ptr blockaddress(@foo, %bb1), ptr blockaddress(@foo, %bb2) ; <ptr> [#uses=1] indirectbr ptr %1, [label %bb1, label %bb2] -; CHECK: .text ; CHECK: .section .text.foo,"ax",@progbits ; CHECK-LABEL: foo: ; CHECK: movl $.Ltmp0, %eax diff --git a/llvm/test/CodeGen/X86/bfloat.ll b/llvm/test/CodeGen/X86/bfloat.ll index 27348dd31958..a6b3e3fd1fd1 100644 --- a/llvm/test/CodeGen/X86/bfloat.ll +++ b/llvm/test/CodeGen/X86/bfloat.ll @@ -1983,8 +1983,6 @@ define bfloat @PR115710(fp128 %0) nounwind { ; X86-NEXT: vmovaps {{[0-9]+}}(%esp), %xmm0 ; X86-NEXT: vmovups %xmm0, (%esp) ; X86-NEXT: calll __trunctfbf2 -; X86-NEXT: # kill: def $ax killed $ax def $eax -; X86-NEXT: vmovw %eax, %xmm0 ; X86-NEXT: addl $28, %esp ; X86-NEXT: retl ; diff --git a/llvm/test/CodeGen/X86/coff-alias-type.ll b/llvm/test/CodeGen/X86/coff-alias-type.ll index 0e041fc032ff..a242cd2d77d7 100644 --- a/llvm/test/CodeGen/X86/coff-alias-type.ll +++ b/llvm/test/CodeGen/X86/coff-alias-type.ll @@ -13,6 +13,7 @@ entry: ; CHECK-NEXT: .scl 2 ; CHECK-NEXT: .type 32 ; CHECK-NEXT: .endef +; CHECK-NEXT: .text ; CHECK-NEXT: .globl _ZN8MyStructC2Ev ; CHECK: {{^}}_ZN8MyStructC2Ev: diff --git a/llvm/test/CodeGen/X86/combine-fabs.ll b/llvm/test/CodeGen/X86/combine-fabs.ll index b9aad9075261..7aa6628cb7f3 100644 --- a/llvm/test/CodeGen/X86/combine-fabs.ll +++ b/llvm/test/CodeGen/X86/combine-fabs.ll @@ -172,6 +172,42 @@ define void @combine_fabs_int_f32(ptr %src, ptr %dst) { ret void } +define void @combine_fabs_int_rmw_bfloat(ptr %ptr) nounwind { +; SSE-LABEL: combine_fabs_int_rmw_bfloat: +; SSE: # %bb.0: +; SSE-NEXT: andb $127, 1(%rdi) +; SSE-NEXT: retq +; +; AVX-LABEL: combine_fabs_int_rmw_bfloat: +; AVX: # %bb.0: +; AVX-NEXT: andb $127, 1(%rdi) +; AVX-NEXT: retq + %1 = load bfloat, ptr %ptr + %2 = call bfloat @llvm.fabs.bf16(bfloat %1) + store bfloat %2, ptr %ptr + ret void +} + +define void @combine_fabs_int_half(ptr %src, ptr %dst) nounwind { +; SSE-LABEL: combine_fabs_int_half: +; SSE: # %bb.0: +; SSE-NEXT: movzwl (%rdi), %eax +; SSE-NEXT: andl $32767, %eax # imm = 0x7FFF +; SSE-NEXT: movw %ax, (%rsi) +; SSE-NEXT: retq +; +; AVX-LABEL: combine_fabs_int_half: +; AVX: # %bb.0: +; AVX-NEXT: movzwl (%rdi), %eax +; AVX-NEXT: andl $32767, %eax # imm = 0x7FFF +; AVX-NEXT: movw %ax, (%rsi) +; AVX-NEXT: retq + %1 = load half, ptr %src + %2 = call half @llvm.fabs.f16(half %1) + store half %2, ptr %dst + ret void +} + ; don't convert vector to scalar define void @combine_fabs_vec_int_v4f32(ptr %src, ptr %dst) { ; SSE-LABEL: combine_fabs_vec_int_v4f32: diff --git a/llvm/test/CodeGen/X86/combine-fneg.ll b/llvm/test/CodeGen/X86/combine-fneg.ll index 855b64229a9c..8ca7fb81563f 100644 --- a/llvm/test/CodeGen/X86/combine-fneg.ll +++ b/llvm/test/CodeGen/X86/combine-fneg.ll @@ -206,6 +206,45 @@ define <4 x float> @fneg(<4 x float> %Q) nounwind { } ; store(fneg(load())) - convert scalar to integer +define void @fneg_int_rmw_half(ptr %ptr) nounwind { +; X86-SSE-LABEL: fneg_int_rmw_half: +; X86-SSE: # %bb.0: +; X86-SSE-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-SSE-NEXT: xorb $-128, 1(%eax) +; X86-SSE-NEXT: retl +; +; X64-SSE-LABEL: fneg_int_rmw_half: +; X64-SSE: # %bb.0: +; X64-SSE-NEXT: xorb $-128, 1(%rdi) +; X64-SSE-NEXT: retq + %1 = load half, ptr %ptr + %2 = fneg half %1 + store half %2, ptr %ptr + ret void +} + +define void @fneg_int_bfloat(ptr %src, ptr %dst) nounwind { +; X86-SSE-LABEL: fneg_int_bfloat: +; X86-SSE: # %bb.0: +; X86-SSE-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-SSE-NEXT: movl {{[0-9]+}}(%esp), %ecx +; X86-SSE-NEXT: movzwl (%ecx), %ecx +; X86-SSE-NEXT: xorl $32768, %ecx # imm = 0x8000 +; X86-SSE-NEXT: movw %cx, (%eax) +; X86-SSE-NEXT: retl +; +; X64-SSE-LABEL: fneg_int_bfloat: +; X64-SSE: # %bb.0: +; X64-SSE-NEXT: movzwl (%rdi), %eax +; X64-SSE-NEXT: xorl $32768, %eax # imm = 0x8000 +; X64-SSE-NEXT: movw %ax, (%rsi) +; X64-SSE-NEXT: retq + %1 = load bfloat, ptr %src + %2 = fneg bfloat %1 + store bfloat %2, ptr %dst + ret void +} + define void @fneg_int_rmw_f32(ptr %ptr) { ; X86-SSE-LABEL: fneg_int_rmw_f32: ; X86-SSE: # %bb.0: diff --git a/llvm/test/CodeGen/X86/combine-sdiv.ll b/llvm/test/CodeGen/X86/combine-sdiv.ll index 76f216045004..42f09d04da26 100644 --- a/llvm/test/CodeGen/X86/combine-sdiv.ll +++ b/llvm/test/CodeGen/X86/combine-sdiv.ll @@ -2190,7 +2190,7 @@ define <16 x i8> @non_splat_minus_one_divisor_1(<16 x i8> %A) { ; SSE41-NEXT: pxor %xmm4, %xmm4 ; SSE41-NEXT: punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3],xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7] ; SSE41-NEXT: pmovzxbw {{.*#+}} xmm2 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero -; SSE41-NEXT: psllw $1, %xmm2 +; SSE41-NEXT: paddw %xmm2, %xmm2 ; SSE41-NEXT: pblendw {{.*#+}} xmm2 = xmm4[0,1],xmm2[2],xmm4[3,4,5],xmm2[6],xmm4[7] ; SSE41-NEXT: psrlw $8, %xmm2 ; SSE41-NEXT: punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm0[8],xmm3[9],xmm0[9],xmm3[10],xmm0[10],xmm3[11],xmm0[11],xmm3[12],xmm0[12],xmm3[13],xmm0[13],xmm3[14],xmm0[14],xmm3[15],xmm0[15] @@ -2202,9 +2202,9 @@ define <16 x i8> @non_splat_minus_one_divisor_1(<16 x i8> %A) { ; SSE41-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm2[8],xmm0[9],xmm2[9],xmm0[10],xmm2[10],xmm0[11],xmm2[11],xmm0[12],xmm2[12],xmm0[13],xmm2[13],xmm0[14],xmm2[14],xmm0[15],xmm2[15] ; SSE41-NEXT: psraw $8, %xmm0 ; SSE41-NEXT: movdqa %xmm0, %xmm3 -; SSE41-NEXT: psllw $1, %xmm3 -; SSE41-NEXT: psllw $7, %xmm0 -; SSE41-NEXT: pblendw {{.*#+}} xmm0 = xmm0[0,1,2,3,4],xmm3[5],xmm0[6],xmm3[7] +; SSE41-NEXT: psllw $7, %xmm3 +; SSE41-NEXT: paddw %xmm0, %xmm0 +; SSE41-NEXT: pblendw {{.*#+}} xmm0 = xmm3[0,1,2,3,4],xmm0[5],xmm3[6],xmm0[7] ; SSE41-NEXT: psrlw $8, %xmm0 ; SSE41-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] ; SSE41-NEXT: psraw $8, %xmm2 @@ -2225,7 +2225,7 @@ define <16 x i8> @non_splat_minus_one_divisor_1(<16 x i8> %A) { ; AVX1-NEXT: vpcmpgtb %xmm0, %xmm1, %xmm2 ; AVX1-NEXT: vpunpcklbw {{.*#+}} xmm3 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7] ; AVX1-NEXT: vpmovzxbw {{.*#+}} xmm4 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero -; AVX1-NEXT: vpsllw $1, %xmm4, %xmm4 +; AVX1-NEXT: vpaddw %xmm4, %xmm4, %xmm4 ; AVX1-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0,1],xmm4[2],xmm3[3,4,5],xmm4[6],xmm3[7] ; AVX1-NEXT: vpsrlw $8, %xmm3, %xmm3 ; AVX1-NEXT: vpunpckhbw {{.*#+}} xmm1 = xmm2[8],xmm1[8],xmm2[9],xmm1[9],xmm2[10],xmm1[10],xmm2[11],xmm1[11],xmm2[12],xmm1[12],xmm2[13],xmm1[13],xmm2[14],xmm1[14],xmm2[15],xmm1[15] @@ -2235,9 +2235,9 @@ define <16 x i8> @non_splat_minus_one_divisor_1(<16 x i8> %A) { ; AVX1-NEXT: vpaddb %xmm1, %xmm0, %xmm1 ; AVX1-NEXT: vpunpckhbw {{.*#+}} xmm2 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15] ; AVX1-NEXT: vpsraw $8, %xmm2, %xmm2 -; AVX1-NEXT: vpsllw $1, %xmm2, %xmm3 -; AVX1-NEXT: vpsllw $7, %xmm2, %xmm2 -; AVX1-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0,1,2,3,4],xmm3[5],xmm2[6],xmm3[7] +; AVX1-NEXT: vpsllw $7, %xmm2, %xmm3 +; AVX1-NEXT: vpaddw %xmm2, %xmm2, %xmm2 +; AVX1-NEXT: vpblendw {{.*#+}} xmm2 = xmm3[0,1,2,3,4],xmm2[5],xmm3[6],xmm2[7] ; AVX1-NEXT: vpsrlw $8, %xmm2, %xmm2 ; AVX1-NEXT: vpunpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] ; AVX1-NEXT: vpsraw $8, %xmm1, %xmm1 diff --git a/llvm/test/CodeGen/X86/elf-unique-sections-by-flags.ll b/llvm/test/CodeGen/X86/elf-unique-sections-by-flags.ll index 0ca08d4b3fea..01a113fdd9b2 100644 --- a/llvm/test/CodeGen/X86/elf-unique-sections-by-flags.ll +++ b/llvm/test/CodeGen/X86/elf-unique-sections-by-flags.ll @@ -11,10 +11,8 @@ define i32 @fn_text() { entry: ret i32 0 } -; CHECK: .text{{$}} -; CHECK-NEXT: .file ; FNSECTIONS: .section .text.fn_text,"ax",@progbits{{$}} -; CHECK-NEXT: .globl fn_text +; CHECK: .globl fn_text ; CHECK: fn_text: ; A second function placed in .text, to check the behaviour with -function-sections. diff --git a/llvm/test/CodeGen/X86/fminimum-fmaximum.ll b/llvm/test/CodeGen/X86/fminimum-fmaximum.ll index 07701f082b0e..1dcce5336895 100644 --- a/llvm/test/CodeGen/X86/fminimum-fmaximum.ll +++ b/llvm/test/CodeGen/X86/fminimum-fmaximum.ll @@ -3,6 +3,7 @@ ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefixes=AVX,AVX1 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefixes=AVX,AVX512,AVX512F ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512dq | FileCheck %s --check-prefixes=AVX,AVX512,AVX512DQ +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx10.2-256 | FileCheck %s --check-prefixes=AVX10_2 ; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx | FileCheck %s --check-prefixes=X86 declare float @llvm.maximum.f32(float, float) @@ -73,6 +74,11 @@ define float @test_fmaximum(float %x, float %y) nounwind { ; AVX512-NEXT: vmovss %xmm1, %xmm0, %xmm0 {%k1} ; AVX512-NEXT: retq ; +; AVX10_2-LABEL: test_fmaximum: +; AVX10_2: # %bb.0: +; AVX10_2-NEXT: vminmaxss $1, %xmm1, %xmm0 +; AVX10_2-NEXT: retq +; ; X86-LABEL: test_fmaximum: ; X86: # %bb.0: ; X86-NEXT: pushl %eax @@ -110,6 +116,11 @@ define <4 x float> @test_fmaximum_scalarize(<4 x float> %x, <4 x float> %y) "no- ; AVX-NEXT: vmaxps %xmm1, %xmm0, %xmm0 ; AVX-NEXT: retq ; +; AVX10_2-LABEL: test_fmaximum_scalarize: +; AVX10_2: # %bb.0: +; AVX10_2-NEXT: vminmaxps $1, %xmm1, %xmm0, %xmm0 +; AVX10_2-NEXT: retq +; ; X86-LABEL: test_fmaximum_scalarize: ; X86: # %bb.0: ; X86-NEXT: vmaxps %xmm1, %xmm0, %xmm0 @@ -129,6 +140,11 @@ define float @test_fmaximum_nan0(float %x, float %y) { ; AVX-NEXT: vmovss {{.*#+}} xmm0 = [NaN,0.0E+0,0.0E+0,0.0E+0] ; AVX-NEXT: retq ; +; AVX10_2-LABEL: test_fmaximum_nan0: +; AVX10_2: # %bb.0: +; AVX10_2-NEXT: vmovss {{.*#+}} xmm0 = [NaN,0.0E+0,0.0E+0,0.0E+0] +; AVX10_2-NEXT: retq +; ; X86-LABEL: test_fmaximum_nan0: ; X86: # %bb.0: ; X86-NEXT: flds {{\.?LCPI[0-9]+_[0-9]+}} @@ -148,6 +164,11 @@ define float @test_fmaximum_nan1(float %x, float %y) { ; AVX-NEXT: vmovss {{.*#+}} xmm0 = [NaN,0.0E+0,0.0E+0,0.0E+0] ; AVX-NEXT: retq ; +; AVX10_2-LABEL: test_fmaximum_nan1: +; AVX10_2: # %bb.0: +; AVX10_2-NEXT: vmovss {{.*#+}} xmm0 = [NaN,0.0E+0,0.0E+0,0.0E+0] +; AVX10_2-NEXT: retq +; ; X86-LABEL: test_fmaximum_nan1: ; X86: # %bb.0: ; X86-NEXT: flds {{\.?LCPI[0-9]+_[0-9]+}} @@ -207,7 +228,7 @@ define float @test_fmaximum_nnan(float %x, float %y) nounwind { ; AVX512DQ: # %bb.0: ; AVX512DQ-NEXT: vaddss %xmm1, %xmm0, %xmm2 ; AVX512DQ-NEXT: vsubss %xmm1, %xmm0, %xmm0 -; AVX512DQ-NEXT: vfpclassss $3, %xmm0, %k0 +; AVX512DQ-NEXT: vfpclassss $3, %xmm0, %k0 # k0 = isQuietNaN(xmm0) | isPositiveZero(xmm0) ; AVX512DQ-NEXT: kmovw %k0, %k1 ; AVX512DQ-NEXT: vmovaps %xmm2, %xmm1 ; AVX512DQ-NEXT: vmovss %xmm0, %xmm1, %xmm1 {%k1} @@ -215,6 +236,13 @@ define float @test_fmaximum_nnan(float %x, float %y) nounwind { ; AVX512DQ-NEXT: vmaxss %xmm1, %xmm0, %xmm0 ; AVX512DQ-NEXT: retq ; +; AVX10_2-LABEL: test_fmaximum_nnan: +; AVX10_2: # %bb.0: +; AVX10_2-NEXT: vaddss %xmm1, %xmm0, %xmm2 +; AVX10_2-NEXT: vsubss %xmm1, %xmm0, %xmm0 +; AVX10_2-NEXT: vminmaxss $1, %xmm0, %xmm2 +; AVX10_2-NEXT: retq +; ; X86-LABEL: test_fmaximum_nnan: ; X86: # %bb.0: ; X86-NEXT: pushl %eax @@ -272,6 +300,12 @@ define double @test_fmaximum_zero0(double %x, double %y) nounwind { ; AVX512-NEXT: vmovsd %xmm1, %xmm0, %xmm0 {%k1} ; AVX512-NEXT: retq ; +; AVX10_2-LABEL: test_fmaximum_zero0: +; AVX10_2: # %bb.0: +; AVX10_2-NEXT: vxorpd %xmm0, %xmm0, %xmm0 +; AVX10_2-NEXT: vminmaxsd $1, %xmm0, %xmm1 +; AVX10_2-NEXT: retq +; ; X86-LABEL: test_fmaximum_zero0: ; X86: # %bb.0: ; X86-NEXT: pushl %ebp @@ -323,6 +357,12 @@ define double @test_fmaximum_zero1(double %x, double %y) nounwind { ; AVX512-NEXT: vmovapd %xmm1, %xmm0 ; AVX512-NEXT: retq ; +; AVX10_2-LABEL: test_fmaximum_zero1: +; AVX10_2: # %bb.0: +; AVX10_2-NEXT: vxorpd %xmm1, %xmm1, %xmm1 +; AVX10_2-NEXT: vminmaxsd $1, %xmm1, %xmm0 +; AVX10_2-NEXT: retq +; ; X86-LABEL: test_fmaximum_zero1: ; X86: # %bb.0: ; X86-NEXT: pushl %ebp @@ -354,6 +394,11 @@ define double @test_fmaximum_zero2(double %x, double %y) { ; AVX-NEXT: vxorps %xmm0, %xmm0, %xmm0 ; AVX-NEXT: retq ; +; AVX10_2-LABEL: test_fmaximum_zero2: +; AVX10_2: # %bb.0: +; AVX10_2-NEXT: vxorps %xmm0, %xmm0, %xmm0 +; AVX10_2-NEXT: retq +; ; X86-LABEL: test_fmaximum_zero2: ; X86: # %bb.0: ; X86-NEXT: fldz @@ -390,6 +435,11 @@ define float @test_fmaximum_nsz(float %x, float %y) "no-signed-zeros-fp-math"="t ; AVX512-NEXT: vmovaps %xmm1, %xmm0 ; AVX512-NEXT: retq ; +; AVX10_2-LABEL: test_fmaximum_nsz: +; AVX10_2: # %bb.0: +; AVX10_2-NEXT: vminmaxss $1, %xmm1, %xmm0 +; AVX10_2-NEXT: retq +; ; X86-LABEL: test_fmaximum_nsz: ; X86: # %bb.0: ; X86-NEXT: pushl %eax @@ -466,7 +516,7 @@ define float @test_fmaximum_combine_cmps(float %x, float %y) nounwind { ; AVX512DQ-LABEL: test_fmaximum_combine_cmps: ; AVX512DQ: # %bb.0: ; AVX512DQ-NEXT: vdivss %xmm0, %xmm1, %xmm1 -; AVX512DQ-NEXT: vfpclassss $3, %xmm0, %k0 +; AVX512DQ-NEXT: vfpclassss $3, %xmm0, %k0 # k0 = isQuietNaN(xmm0) | isPositiveZero(xmm0) ; AVX512DQ-NEXT: kmovw %k0, %k1 ; AVX512DQ-NEXT: vmovaps %xmm1, %xmm2 ; AVX512DQ-NEXT: vmovss %xmm0, %xmm2, %xmm2 {%k1} @@ -474,6 +524,12 @@ define float @test_fmaximum_combine_cmps(float %x, float %y) nounwind { ; AVX512DQ-NEXT: vmaxss %xmm2, %xmm0, %xmm0 ; AVX512DQ-NEXT: retq ; +; AVX10_2-LABEL: test_fmaximum_combine_cmps: +; AVX10_2: # %bb.0: +; AVX10_2-NEXT: vdivss %xmm0, %xmm1, %xmm1 +; AVX10_2-NEXT: vminmaxss $1, %xmm1, %xmm0 +; AVX10_2-NEXT: retq +; ; X86-LABEL: test_fmaximum_combine_cmps: ; X86: # %bb.0: ; X86-NEXT: pushl %eax @@ -562,6 +618,11 @@ define float @test_fminimum(float %x, float %y) nounwind { ; AVX512-NEXT: vmovaps %xmm1, %xmm0 ; AVX512-NEXT: retq ; +; AVX10_2-LABEL: test_fminimum: +; AVX10_2: # %bb.0: +; AVX10_2-NEXT: vminmaxss $0, %xmm1, %xmm0 +; AVX10_2-NEXT: retq +; ; X86-LABEL: test_fminimum: ; X86: # %bb.0: ; X86-NEXT: pushl %eax @@ -599,6 +660,11 @@ define <2 x double> @test_fminimum_scalarize(<2 x double> %x, <2 x double> %y) " ; AVX-NEXT: vminpd %xmm1, %xmm0, %xmm0 ; AVX-NEXT: retq ; +; AVX10_2-LABEL: test_fminimum_scalarize: +; AVX10_2: # %bb.0: +; AVX10_2-NEXT: vminmaxpd $0, %xmm1, %xmm0, %xmm0 +; AVX10_2-NEXT: retq +; ; X86-LABEL: test_fminimum_scalarize: ; X86: # %bb.0: ; X86-NEXT: vminpd %xmm1, %xmm0, %xmm0 @@ -618,6 +684,11 @@ define float @test_fminimum_nan0(float %x, float %y) { ; AVX-NEXT: vmovss {{.*#+}} xmm0 = [NaN,0.0E+0,0.0E+0,0.0E+0] ; AVX-NEXT: retq ; +; AVX10_2-LABEL: test_fminimum_nan0: +; AVX10_2: # %bb.0: +; AVX10_2-NEXT: vmovss {{.*#+}} xmm0 = [NaN,0.0E+0,0.0E+0,0.0E+0] +; AVX10_2-NEXT: retq +; ; X86-LABEL: test_fminimum_nan0: ; X86: # %bb.0: ; X86-NEXT: flds {{\.?LCPI[0-9]+_[0-9]+}} @@ -637,6 +708,11 @@ define float @test_fminimum_nan1(float %x, float %y) { ; AVX-NEXT: vmovss {{.*#+}} xmm0 = [NaN,0.0E+0,0.0E+0,0.0E+0] ; AVX-NEXT: retq ; +; AVX10_2-LABEL: test_fminimum_nan1: +; AVX10_2: # %bb.0: +; AVX10_2-NEXT: vmovss {{.*#+}} xmm0 = [NaN,0.0E+0,0.0E+0,0.0E+0] +; AVX10_2-NEXT: retq +; ; X86-LABEL: test_fminimum_nan1: ; X86: # %bb.0: ; X86-NEXT: flds {{\.?LCPI[0-9]+_[0-9]+}} @@ -687,7 +763,7 @@ define double @test_fminimum_nnan(double %x, double %y) "no-nans-fp-math"="true" ; ; AVX512DQ-LABEL: test_fminimum_nnan: ; AVX512DQ: # %bb.0: -; AVX512DQ-NEXT: vfpclasssd $5, %xmm1, %k0 +; AVX512DQ-NEXT: vfpclasssd $5, %xmm1, %k0 # k0 = isQuietNaN(xmm1) | isNegativeZero(xmm1) ; AVX512DQ-NEXT: kmovw %k0, %k1 ; AVX512DQ-NEXT: vmovapd %xmm0, %xmm2 ; AVX512DQ-NEXT: vmovsd %xmm1, %xmm2, %xmm2 {%k1} @@ -695,6 +771,11 @@ define double @test_fminimum_nnan(double %x, double %y) "no-nans-fp-math"="true" ; AVX512DQ-NEXT: vminsd %xmm2, %xmm1, %xmm0 ; AVX512DQ-NEXT: retq ; +; AVX10_2-LABEL: test_fminimum_nnan: +; AVX10_2: # %bb.0: +; AVX10_2-NEXT: vminmaxsd $0, %xmm1, %xmm0 +; AVX10_2-NEXT: retq +; ; X86-LABEL: test_fminimum_nnan: ; X86: # %bb.0: ; X86-NEXT: pushl %ebp @@ -749,6 +830,11 @@ define double @test_fminimum_zero0(double %x, double %y) nounwind { ; AVX512-NEXT: vmovsd %xmm1, %xmm0, %xmm0 {%k1} ; AVX512-NEXT: retq ; +; AVX10_2-LABEL: test_fminimum_zero0: +; AVX10_2: # %bb.0: +; AVX10_2-NEXT: vminmaxsd $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 +; AVX10_2-NEXT: retq +; ; X86-LABEL: test_fminimum_zero0: ; X86: # %bb.0: ; X86-NEXT: pushl %ebp @@ -796,6 +882,11 @@ define double @test_fminimum_zero1(double %x, double %y) nounwind { ; AVX512-NEXT: vmovapd %xmm1, %xmm0 ; AVX512-NEXT: retq ; +; AVX10_2-LABEL: test_fminimum_zero1: +; AVX10_2: # %bb.0: +; AVX10_2-NEXT: vminmaxsd $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 +; AVX10_2-NEXT: retq +; ; X86-LABEL: test_fminimum_zero1: ; X86: # %bb.0: ; X86-NEXT: pushl %ebp @@ -826,6 +917,11 @@ define double @test_fminimum_zero2(double %x, double %y) { ; AVX-NEXT: vmovsd {{.*#+}} xmm0 = [-0.0E+0,0.0E+0] ; AVX-NEXT: retq ; +; AVX10_2-LABEL: test_fminimum_zero2: +; AVX10_2: # %bb.0: +; AVX10_2-NEXT: vmovsd {{.*#+}} xmm0 = [-0.0E+0,0.0E+0] +; AVX10_2-NEXT: retq +; ; X86-LABEL: test_fminimum_zero2: ; X86: # %bb.0: ; X86-NEXT: fldz @@ -863,6 +959,11 @@ define float @test_fminimum_nsz(float %x, float %y) nounwind { ; AVX512-NEXT: vmovaps %xmm1, %xmm0 ; AVX512-NEXT: retq ; +; AVX10_2-LABEL: test_fminimum_nsz: +; AVX10_2: # %bb.0: +; AVX10_2-NEXT: vminmaxss $0, %xmm1, %xmm0 +; AVX10_2-NEXT: retq +; ; X86-LABEL: test_fminimum_nsz: ; X86: # %bb.0: ; X86-NEXT: pushl %eax @@ -940,7 +1041,7 @@ define float @test_fminimum_combine_cmps(float %x, float %y) nounwind { ; AVX512DQ-LABEL: test_fminimum_combine_cmps: ; AVX512DQ: # %bb.0: ; AVX512DQ-NEXT: vdivss %xmm0, %xmm1, %xmm1 -; AVX512DQ-NEXT: vfpclassss $5, %xmm0, %k0 +; AVX512DQ-NEXT: vfpclassss $5, %xmm0, %k0 # k0 = isQuietNaN(xmm0) | isNegativeZero(xmm0) ; AVX512DQ-NEXT: kmovw %k0, %k1 ; AVX512DQ-NEXT: vmovaps %xmm1, %xmm2 ; AVX512DQ-NEXT: vmovss %xmm0, %xmm2, %xmm2 {%k1} @@ -948,6 +1049,12 @@ define float @test_fminimum_combine_cmps(float %x, float %y) nounwind { ; AVX512DQ-NEXT: vminss %xmm2, %xmm0, %xmm0 ; AVX512DQ-NEXT: retq ; +; AVX10_2-LABEL: test_fminimum_combine_cmps: +; AVX10_2: # %bb.0: +; AVX10_2-NEXT: vdivss %xmm0, %xmm1, %xmm1 +; AVX10_2-NEXT: vminmaxss $0, %xmm1, %xmm0 +; AVX10_2-NEXT: retq +; ; X86-LABEL: test_fminimum_combine_cmps: ; X86: # %bb.0: ; X86-NEXT: pushl %eax @@ -1009,6 +1116,11 @@ define <2 x double> @test_fminimum_vector(<2 x double> %x, <2 x double> %y) { ; AVX-NEXT: vblendvpd %xmm2, %xmm0, %xmm1, %xmm0 ; AVX-NEXT: retq ; +; AVX10_2-LABEL: test_fminimum_vector: +; AVX10_2: # %bb.0: +; AVX10_2-NEXT: vminmaxpd $0, %xmm1, %xmm0, %xmm0 +; AVX10_2-NEXT: retq +; ; X86-LABEL: test_fminimum_vector: ; X86: # %bb.0: ; X86-NEXT: vblendvpd %xmm0, %xmm0, %xmm1, %xmm2 @@ -1032,6 +1144,11 @@ define <4 x float> @test_fmaximum_vector(<4 x float> %x, <4 x float> %y) "no-nan ; AVX-NEXT: vmaxps %xmm1, %xmm0, %xmm0 ; AVX-NEXT: retq ; +; AVX10_2-LABEL: test_fmaximum_vector: +; AVX10_2: # %bb.0: +; AVX10_2-NEXT: vminmaxps $1, %xmm1, %xmm0, %xmm0 +; AVX10_2-NEXT: retq +; ; X86-LABEL: test_fmaximum_vector: ; X86: # %bb.0: ; X86-NEXT: vmaxps %xmm1, %xmm0, %xmm0 @@ -1054,6 +1171,12 @@ define <2 x double> @test_fminimum_vector_zero(<2 x double> %x) { ; AVX-NEXT: vminpd %xmm0, %xmm1, %xmm0 ; AVX-NEXT: retq ; +; AVX10_2-LABEL: test_fminimum_vector_zero: +; AVX10_2: # %bb.0: +; AVX10_2-NEXT: vxorpd %xmm1, %xmm1, %xmm1 +; AVX10_2-NEXT: vminmaxpd $0, %xmm1, %xmm0, %xmm0 +; AVX10_2-NEXT: retq +; ; X86-LABEL: test_fminimum_vector_zero: ; X86: # %bb.0: ; X86-NEXT: vxorpd %xmm1, %xmm1, %xmm1 @@ -1077,6 +1200,11 @@ define <4 x float> @test_fmaximum_vector_signed_zero(<4 x float> %x) { ; AVX-NEXT: vmaxps %xmm0, %xmm1, %xmm0 ; AVX-NEXT: retq ; +; AVX10_2-LABEL: test_fmaximum_vector_signed_zero: +; AVX10_2: # %bb.0: +; AVX10_2-NEXT: vminmaxps $1, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm0, %xmm0 +; AVX10_2-NEXT: retq +; ; X86-LABEL: test_fmaximum_vector_signed_zero: ; X86: # %bb.0: ; X86-NEXT: vbroadcastss {{.*#+}} xmm1 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0] @@ -1102,6 +1230,13 @@ define <2 x double> @test_fminimum_vector_partially_zero(<2 x double> %x) { ; AVX-NEXT: vminpd %xmm0, %xmm1, %xmm0 ; AVX-NEXT: retq ; +; AVX10_2-LABEL: test_fminimum_vector_partially_zero: +; AVX10_2: # %bb.0: +; AVX10_2-NEXT: vxorpd %xmm1, %xmm1, %xmm1 +; AVX10_2-NEXT: vmovhpd {{.*#+}} xmm1 = xmm1[0],mem[0] +; AVX10_2-NEXT: vminmaxpd $0, %xmm1, %xmm0, %xmm0 +; AVX10_2-NEXT: retq +; ; X86-LABEL: test_fminimum_vector_partially_zero: ; X86: # %bb.0: ; X86-NEXT: vxorpd %xmm1, %xmm1, %xmm1 @@ -1149,6 +1284,13 @@ define <2 x double> @test_fminimum_vector_different_zeros(<2 x double> %x) { ; AVX-NEXT: vblendvpd %xmm2, %xmm0, %xmm1, %xmm0 ; AVX-NEXT: retq ; +; AVX10_2-LABEL: test_fminimum_vector_different_zeros: +; AVX10_2: # %bb.0: +; AVX10_2-NEXT: vxorpd %xmm1, %xmm1, %xmm1 +; AVX10_2-NEXT: vmovhpd {{.*#+}} xmm1 = xmm1[0],mem[0] +; AVX10_2-NEXT: vminmaxpd $0, %xmm1, %xmm0, %xmm0 +; AVX10_2-NEXT: retq +; ; X86-LABEL: test_fminimum_vector_different_zeros: ; X86: # %bb.0: ; X86-NEXT: vxorpd %xmm1, %xmm1, %xmm1 @@ -1177,6 +1319,11 @@ define <4 x float> @test_fmaximum_vector_non_zero(<4 x float> %x) { ; AVX-NEXT: vmaxps %xmm0, %xmm1, %xmm0 ; AVX-NEXT: retq ; +; AVX10_2-LABEL: test_fmaximum_vector_non_zero: +; AVX10_2: # %bb.0: +; AVX10_2-NEXT: vminmaxps $1, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 +; AVX10_2-NEXT: retq +; ; X86-LABEL: test_fmaximum_vector_non_zero: ; X86: # %bb.0: ; X86-NEXT: vmovaps {{.*#+}} xmm1 = [5.0E+0,4.0E+0,3.0E+0,2.0E+0] @@ -1206,6 +1353,13 @@ define <2 x double> @test_fminimum_vector_nan(<2 x double> %x) { ; AVX-NEXT: vunpcklpd {{.*#+}} xmm0 = xmm0[0],xmm1[0] ; AVX-NEXT: retq ; +; AVX10_2-LABEL: test_fminimum_vector_nan: +; AVX10_2: # %bb.0: +; AVX10_2-NEXT: vxorpd %xmm1, %xmm1, %xmm1 +; AVX10_2-NEXT: vmovhpd {{.*#+}} xmm1 = xmm1[0],mem[0] +; AVX10_2-NEXT: vminmaxpd $0, %xmm1, %xmm0, %xmm0 +; AVX10_2-NEXT: retq +; ; X86-LABEL: test_fminimum_vector_nan: ; X86: # %bb.0: ; X86-NEXT: vxorpd %xmm1, %xmm1, %xmm1 @@ -1232,6 +1386,12 @@ define <2 x double> @test_fminimum_vector_zero_first(<2 x double> %x) { ; AVX-NEXT: vminpd %xmm0, %xmm1, %xmm0 ; AVX-NEXT: retq ; +; AVX10_2-LABEL: test_fminimum_vector_zero_first: +; AVX10_2: # %bb.0: +; AVX10_2-NEXT: vxorpd %xmm1, %xmm1, %xmm1 +; AVX10_2-NEXT: vminmaxpd $0, %xmm1, %xmm0, %xmm0 +; AVX10_2-NEXT: retq +; ; X86-LABEL: test_fminimum_vector_zero_first: ; X86: # %bb.0: ; X86-NEXT: vxorpd %xmm1, %xmm1, %xmm1 @@ -1260,6 +1420,11 @@ define <2 x double> @test_fminimum_vector_signed_zero(<2 x double> %x) { ; AVX-NEXT: vblendvpd %xmm1, %xmm0, %xmm2, %xmm0 ; AVX-NEXT: retq ; +; AVX10_2-LABEL: test_fminimum_vector_signed_zero: +; AVX10_2: # %bb.0: +; AVX10_2-NEXT: vminmaxpd $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to2}, %xmm0, %xmm0 +; AVX10_2-NEXT: retq +; ; X86-LABEL: test_fminimum_vector_signed_zero: ; X86: # %bb.0: ; X86-NEXT: vcmpunordpd %xmm0, %xmm0, %xmm1 @@ -1284,6 +1449,11 @@ define <4 x float> @test_fmaximum_vector_signed_zero_first(<4 x float> %x) { ; AVX-NEXT: vmaxps %xmm0, %xmm1, %xmm0 ; AVX-NEXT: retq ; +; AVX10_2-LABEL: test_fmaximum_vector_signed_zero_first: +; AVX10_2: # %bb.0: +; AVX10_2-NEXT: vminmaxps $1, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm0, %xmm0 +; AVX10_2-NEXT: retq +; ; X86-LABEL: test_fmaximum_vector_signed_zero_first: ; X86: # %bb.0: ; X86-NEXT: vbroadcastss {{.*#+}} xmm1 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0] @@ -1314,6 +1484,12 @@ define <4 x float> @test_fmaximum_vector_zero(<4 x float> %x) { ; AVX-NEXT: vblendvps %xmm2, %xmm0, %xmm1, %xmm0 ; AVX-NEXT: retq ; +; AVX10_2-LABEL: test_fmaximum_vector_zero: +; AVX10_2: # %bb.0: +; AVX10_2-NEXT: vxorpd %xmm1, %xmm1, %xmm1 +; AVX10_2-NEXT: vminmaxps $1, %xmm1, %xmm0, %xmm0 +; AVX10_2-NEXT: retq +; ; X86-LABEL: test_fmaximum_vector_zero: ; X86: # %bb.0: ; X86-NEXT: vxorps %xmm1, %xmm1, %xmm1 @@ -1369,6 +1545,12 @@ define <4 x float> @test_fmaximum_v4f32_splat(<4 x float> %x, float %y) { ; AVX512-NEXT: vblendvps %xmm2, %xmm0, %xmm1, %xmm0 ; AVX512-NEXT: retq ; +; AVX10_2-LABEL: test_fmaximum_v4f32_splat: +; AVX10_2: # %bb.0: +; AVX10_2-NEXT: vbroadcastss %xmm1, %xmm1 +; AVX10_2-NEXT: vminmaxps $1, %xmm1, %xmm0, %xmm0 +; AVX10_2-NEXT: retq +; ; X86-LABEL: test_fmaximum_v4f32_splat: ; X86: # %bb.0: ; X86-NEXT: vbroadcastss {{[0-9]+}}(%esp), %xmm1 @@ -1384,11 +1566,10 @@ define <4 x float> @test_fmaximum_v4f32_splat(<4 x float> %x, float %y) { ret <4 x float> %r } -define <4 x half> @test_fmaximum_v4f16(<4 x half> %x, <4 x half> %y) { +define <4 x half> @test_fmaximum_v4f16(<4 x half> %x, <4 x half> %y) nounwind { ; SSE2-LABEL: test_fmaximum_v4f16: ; SSE2: # %bb.0: ; SSE2-NEXT: subq $104, %rsp -; SSE2-NEXT: .cfi_def_cfa_offset 112 ; SSE2-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; SSE2-NEXT: psrld $16, %xmm0 ; SSE2-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill @@ -1524,13 +1705,11 @@ define <4 x half> @test_fmaximum_v4f16(<4 x half> %x, <4 x half> %y) { ; SSE2-NEXT: # xmm0 = xmm0[0],mem[0],xmm0[1],mem[1],xmm0[2],mem[2],xmm0[3],mem[3] ; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] ; SSE2-NEXT: addq $104, %rsp -; SSE2-NEXT: .cfi_def_cfa_offset 8 ; SSE2-NEXT: retq ; ; AVX1-LABEL: test_fmaximum_v4f16: ; AVX1: # %bb.0: ; AVX1-NEXT: subq $120, %rsp -; AVX1-NEXT: .cfi_def_cfa_offset 128 ; AVX1-NEXT: vmovaps %xmm0, %xmm2 ; AVX1-NEXT: vmovshdup {{.*#+}} xmm0 = xmm0[1,1,3,3] ; AVX1-NEXT: vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill @@ -1636,37 +1815,184 @@ define <4 x half> @test_fmaximum_v4f16(<4 x half> %x, <4 x half> %y) { ; AVX1-NEXT: # xmm1 = xmm1[0],mem[0],xmm1[1],mem[1],xmm1[2],mem[2],xmm1[3],mem[3] ; AVX1-NEXT: vinsertps {{.*#+}} xmm0 = xmm1[0],xmm0[0],zero,zero ; AVX1-NEXT: addq $120, %rsp -; AVX1-NEXT: .cfi_def_cfa_offset 8 ; AVX1-NEXT: retq ; ; AVX512-LABEL: test_fmaximum_v4f16: ; AVX512: # %bb.0: -; AVX512-NEXT: vcvtph2ps %xmm0, %ymm2 -; AVX512-NEXT: vcvtph2ps %xmm1, %ymm3 -; AVX512-NEXT: vcmpltps %ymm2, %ymm3, %ymm4 -; AVX512-NEXT: vpmovdw %zmm4, %ymm4 -; AVX512-NEXT: vpblendvb %xmm4, %xmm0, %xmm1, %xmm4 -; AVX512-NEXT: vcmpunordps %ymm3, %ymm2, %ymm2 -; AVX512-NEXT: vpmovdw %zmm2, %ymm2 -; AVX512-NEXT: vpbroadcastw {{.*#+}} xmm3 = [NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN] -; AVX512-NEXT: vpblendvb %xmm2, %xmm3, %xmm4, %xmm2 -; AVX512-NEXT: vpxor %xmm3, %xmm3, %xmm3 -; AVX512-NEXT: vpcmpeqw %xmm3, %xmm0, %xmm4 -; AVX512-NEXT: vpblendvb %xmm4, %xmm0, %xmm2, %xmm0 -; AVX512-NEXT: vpcmpeqw %xmm3, %xmm1, %xmm3 -; AVX512-NEXT: vpblendvb %xmm3, %xmm1, %xmm0, %xmm0 -; AVX512-NEXT: vcvtph2ps %xmm2, %ymm1 -; AVX512-NEXT: vpxor %xmm3, %xmm3, %xmm3 -; AVX512-NEXT: vcmpeqps %ymm3, %ymm1, %ymm1 -; AVX512-NEXT: vpmovdw %zmm1, %ymm1 -; AVX512-NEXT: vpblendvb %xmm1, %xmm0, %xmm2, %xmm0 -; AVX512-NEXT: vzeroupper +; AVX512-NEXT: pushq %rbp +; AVX512-NEXT: pushq %r15 +; AVX512-NEXT: pushq %r14 +; AVX512-NEXT: pushq %r13 +; AVX512-NEXT: pushq %r12 +; AVX512-NEXT: pushq %rbx +; AVX512-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[3,3,3,3] +; AVX512-NEXT: vcvtph2ps %xmm2, %xmm2 +; AVX512-NEXT: vpshufd {{.*#+}} xmm3 = xmm0[3,3,3,3] +; AVX512-NEXT: vcvtph2ps %xmm3, %xmm3 +; AVX512-NEXT: xorl %eax, %eax +; AVX512-NEXT: vucomiss %xmm2, %xmm3 +; AVX512-NEXT: movl $65535, %ecx # imm = 0xFFFF +; AVX512-NEXT: movl $0, %edx +; AVX512-NEXT: cmovpl %ecx, %edx +; AVX512-NEXT: movl $0, %edi +; AVX512-NEXT: cmoval %ecx, %edi +; AVX512-NEXT: vpsrldq {{.*#+}} xmm2 = xmm1[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero +; AVX512-NEXT: vcvtph2ps %xmm2, %xmm2 +; AVX512-NEXT: vpsrldq {{.*#+}} xmm3 = xmm0[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero +; AVX512-NEXT: vcvtph2ps %xmm3, %xmm3 +; AVX512-NEXT: vucomiss %xmm2, %xmm3 +; AVX512-NEXT: movl $0, %esi +; AVX512-NEXT: cmovpl %ecx, %esi +; AVX512-NEXT: movl $0, %r9d +; AVX512-NEXT: cmoval %ecx, %r9d +; AVX512-NEXT: vshufpd {{.*#+}} xmm2 = xmm1[1,0] +; AVX512-NEXT: vcvtph2ps %xmm2, %xmm2 +; AVX512-NEXT: vshufpd {{.*#+}} xmm3 = xmm0[1,0] +; AVX512-NEXT: vcvtph2ps %xmm3, %xmm3 +; AVX512-NEXT: vucomiss %xmm2, %xmm3 +; AVX512-NEXT: movl $0, %r8d +; AVX512-NEXT: cmovpl %ecx, %r8d +; AVX512-NEXT: movl $0, %r11d +; AVX512-NEXT: cmoval %ecx, %r11d +; AVX512-NEXT: vpshuflw {{.*#+}} xmm2 = xmm1[3,3,3,3,4,5,6,7] +; AVX512-NEXT: vcvtph2ps %xmm2, %xmm2 +; AVX512-NEXT: vpshuflw {{.*#+}} xmm3 = xmm0[3,3,3,3,4,5,6,7] +; AVX512-NEXT: vcvtph2ps %xmm3, %xmm3 +; AVX512-NEXT: vucomiss %xmm2, %xmm3 +; AVX512-NEXT: movl $0, %r10d +; AVX512-NEXT: cmovpl %ecx, %r10d +; AVX512-NEXT: movl $0, %ebp +; AVX512-NEXT: cmoval %ecx, %ebp +; AVX512-NEXT: vmovshdup {{.*#+}} xmm2 = xmm1[1,1,3,3] +; AVX512-NEXT: vcvtph2ps %xmm2, %xmm2 +; AVX512-NEXT: vmovshdup {{.*#+}} xmm3 = xmm0[1,1,3,3] +; AVX512-NEXT: vcvtph2ps %xmm3, %xmm3 +; AVX512-NEXT: vucomiss %xmm2, %xmm3 +; AVX512-NEXT: movl $0, %ebx +; AVX512-NEXT: cmovpl %ecx, %ebx +; AVX512-NEXT: movl $0, %r14d +; AVX512-NEXT: cmoval %ecx, %r14d +; AVX512-NEXT: vpshuflw {{.*#+}} xmm2 = xmm1[1,1,1,1,4,5,6,7] +; AVX512-NEXT: vcvtph2ps %xmm2, %xmm2 +; AVX512-NEXT: vpshuflw {{.*#+}} xmm3 = xmm0[1,1,1,1,4,5,6,7] +; AVX512-NEXT: vcvtph2ps %xmm3, %xmm3 +; AVX512-NEXT: vucomiss %xmm2, %xmm3 +; AVX512-NEXT: movl $0, %r15d +; AVX512-NEXT: cmovpl %ecx, %r15d +; AVX512-NEXT: movl $0, %r12d +; AVX512-NEXT: cmoval %ecx, %r12d +; AVX512-NEXT: vcvtph2ps %xmm1, %xmm2 +; AVX512-NEXT: vcvtph2ps %xmm0, %xmm3 +; AVX512-NEXT: vucomiss %xmm2, %xmm3 +; AVX512-NEXT: movl $0, %r13d +; AVX512-NEXT: cmoval %ecx, %r13d +; AVX512-NEXT: vmovd %r13d, %xmm2 +; AVX512-NEXT: vpinsrw $1, %r12d, %xmm2, %xmm2 +; AVX512-NEXT: vpinsrw $2, %r14d, %xmm2, %xmm2 +; AVX512-NEXT: vpinsrw $3, %ebp, %xmm2, %xmm2 +; AVX512-NEXT: vpinsrw $4, %r11d, %xmm2, %xmm2 +; AVX512-NEXT: vpinsrw $5, %r9d, %xmm2, %xmm2 +; AVX512-NEXT: vpinsrw $6, %edi, %xmm2, %xmm2 +; AVX512-NEXT: movl $0, %edi +; AVX512-NEXT: cmovpl %ecx, %edi +; AVX512-NEXT: vpsrldq {{.*#+}} xmm3 = xmm1[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero +; AVX512-NEXT: vcvtph2ps %xmm3, %xmm3 +; AVX512-NEXT: vpsrldq {{.*#+}} xmm4 = xmm0[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero +; AVX512-NEXT: vcvtph2ps %xmm4, %xmm4 +; AVX512-NEXT: vucomiss %xmm3, %xmm4 +; AVX512-NEXT: movl $0, %r9d +; AVX512-NEXT: cmoval %ecx, %r9d +; AVX512-NEXT: vpinsrw $7, %r9d, %xmm2, %xmm2 +; AVX512-NEXT: vpblendvb %xmm2, %xmm0, %xmm1, %xmm2 +; AVX512-NEXT: vmovd %edi, %xmm3 +; AVX512-NEXT: vpinsrw $1, %r15d, %xmm3, %xmm3 +; AVX512-NEXT: vpinsrw $2, %ebx, %xmm3, %xmm3 +; AVX512-NEXT: vpinsrw $3, %r10d, %xmm3, %xmm3 +; AVX512-NEXT: vpinsrw $4, %r8d, %xmm3, %xmm3 +; AVX512-NEXT: vpinsrw $5, %esi, %xmm3, %xmm3 +; AVX512-NEXT: vpinsrw $6, %edx, %xmm3, %xmm3 +; AVX512-NEXT: movl $0, %edx +; AVX512-NEXT: cmovpl %ecx, %edx +; AVX512-NEXT: vpinsrw $7, %edx, %xmm3, %xmm3 +; AVX512-NEXT: vpbroadcastw {{.*#+}} xmm4 = [NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN] +; AVX512-NEXT: vpblendvb %xmm3, %xmm4, %xmm2, %xmm2 +; AVX512-NEXT: vpshuflw {{.*#+}} xmm3 = xmm2[1,1,1,1,4,5,6,7] +; AVX512-NEXT: vcvtph2ps %xmm3, %xmm3 +; AVX512-NEXT: vpxor %xmm4, %xmm4, %xmm4 +; AVX512-NEXT: vucomiss %xmm4, %xmm3 +; AVX512-NEXT: movl $65535, %edx # imm = 0xFFFF +; AVX512-NEXT: cmovnel %eax, %edx +; AVX512-NEXT: cmovpl %eax, %edx +; AVX512-NEXT: vcvtph2ps %xmm2, %xmm3 +; AVX512-NEXT: vucomiss %xmm4, %xmm3 +; AVX512-NEXT: movl $65535, %esi # imm = 0xFFFF +; AVX512-NEXT: cmovnel %eax, %esi +; AVX512-NEXT: cmovpl %eax, %esi +; AVX512-NEXT: vmovd %esi, %xmm3 +; AVX512-NEXT: vpinsrw $1, %edx, %xmm3, %xmm3 +; AVX512-NEXT: vpshufd {{.*#+}} xmm5 = xmm2[1,1,1,1] +; AVX512-NEXT: vcvtph2ps %xmm5, %xmm5 +; AVX512-NEXT: vucomiss %xmm4, %xmm5 +; AVX512-NEXT: movl $65535, %edx # imm = 0xFFFF +; AVX512-NEXT: cmovnel %eax, %edx +; AVX512-NEXT: cmovpl %eax, %edx +; AVX512-NEXT: vpinsrw $2, %edx, %xmm3, %xmm3 +; AVX512-NEXT: vpshuflw {{.*#+}} xmm5 = xmm2[3,3,3,3,4,5,6,7] +; AVX512-NEXT: vcvtph2ps %xmm5, %xmm5 +; AVX512-NEXT: vucomiss %xmm4, %xmm5 +; AVX512-NEXT: movl $65535, %edx # imm = 0xFFFF +; AVX512-NEXT: cmovnel %eax, %edx +; AVX512-NEXT: cmovpl %eax, %edx +; AVX512-NEXT: vpinsrw $3, %edx, %xmm3, %xmm3 +; AVX512-NEXT: vpshufd {{.*#+}} xmm5 = xmm2[2,3,2,3] +; AVX512-NEXT: vcvtph2ps %xmm5, %xmm5 +; AVX512-NEXT: vucomiss %xmm4, %xmm5 +; AVX512-NEXT: movl $65535, %edx # imm = 0xFFFF +; AVX512-NEXT: cmovnel %eax, %edx +; AVX512-NEXT: cmovpl %eax, %edx +; AVX512-NEXT: vpinsrw $4, %edx, %xmm3, %xmm3 +; AVX512-NEXT: vpsrldq {{.*#+}} xmm5 = xmm2[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero +; AVX512-NEXT: vcvtph2ps %xmm5, %xmm5 +; AVX512-NEXT: vucomiss %xmm4, %xmm5 +; AVX512-NEXT: movl $65535, %edx # imm = 0xFFFF +; AVX512-NEXT: cmovnel %eax, %edx +; AVX512-NEXT: cmovpl %eax, %edx +; AVX512-NEXT: vpinsrw $5, %edx, %xmm3, %xmm3 +; AVX512-NEXT: vpshufd {{.*#+}} xmm5 = xmm2[3,3,3,3] +; AVX512-NEXT: vcvtph2ps %xmm5, %xmm5 +; AVX512-NEXT: vucomiss %xmm4, %xmm5 +; AVX512-NEXT: movl $65535, %edx # imm = 0xFFFF +; AVX512-NEXT: cmovnel %eax, %edx +; AVX512-NEXT: cmovpl %eax, %edx +; AVX512-NEXT: vpinsrw $6, %edx, %xmm3, %xmm3 +; AVX512-NEXT: vpsrldq {{.*#+}} xmm5 = xmm2[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero +; AVX512-NEXT: vcvtph2ps %xmm5, %xmm5 +; AVX512-NEXT: vucomiss %xmm4, %xmm5 +; AVX512-NEXT: cmovnel %eax, %ecx +; AVX512-NEXT: cmovpl %eax, %ecx +; AVX512-NEXT: vpinsrw $7, %ecx, %xmm3, %xmm3 +; AVX512-NEXT: vpxor %xmm4, %xmm4, %xmm4 +; AVX512-NEXT: vpcmpeqw %xmm4, %xmm0, %xmm5 +; AVX512-NEXT: vpblendvb %xmm5, %xmm0, %xmm2, %xmm0 +; AVX512-NEXT: vpcmpeqw %xmm4, %xmm1, %xmm4 +; AVX512-NEXT: vpblendvb %xmm4, %xmm1, %xmm0, %xmm0 +; AVX512-NEXT: vpblendvb %xmm3, %xmm0, %xmm2, %xmm0 +; AVX512-NEXT: popq %rbx +; AVX512-NEXT: popq %r12 +; AVX512-NEXT: popq %r13 +; AVX512-NEXT: popq %r14 +; AVX512-NEXT: popq %r15 +; AVX512-NEXT: popq %rbp ; AVX512-NEXT: retq ; +; AVX10_2-LABEL: test_fmaximum_v4f16: +; AVX10_2: # %bb.0: +; AVX10_2-NEXT: vminmaxph $1, %xmm1, %xmm0, %xmm0 +; AVX10_2-NEXT: retq +; ; X86-LABEL: test_fmaximum_v4f16: ; X86: # %bb.0: ; X86-NEXT: subl $164, %esp -; X86-NEXT: .cfi_def_cfa_offset 168 ; X86-NEXT: vmovdqa %xmm0, %xmm2 ; X86-NEXT: vmovdqu %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill ; X86-NEXT: vpsrlq $48, %xmm0, %xmm0 @@ -1806,7 +2132,6 @@ define <4 x half> @test_fmaximum_v4f16(<4 x half> %x, <4 x half> %y) { ; X86-NEXT: # xmm1 = xmm1[0],mem[0],xmm1[1],mem[1],xmm1[2],mem[2],xmm1[3],mem[3] ; X86-NEXT: vinsertps {{.*#+}} xmm0 = xmm1[0],xmm0[0],zero,zero ; X86-NEXT: addl $164, %esp -; X86-NEXT: .cfi_def_cfa_offset 4 ; X86-NEXT: retl %r = call <4 x half> @llvm.maximum.v4f16(<4 x half> %x, <4 x half> %y) ret <4 x half> %r @@ -2192,6 +2517,11 @@ define <4 x bfloat> @test_fmaximum_v4bf16(<4 x bfloat> %x, <4 x bfloat> %y) { ; AVX512-NEXT: .cfi_def_cfa_offset 8 ; AVX512-NEXT: retq ; +; AVX10_2-LABEL: test_fmaximum_v4bf16: +; AVX10_2: # %bb.0: +; AVX10_2-NEXT: vminmaxnepbf16 $1, %xmm1, %xmm0, %xmm0 +; AVX10_2-NEXT: retq +; ; X86-LABEL: test_fmaximum_v4bf16: ; X86: # %bb.0: ; X86-NEXT: pushl %ebp diff --git a/llvm/test/CodeGen/X86/fp16-libcalls.ll b/llvm/test/CodeGen/X86/fp16-libcalls.ll index 5c6c3653a27f..1515cd1366bc 100644 --- a/llvm/test/CodeGen/X86/fp16-libcalls.ll +++ b/llvm/test/CodeGen/X86/fp16-libcalls.ll @@ -73,7 +73,7 @@ define void @test_half_copysign(half %a0, half %a1, ptr %p0) nounwind { ; FP16-LABEL: test_half_copysign: ; FP16: # %bb.0: ; FP16-NEXT: vpbroadcastw {{.*#+}} xmm2 = [NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN] -; FP16-NEXT: vpternlogd $202, %xmm1, %xmm0, %xmm2 +; FP16-NEXT: vpternlogd {{.*#+}} xmm2 = xmm1 ^ (xmm2 & (xmm0 ^ xmm1)) ; FP16-NEXT: vmovsh %xmm2, (%rdi) ; FP16-NEXT: retq ; @@ -379,22 +379,10 @@ define void @test_half_fabs(half %a0, ptr %p0) nounwind { ; ; X86-LABEL: test_half_fabs: ; X86: # %bb.0: -; X86-NEXT: pushl %esi -; X86-NEXT: subl $8, %esp -; X86-NEXT: pinsrw $0, {{[0-9]+}}(%esp), %xmm0 -; X86-NEXT: movl {{[0-9]+}}(%esp), %esi -; X86-NEXT: pextrw $0, %xmm0, %eax -; X86-NEXT: movw %ax, (%esp) -; X86-NEXT: calll __extendhfsf2 -; X86-NEXT: fstps {{[0-9]+}}(%esp) -; X86-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero -; X86-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0 -; X86-NEXT: movd %xmm0, (%esp) -; X86-NEXT: calll __truncsfhf2 -; X86-NEXT: pextrw $0, %xmm0, %eax -; X86-NEXT: movw %ax, (%esi) -; X86-NEXT: addl $8, %esp -; X86-NEXT: popl %esi +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: movzwl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: andl $32767, %ecx # imm = 0x7FFF +; X86-NEXT: movw %cx, (%eax) ; X86-NEXT: retl %res = call half @llvm.fabs.half(half %a0) store half %res, ptr %p0, align 2 @@ -584,22 +572,10 @@ define void @test_half_fneg(half %a0, ptr %p0) nounwind { ; ; X86-LABEL: test_half_fneg: ; X86: # %bb.0: -; X86-NEXT: pushl %esi -; X86-NEXT: subl $8, %esp -; X86-NEXT: pinsrw $0, {{[0-9]+}}(%esp), %xmm0 -; X86-NEXT: movl {{[0-9]+}}(%esp), %esi -; X86-NEXT: pextrw $0, %xmm0, %eax -; X86-NEXT: movw %ax, (%esp) -; X86-NEXT: calll __extendhfsf2 -; X86-NEXT: fstps {{[0-9]+}}(%esp) -; X86-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero -; X86-NEXT: pxor {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0 -; X86-NEXT: movd %xmm0, (%esp) -; X86-NEXT: calll __truncsfhf2 -; X86-NEXT: pextrw $0, %xmm0, %eax -; X86-NEXT: movw %ax, (%esi) -; X86-NEXT: addl $8, %esp -; X86-NEXT: popl %esi +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: movl $32768, %ecx # imm = 0x8000 +; X86-NEXT: xorl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: movw %cx, (%eax) ; X86-NEXT: retl %res = fneg half %a0 store half %res, ptr %p0, align 2 diff --git a/llvm/test/CodeGen/X86/handle-move.ll b/llvm/test/CodeGen/X86/handle-move.ll index 0a43ef3fc22d..c6da9589ff46 100644 --- a/llvm/test/CodeGen/X86/handle-move.ll +++ b/llvm/test/CodeGen/X86/handle-move.ll @@ -1,5 +1,5 @@ -; RUN: llc -mtriple=x86_64-- -mcpu=core2 -fast-isel -enable-misched -misched=shuffle -misched-bottomup -verify-machineinstrs < %s -; RUN: llc -mtriple=x86_64-- -mcpu=core2 -fast-isel -enable-misched -misched=shuffle -misched-topdown -verify-machineinstrs < %s +; RUN: llc -mtriple=x86_64-- -mcpu=core2 -fast-isel -enable-misched -misched=shuffle -misched-prera-direction=bottomup -verify-machineinstrs < %s +; RUN: llc -mtriple=x86_64-- -mcpu=core2 -fast-isel -enable-misched -misched=shuffle -misched-prera-direction=topdown -verify-machineinstrs < %s ; REQUIRES: asserts ; ; Test the LiveIntervals::handleMove() function. diff --git a/llvm/test/CodeGen/X86/illegal-bitfield-loadstore.ll b/llvm/test/CodeGen/X86/illegal-bitfield-loadstore.ll index 7fb07c6b3163..338163d0e1cf 100644 --- a/llvm/test/CodeGen/X86/illegal-bitfield-loadstore.ll +++ b/llvm/test/CodeGen/X86/illegal-bitfield-loadstore.ll @@ -6,22 +6,12 @@ define void @i24_or(ptr %a) { ; X86-LABEL: i24_or: ; X86: # %bb.0: ; X86-NEXT: movl {{[0-9]+}}(%esp), %eax -; X86-NEXT: movzwl (%eax), %ecx -; X86-NEXT: movzbl 2(%eax), %edx -; X86-NEXT: shll $16, %edx -; X86-NEXT: orl %ecx, %edx -; X86-NEXT: orl $384, %edx # imm = 0x180 -; X86-NEXT: movw %dx, (%eax) +; X86-NEXT: orw $384, (%eax) # imm = 0x180 ; X86-NEXT: retl ; ; X64-LABEL: i24_or: ; X64: # %bb.0: -; X64-NEXT: movzwl (%rdi), %eax -; X64-NEXT: movzbl 2(%rdi), %ecx -; X64-NEXT: shll $16, %ecx -; X64-NEXT: orl %eax, %ecx -; X64-NEXT: orl $384, %ecx # imm = 0x180 -; X64-NEXT: movw %cx, (%rdi) +; X64-NEXT: orw $384, (%rdi) # imm = 0x180 ; X64-NEXT: retq %aa = load i24, ptr %a, align 1 %b = or i24 %aa, 384 @@ -103,12 +93,12 @@ define void @i56_or(ptr %a) { ; X86-LABEL: i56_or: ; X86: # %bb.0: ; X86-NEXT: movl {{[0-9]+}}(%esp), %eax -; X86-NEXT: orl $384, (%eax) # imm = 0x180 +; X86-NEXT: orw $384, (%eax) # imm = 0x180 ; X86-NEXT: retl ; ; X64-LABEL: i56_or: ; X64: # %bb.0: -; X64-NEXT: orl $384, (%rdi) # imm = 0x180 +; X64-NEXT: orw $384, (%rdi) # imm = 0x180 ; X64-NEXT: retq %aa = load i56, ptr %a, align 1 %b = or i56 %aa, 384 diff --git a/llvm/test/CodeGen/X86/isel-select-cmov.ll b/llvm/test/CodeGen/X86/isel-select-cmov.ll index 39a20bf6637b..fbd9bd073155 100644 --- a/llvm/test/CodeGen/X86/isel-select-cmov.ll +++ b/llvm/test/CodeGen/X86/isel-select-cmov.ll @@ -741,18 +741,18 @@ define i64 @select_cmp_cmov_i64(i64 %a, i64 %b) nounwind { ; GISEL-X86-NEXT: movl {{[0-9]+}}(%esp), %ebp ; GISEL-X86-NEXT: movl {{[0-9]+}}(%esp), %eax ; GISEL-X86-NEXT: movl {{[0-9]+}}(%esp), %edx +; GISEL-X86-NEXT: cmpl %eax, %esi +; GISEL-X86-NEXT: setb %bl ; GISEL-X86-NEXT: xorl %ecx, %ecx ; GISEL-X86-NEXT: cmpl %edx, %ebp -; GISEL-X86-NEXT: setb %bl -; GISEL-X86-NEXT: sete %cl -; GISEL-X86-NEXT: cmpl %eax, %esi ; GISEL-X86-NEXT: setb %bh +; GISEL-X86-NEXT: sete %cl ; GISEL-X86-NEXT: testl %ecx, %ecx ; GISEL-X86-NEXT: je LBB6_2 ; GISEL-X86-NEXT: ## %bb.1: -; GISEL-X86-NEXT: movb %bh, %bl +; GISEL-X86-NEXT: movb %bl, %bh ; GISEL-X86-NEXT: LBB6_2: -; GISEL-X86-NEXT: movzbl %bl, %edi +; GISEL-X86-NEXT: movzbl %bh, %edi ; GISEL-X86-NEXT: andl $1, %edi ; GISEL-X86-NEXT: je LBB6_4 ; GISEL-X86-NEXT: ## %bb.3: @@ -779,16 +779,16 @@ define i64 @select_cmp_cmov_i64(i64 %a, i64 %b) nounwind { ; GISEL-X86-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edx ; GISEL-X86-CMOV-NEXT: movl {{[0-9]+}}(%esp), %esi ; GISEL-X86-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edi -; GISEL-X86-CMOV-NEXT: xorl %ebx, %ebx ; GISEL-X86-CMOV-NEXT: xorl %ecx, %ecx +; GISEL-X86-CMOV-NEXT: cmpl %esi, %ebp +; GISEL-X86-CMOV-NEXT: setb %cl +; GISEL-X86-CMOV-NEXT: xorl %ebx, %ebx +; GISEL-X86-CMOV-NEXT: xorl %eax, %eax ; GISEL-X86-CMOV-NEXT: cmpl %edi, %edx ; GISEL-X86-CMOV-NEXT: setb %bl -; GISEL-X86-CMOV-NEXT: sete %cl -; GISEL-X86-CMOV-NEXT: xorl %eax, %eax -; GISEL-X86-CMOV-NEXT: cmpl %esi, %ebp -; GISEL-X86-CMOV-NEXT: setb %al -; GISEL-X86-CMOV-NEXT: testl %ecx, %ecx -; GISEL-X86-CMOV-NEXT: cmovnew %ax, %bx +; GISEL-X86-CMOV-NEXT: sete %al +; GISEL-X86-CMOV-NEXT: testl %eax, %eax +; GISEL-X86-CMOV-NEXT: cmovnew %cx, %bx ; GISEL-X86-CMOV-NEXT: andl $1, %ebx ; GISEL-X86-CMOV-NEXT: cmovel %esi, %ebp ; GISEL-X86-CMOV-NEXT: cmovel %edi, %edx diff --git a/llvm/test/CodeGen/X86/llvm.frexp.ll b/llvm/test/CodeGen/X86/llvm.frexp.ll index cd560ad627de..96de34519556 100644 --- a/llvm/test/CodeGen/X86/llvm.frexp.ll +++ b/llvm/test/CodeGen/X86/llvm.frexp.ll @@ -325,28 +325,27 @@ define { <4 x float>, <4 x i32> } @test_frexp_v4f32_v4i32(<4 x float> %a) { ; ; WIN32-LABEL: test_frexp_v4f32_v4i32: ; WIN32: # %bb.0: -; WIN32-NEXT: pushl %edi ; WIN32-NEXT: pushl %esi -; WIN32-NEXT: subl $60, %esp +; WIN32-NEXT: subl $44, %esp ; WIN32-NEXT: movl {{[0-9]+}}(%esp), %esi -; WIN32-NEXT: leal {{[0-9]+}}(%esp), %eax +; WIN32-NEXT: leal 24(%esi), %eax ; WIN32-NEXT: movl %eax, {{[0-9]+}}(%esp) ; WIN32-NEXT: flds {{[0-9]+}}(%esp) ; WIN32-NEXT: fstpl (%esp) ; WIN32-NEXT: calll _frexp ; WIN32-NEXT: fstpl {{[-0-9]+}}(%e{{[sb]}}p) # 8-byte Folded Spill -; WIN32-NEXT: leal {{[0-9]+}}(%esp), %eax +; WIN32-NEXT: leal 20(%esi), %eax ; WIN32-NEXT: movl %eax, {{[0-9]+}}(%esp) ; WIN32-NEXT: flds {{[0-9]+}}(%esp) ; WIN32-NEXT: fstpl (%esp) ; WIN32-NEXT: calll _frexp ; WIN32-NEXT: fstpl {{[-0-9]+}}(%e{{[sb]}}p) # 8-byte Folded Spill -; WIN32-NEXT: leal {{[0-9]+}}(%esp), %eax +; WIN32-NEXT: leal 16(%esi), %eax ; WIN32-NEXT: movl %eax, {{[0-9]+}}(%esp) ; WIN32-NEXT: flds {{[0-9]+}}(%esp) ; WIN32-NEXT: fstpl (%esp) ; WIN32-NEXT: calll _frexp -; WIN32-NEXT: leal {{[0-9]+}}(%esp), %eax +; WIN32-NEXT: leal 28(%esi), %eax ; WIN32-NEXT: movl %eax, {{[0-9]+}}(%esp) ; WIN32-NEXT: flds {{[0-9]+}}(%esp) ; WIN32-NEXT: fstpl (%esp) @@ -361,22 +360,13 @@ define { <4 x float>, <4 x i32> } @test_frexp_v4f32_v4i32(<4 x float> %a) { ; WIN32-NEXT: flds {{[0-9]+}}(%esp) ; WIN32-NEXT: flds {{[0-9]+}}(%esp) ; WIN32-NEXT: flds {{[0-9]+}}(%esp) -; WIN32-NEXT: movl {{[0-9]+}}(%esp), %eax -; WIN32-NEXT: movl {{[0-9]+}}(%esp), %ecx -; WIN32-NEXT: movl {{[0-9]+}}(%esp), %edx -; WIN32-NEXT: movl {{[0-9]+}}(%esp), %edi -; WIN32-NEXT: movl %edi, 28(%esi) -; WIN32-NEXT: movl %edx, 24(%esi) -; WIN32-NEXT: movl %ecx, 20(%esi) -; WIN32-NEXT: movl %eax, 16(%esi) ; WIN32-NEXT: fstps 12(%esi) ; WIN32-NEXT: fstps 8(%esi) ; WIN32-NEXT: fstps 4(%esi) ; WIN32-NEXT: fstps (%esi) ; WIN32-NEXT: movl %esi, %eax -; WIN32-NEXT: addl $60, %esp +; WIN32-NEXT: addl $44, %esp ; WIN32-NEXT: popl %esi -; WIN32-NEXT: popl %edi ; WIN32-NEXT: retl %result = call { <4 x float>, <4 x i32> } @llvm.frexp.v4f32.v4i32(<4 x float> %a) ret { <4 x float>, <4 x i32> } %result @@ -499,46 +489,35 @@ define <4 x i32> @test_frexp_v4f32_v4i32_only_use_exp(<4 x float> %a) { ; ; WIN32-LABEL: test_frexp_v4f32_v4i32_only_use_exp: ; WIN32: # %bb.0: -; WIN32-NEXT: pushl %edi ; WIN32-NEXT: pushl %esi -; WIN32-NEXT: subl $28, %esp +; WIN32-NEXT: subl $12, %esp ; WIN32-NEXT: movl {{[0-9]+}}(%esp), %esi -; WIN32-NEXT: leal {{[0-9]+}}(%esp), %eax +; WIN32-NEXT: leal 8(%esi), %eax ; WIN32-NEXT: movl %eax, {{[0-9]+}}(%esp) ; WIN32-NEXT: flds {{[0-9]+}}(%esp) ; WIN32-NEXT: fstpl (%esp) ; WIN32-NEXT: calll _frexp ; WIN32-NEXT: fstp %st(0) -; WIN32-NEXT: leal {{[0-9]+}}(%esp), %eax +; WIN32-NEXT: leal 4(%esi), %eax ; WIN32-NEXT: movl %eax, {{[0-9]+}}(%esp) ; WIN32-NEXT: flds {{[0-9]+}}(%esp) ; WIN32-NEXT: fstpl (%esp) ; WIN32-NEXT: calll _frexp ; WIN32-NEXT: fstp %st(0) -; WIN32-NEXT: leal {{[0-9]+}}(%esp), %eax +; WIN32-NEXT: leal 12(%esi), %eax ; WIN32-NEXT: movl %eax, {{[0-9]+}}(%esp) ; WIN32-NEXT: flds {{[0-9]+}}(%esp) ; WIN32-NEXT: fstpl (%esp) ; WIN32-NEXT: calll _frexp ; WIN32-NEXT: fstp %st(0) -; WIN32-NEXT: leal {{[0-9]+}}(%esp), %eax -; WIN32-NEXT: movl %eax, {{[0-9]+}}(%esp) +; WIN32-NEXT: movl %esi, {{[0-9]+}}(%esp) ; WIN32-NEXT: flds {{[0-9]+}}(%esp) ; WIN32-NEXT: fstpl (%esp) ; WIN32-NEXT: calll _frexp ; WIN32-NEXT: fstp %st(0) -; WIN32-NEXT: movl {{[0-9]+}}(%esp), %eax -; WIN32-NEXT: movl {{[0-9]+}}(%esp), %ecx -; WIN32-NEXT: movl {{[0-9]+}}(%esp), %edx -; WIN32-NEXT: movl {{[0-9]+}}(%esp), %edi -; WIN32-NEXT: movl %edi, 12(%esi) -; WIN32-NEXT: movl %edx, 8(%esi) -; WIN32-NEXT: movl %ecx, 4(%esi) -; WIN32-NEXT: movl %eax, (%esi) ; WIN32-NEXT: movl %esi, %eax -; WIN32-NEXT: addl $28, %esp +; WIN32-NEXT: addl $12, %esp ; WIN32-NEXT: popl %esi -; WIN32-NEXT: popl %edi ; WIN32-NEXT: retl %result = call { <4 x float>, <4 x i32> } @llvm.frexp.v4f32.v4i32(<4 x float> %a) %result.1 = extractvalue { <4 x float>, <4 x i32> } %result, 1 diff --git a/llvm/test/CodeGen/X86/llvm.sincos.ll b/llvm/test/CodeGen/X86/llvm.sincos.ll new file mode 100644 index 000000000000..a429314630e5 --- /dev/null +++ b/llvm/test/CodeGen/X86/llvm.sincos.ll @@ -0,0 +1,162 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --no_x86_scrub_sp --version 5 +; RUN: llc < %s -mtriple=i386-unknown-linux-gnu | FileCheck %s + +define void @test_sincos_v4f32(<4 x float> %x, ptr noalias %out_sin, ptr noalias %out_cos) { +; CHECK-LABEL: test_sincos_v4f32: +; CHECK: # %bb.0: +; CHECK-NEXT: pushl %edi +; CHECK-NEXT: .cfi_def_cfa_offset 8 +; CHECK-NEXT: pushl %esi +; CHECK-NEXT: .cfi_def_cfa_offset 12 +; CHECK-NEXT: subl $52, %esp +; CHECK-NEXT: .cfi_def_cfa_offset 64 +; CHECK-NEXT: .cfi_offset %esi, -12 +; CHECK-NEXT: .cfi_offset %edi, -8 +; CHECK-NEXT: movl 84(%esp), %esi +; CHECK-NEXT: flds 76(%esp) +; CHECK-NEXT: fstps {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill +; CHECK-NEXT: flds 64(%esp) +; CHECK-NEXT: fstps {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill +; CHECK-NEXT: flds 72(%esp) +; CHECK-NEXT: fstps {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill +; CHECK-NEXT: flds 68(%esp) +; CHECK-NEXT: movl 80(%esp), %edi +; CHECK-NEXT: leal 40(%esp), %eax +; CHECK-NEXT: movl %eax, 8(%esp) +; CHECK-NEXT: leal 4(%edi), %eax +; CHECK-NEXT: movl %eax, 4(%esp) +; CHECK-NEXT: fstps (%esp) +; CHECK-NEXT: calll sincosf +; CHECK-NEXT: leal 44(%esp), %eax +; CHECK-NEXT: movl %eax, 8(%esp) +; CHECK-NEXT: leal 8(%edi), %eax +; CHECK-NEXT: movl %eax, 4(%esp) +; CHECK-NEXT: flds {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload +; CHECK-NEXT: fstps (%esp) +; CHECK-NEXT: calll sincosf +; CHECK-NEXT: leal 36(%esp), %eax +; CHECK-NEXT: movl %eax, 8(%esp) +; CHECK-NEXT: movl %edi, 4(%esp) +; CHECK-NEXT: flds {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload +; CHECK-NEXT: fstps (%esp) +; CHECK-NEXT: calll sincosf +; CHECK-NEXT: leal 48(%esp), %eax +; CHECK-NEXT: movl %eax, 8(%esp) +; CHECK-NEXT: addl $12, %edi +; CHECK-NEXT: movl %edi, 4(%esp) +; CHECK-NEXT: flds {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload +; CHECK-NEXT: fstps (%esp) +; CHECK-NEXT: calll sincosf +; CHECK-NEXT: flds 36(%esp) +; CHECK-NEXT: flds 40(%esp) +; CHECK-NEXT: flds 44(%esp) +; CHECK-NEXT: flds 48(%esp) +; CHECK-NEXT: fstps 12(%esi) +; CHECK-NEXT: fstps 8(%esi) +; CHECK-NEXT: fstps 4(%esi) +; CHECK-NEXT: fstps (%esi) +; CHECK-NEXT: addl $52, %esp +; CHECK-NEXT: .cfi_def_cfa_offset 12 +; CHECK-NEXT: popl %esi +; CHECK-NEXT: .cfi_def_cfa_offset 8 +; CHECK-NEXT: popl %edi +; CHECK-NEXT: .cfi_def_cfa_offset 4 +; CHECK-NEXT: retl + %result = call { <4 x float>, <4 x float> } @llvm.sincos.v4f32(<4 x float> %x) + %result.0 = extractvalue { <4 x float>, <4 x float> } %result, 0 + %result.1 = extractvalue { <4 x float>, <4 x float> } %result, 1 + store <4 x float> %result.0, ptr %out_sin, align 4 + store <4 x float> %result.1, ptr %out_cos, align 4 + ret void +} + +define void @test_sincos_v2f64(<2 x double> %x, ptr noalias %out_sin, ptr noalias %out_cos) { +; CHECK-LABEL: test_sincos_v2f64: +; CHECK: # %bb.0: +; CHECK-NEXT: pushl %edi +; CHECK-NEXT: .cfi_def_cfa_offset 8 +; CHECK-NEXT: pushl %esi +; CHECK-NEXT: .cfi_def_cfa_offset 12 +; CHECK-NEXT: subl $52, %esp +; CHECK-NEXT: .cfi_def_cfa_offset 64 +; CHECK-NEXT: .cfi_offset %esi, -12 +; CHECK-NEXT: .cfi_offset %edi, -8 +; CHECK-NEXT: movl 84(%esp), %esi +; CHECK-NEXT: fldl 72(%esp) +; CHECK-NEXT: fstpl {{[-0-9]+}}(%e{{[sb]}}p) # 8-byte Folded Spill +; CHECK-NEXT: fldl 64(%esp) +; CHECK-NEXT: movl 80(%esp), %edi +; CHECK-NEXT: leal 24(%esp), %eax +; CHECK-NEXT: movl %eax, 12(%esp) +; CHECK-NEXT: movl %edi, 8(%esp) +; CHECK-NEXT: fstpl (%esp) +; CHECK-NEXT: calll sincos +; CHECK-NEXT: leal 32(%esp), %eax +; CHECK-NEXT: movl %eax, 12(%esp) +; CHECK-NEXT: addl $8, %edi +; CHECK-NEXT: movl %edi, 8(%esp) +; CHECK-NEXT: fldl {{[-0-9]+}}(%e{{[sb]}}p) # 8-byte Folded Reload +; CHECK-NEXT: fstpl (%esp) +; CHECK-NEXT: calll sincos +; CHECK-NEXT: fldl 24(%esp) +; CHECK-NEXT: fldl 32(%esp) +; CHECK-NEXT: fstpl 8(%esi) +; CHECK-NEXT: fstpl (%esi) +; CHECK-NEXT: addl $52, %esp +; CHECK-NEXT: .cfi_def_cfa_offset 12 +; CHECK-NEXT: popl %esi +; CHECK-NEXT: .cfi_def_cfa_offset 8 +; CHECK-NEXT: popl %edi +; CHECK-NEXT: .cfi_def_cfa_offset 4 +; CHECK-NEXT: retl + %result = call { <2 x double>, <2 x double> } @llvm.sincos.v2f64(<2 x double> %x) + %result.0 = extractvalue { <2 x double>, <2 x double> } %result, 0 + %result.1 = extractvalue { <2 x double>, <2 x double> } %result, 1 + store <2 x double> %result.0, ptr %out_sin, align 8 + store <2 x double> %result.1, ptr %out_cos, align 8 + ret void +} + +declare void @foo(ptr, ptr) + +define void @can_fold_with_call_in_chain(float %x, ptr noalias %a, ptr noalias %b) { +; CHECK-LABEL: can_fold_with_call_in_chain: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: pushl %edi +; CHECK-NEXT: .cfi_def_cfa_offset 8 +; CHECK-NEXT: pushl %esi +; CHECK-NEXT: .cfi_def_cfa_offset 12 +; CHECK-NEXT: subl $20, %esp +; CHECK-NEXT: .cfi_def_cfa_offset 32 +; CHECK-NEXT: .cfi_offset %esi, -12 +; CHECK-NEXT: .cfi_offset %edi, -8 +; CHECK-NEXT: flds 32(%esp) +; CHECK-NEXT: fstps {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill +; CHECK-NEXT: movl 36(%esp), %edi +; CHECK-NEXT: movl 40(%esp), %esi +; CHECK-NEXT: movl %esi, 4(%esp) +; CHECK-NEXT: movl %edi, (%esp) +; CHECK-NEXT: calll foo@PLT +; CHECK-NEXT: leal 16(%esp), %eax +; CHECK-NEXT: movl %eax, 8(%esp) +; CHECK-NEXT: movl %edi, 4(%esp) +; CHECK-NEXT: flds {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload +; CHECK-NEXT: fstps (%esp) +; CHECK-NEXT: calll sincosf +; CHECK-NEXT: flds 16(%esp) +; CHECK-NEXT: fstps (%esi) +; CHECK-NEXT: addl $20, %esp +; CHECK-NEXT: .cfi_def_cfa_offset 12 +; CHECK-NEXT: popl %esi +; CHECK-NEXT: .cfi_def_cfa_offset 8 +; CHECK-NEXT: popl %edi +; CHECK-NEXT: .cfi_def_cfa_offset 4 +; CHECK-NEXT: retl +entry: + %sin = tail call float @llvm.sin.f32(float %x) + %cos = tail call float @llvm.cos.f32(float %x) + call void @foo(ptr %a, ptr %b) + store float %sin, ptr %a, align 4 + store float %cos, ptr %b, align 4 + ret void +} diff --git a/llvm/test/CodeGen/X86/lower-vec-shift.ll b/llvm/test/CodeGen/X86/lower-vec-shift.ll index 67e0c1b3cf2b..9d4935ef564d 100644 --- a/llvm/test/CodeGen/X86/lower-vec-shift.ll +++ b/llvm/test/CodeGen/X86/lower-vec-shift.ll @@ -265,11 +265,11 @@ define <16 x i16> @test11(<16 x i16> %a) { ; AVX1-LABEL: test11: ; AVX1: # %bb.0: ; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1 -; AVX1-NEXT: vpsllw $1, %xmm1, %xmm2 -; AVX1-NEXT: vpsllw $3, %xmm1, %xmm1 -; AVX1-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1,2],xmm2[3,4,5],xmm1[6],xmm2[7] +; AVX1-NEXT: vpsllw $3, %xmm1, %xmm2 +; AVX1-NEXT: vpaddw %xmm1, %xmm1, %xmm1 +; AVX1-NEXT: vpblendw {{.*#+}} xmm1 = xmm2[0,1,2],xmm1[3,4,5],xmm2[6],xmm1[7] ; AVX1-NEXT: vpsllw $3, %xmm0, %xmm2 -; AVX1-NEXT: vpsllw $1, %xmm0, %xmm0 +; AVX1-NEXT: vpaddw %xmm0, %xmm0, %xmm0 ; AVX1-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2,3,4],xmm2[5,6,7] ; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 ; AVX1-NEXT: retq @@ -294,10 +294,10 @@ define <16 x i16> @test12(<16 x i16> %a) { ; AVX1: # %bb.0: ; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1 ; AVX1-NEXT: vpsllw $3, %xmm1, %xmm2 -; AVX1-NEXT: vpsllw $1, %xmm1, %xmm1 +; AVX1-NEXT: vpaddw %xmm1, %xmm1, %xmm1 ; AVX1-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2,3,4],xmm2[5,6,7] ; AVX1-NEXT: vpsllw $3, %xmm0, %xmm2 -; AVX1-NEXT: vpsllw $1, %xmm0, %xmm0 +; AVX1-NEXT: vpaddw %xmm0, %xmm0, %xmm0 ; AVX1-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2,3,4],xmm2[5,6,7] ; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 ; AVX1-NEXT: retq @@ -305,7 +305,7 @@ define <16 x i16> @test12(<16 x i16> %a) { ; AVX2-LABEL: test12: ; AVX2: # %bb.0: ; AVX2-NEXT: vpsllw $3, %ymm0, %ymm1 -; AVX2-NEXT: vpsllw $1, %ymm0, %ymm0 +; AVX2-NEXT: vpaddw %ymm0, %ymm0, %ymm0 ; AVX2-NEXT: vpblendw {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2,3,4],ymm1[5,6,7],ymm0[8],ymm1[9],ymm0[10,11,12],ymm1[13,14,15] ; AVX2-NEXT: retq %lshr = shl <16 x i16> %a, <i16 1, i16 3, i16 1, i16 1, i16 1, i16 3, i16 3, i16 3, i16 1, i16 3, i16 1, i16 1, i16 1, i16 3, i16 3, i16 3> diff --git a/llvm/test/CodeGen/X86/mingw-refptr.ll b/llvm/test/CodeGen/X86/mingw-refptr.ll index 73f1a9880913..82a90aba3865 100644 --- a/llvm/test/CodeGen/X86/mingw-refptr.ll +++ b/llvm/test/CodeGen/X86/mingw-refptr.ll @@ -1,4 +1,5 @@ ; RUN: llc < %s -mtriple=x86_64-w64-mingw32 | FileCheck %s -check-prefix=CHECK-X64 +; RUN: llc < %s -mtriple=x86_64-pc-cygwin | FileCheck %s -check-prefix=CHECK-X64 ; RUN: llc < %s -mtriple=i686-w64-mingw32 | FileCheck %s -check-prefix=CHECK-X86 ; RUN: llc < %s -mtriple=i686-w64-mingw32-none-elf | FileCheck %s -check-prefix=CHECK-X86-ELF diff --git a/llvm/test/CodeGen/X86/misched-aa-colored.ll b/llvm/test/CodeGen/X86/misched-aa-colored.ll index 73626de163d0..3504e555cd9c 100644 --- a/llvm/test/CodeGen/X86/misched-aa-colored.ll +++ b/llvm/test/CodeGen/X86/misched-aa-colored.ll @@ -1,4 +1,4 @@ -; RUN: llc < %s -mcpu=x86-64 -enable-misched -misched-bottomup=0 -misched-topdown=0 -misched=shuffle -enable-aa-sched-mi | FileCheck %s +; RUN: llc < %s -mcpu=x86-64 -enable-misched -misched-prera-direction=bidirectional -misched=shuffle -enable-aa-sched-mi | FileCheck %s ; REQUIRES: asserts ; -misched=shuffle is NDEBUG only! diff --git a/llvm/test/CodeGen/X86/misched-matrix.ll b/llvm/test/CodeGen/X86/misched-matrix.ll index e909348eaa38..f44bf39e76f6 100644 --- a/llvm/test/CodeGen/X86/misched-matrix.ll +++ b/llvm/test/CodeGen/X86/misched-matrix.ll @@ -1,5 +1,5 @@ ; RUN: llc < %s -mtriple=x86_64-- -mcpu=generic -pre-RA-sched=source -enable-misched \ -; RUN: -misched-topdown -verify-machineinstrs \ +; RUN: -misched-prera-direction=topdown -verify-machineinstrs \ ; RUN: | FileCheck %s -check-prefix=TOPDOWN ; RUN: llc < %s -mtriple=x86_64-- -mcpu=generic -pre-RA-sched=source -enable-misched \ ; RUN: -misched=ilpmin -verify-machineinstrs \ diff --git a/llvm/test/CodeGen/X86/misched-new.ll b/llvm/test/CodeGen/X86/misched-new.ll index 06ae8ff43d5a..d7b3604ceefc 100644 --- a/llvm/test/CodeGen/X86/misched-new.ll +++ b/llvm/test/CodeGen/X86/misched-new.ll @@ -1,8 +1,8 @@ ; RUN: llc < %s -mtriple=x86_64-- -mcpu=core2 -x86-early-ifcvt -enable-misched \ -; RUN: -misched=shuffle -misched-bottomup -verify-machineinstrs \ +; RUN: -misched=shuffle -misched-prera-direction=bottomup -verify-machineinstrs \ ; RUN: | FileCheck %s ; RUN: llc < %s -mtriple=x86_64-- -mcpu=core2 -x86-early-ifcvt -enable-misched \ -; RUN: -misched=shuffle -misched-topdown -verify-machineinstrs \ +; RUN: -misched=shuffle -misched-prera-direction=topdown -verify-machineinstrs \ ; RUN: | FileCheck %s --check-prefix TOPDOWN ; REQUIRES: asserts ; diff --git a/llvm/test/CodeGen/X86/oddshuffles.ll b/llvm/test/CodeGen/X86/oddshuffles.ll index 90c1d42a929c..8fd8e0e8120c 100644 --- a/llvm/test/CodeGen/X86/oddshuffles.ll +++ b/llvm/test/CodeGen/X86/oddshuffles.ll @@ -2400,34 +2400,32 @@ define void @D107009(ptr %input, ptr %output) { ; ; AVX1-LABEL: D107009: ; AVX1: # %bb.0: -; AVX1-NEXT: vmovups 96(%rdi), %ymm0 -; AVX1-NEXT: vmovups 128(%rdi), %ymm1 -; AVX1-NEXT: vmovups 224(%rdi), %ymm2 -; AVX1-NEXT: vunpcklpd {{.*#+}} ymm2 = ymm2[0],mem[0],ymm2[2],mem[2] -; AVX1-NEXT: vunpcklps {{.*#+}} ymm1 = ymm1[0],mem[0],ymm1[1],mem[1],ymm1[4],mem[4],ymm1[5],mem[5] -; AVX1-NEXT: vshufps {{.*#+}} ymm1 = ymm1[0,1],ymm2[2,0],ymm1[4,5],ymm2[6,4] -; AVX1-NEXT: vmovdqa 16(%rdi), %xmm2 -; AVX1-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm2[0],mem[0],xmm2[1],mem[1] -; AVX1-NEXT: vunpcklpd {{.*#+}} ymm0 = ymm0[0],mem[0],ymm0[2],mem[2] -; AVX1-NEXT: vshufps {{.*#+}} ymm0 = ymm0[0,1,2,0,4,5,6,4] +; AVX1-NEXT: vmovups 128(%rdi), %ymm0 +; AVX1-NEXT: vmovups 224(%rdi), %ymm1 +; AVX1-NEXT: vunpcklpd {{.*#+}} ymm1 = ymm1[0],mem[0],ymm1[2],mem[2] +; AVX1-NEXT: vunpcklps {{.*#+}} ymm0 = ymm0[0],mem[0],ymm0[1],mem[1],ymm0[4],mem[4],ymm0[5],mem[5] +; AVX1-NEXT: vshufps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,0],ymm0[4,5],ymm1[6,4] +; AVX1-NEXT: vmovaps 112(%rdi), %xmm1 +; AVX1-NEXT: vunpcklpd {{.*#+}} xmm1 = xmm1[0],mem[0] +; AVX1-NEXT: vmovaps 16(%rdi), %xmm2 +; AVX1-NEXT: vunpcklps {{.*#+}} xmm2 = xmm2[0],mem[0],xmm2[1],mem[1] +; AVX1-NEXT: vshufps {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,0] +; AVX1-NEXT: vpsrld $16, %xmm1, %xmm1 ; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm0 -; AVX1-NEXT: vpblendw {{.*#+}} xmm0 = xmm2[0,1,2,3],xmm0[4,5,6,7] ; AVX1-NEXT: vpsrld $16, %xmm0, %xmm0 -; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm1 -; AVX1-NEXT: vpsrld $16, %xmm1, %xmm1 -; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm1 -; AVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[0,1,1,3] -; AVX1-NEXT: vpshufd {{.*#+}} xmm3 = xmm0[2,1,3,3] +; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0 +; AVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[0,1,1,3] +; AVX1-NEXT: vpshufd {{.*#+}} xmm3 = xmm1[2,1,3,3] ; AVX1-NEXT: vinsertf128 $1, %xmm3, %ymm2, %ymm2 -; AVX1-NEXT: vshufps {{.*#+}} ymm3 = ymm1[3,3,3,3,7,7,7,7] -; AVX1-NEXT: vshufpd {{.*#+}} ymm4 = ymm1[0,0,3,2] -; AVX1-NEXT: vmovshdup {{.*#+}} ymm5 = ymm1[1,1,3,3,5,5,7,7] -; AVX1-NEXT: vpshufd {{.*#+}} xmm6 = xmm0[3,3,3,3] -; AVX1-NEXT: vpshufd {{.*#+}} xmm7 = xmm0[1,1,1,1] -; AVX1-NEXT: vmovdqa %xmm0, 16(%rsi) +; AVX1-NEXT: vshufps {{.*#+}} ymm3 = ymm0[3,3,3,3,7,7,7,7] +; AVX1-NEXT: vshufpd {{.*#+}} ymm4 = ymm0[0,0,3,2] +; AVX1-NEXT: vmovshdup {{.*#+}} ymm5 = ymm0[1,1,3,3,5,5,7,7] +; AVX1-NEXT: vpshufd {{.*#+}} xmm6 = xmm1[3,3,3,3] +; AVX1-NEXT: vpshufd {{.*#+}} xmm7 = xmm1[1,1,1,1] +; AVX1-NEXT: vmovdqa %xmm1, 16(%rsi) ; AVX1-NEXT: vmovdqa %xmm7, 48(%rsi) ; AVX1-NEXT: vmovdqa %xmm6, 112(%rsi) -; AVX1-NEXT: vmovups %ymm1, 128(%rsi) +; AVX1-NEXT: vmovups %ymm0, 128(%rsi) ; AVX1-NEXT: vmovups %ymm5, 160(%rsi) ; AVX1-NEXT: vmovupd %ymm4, 192(%rsi) ; AVX1-NEXT: vmovupd %ymm3, 224(%rsi) @@ -2470,34 +2468,32 @@ define void @D107009(ptr %input, ptr %output) { ; ; XOP-LABEL: D107009: ; XOP: # %bb.0: -; XOP-NEXT: vmovups 96(%rdi), %ymm0 -; XOP-NEXT: vmovups 128(%rdi), %ymm1 -; XOP-NEXT: vmovups 224(%rdi), %ymm2 -; XOP-NEXT: vunpcklpd {{.*#+}} ymm2 = ymm2[0],mem[0],ymm2[2],mem[2] -; XOP-NEXT: vunpcklps {{.*#+}} ymm1 = ymm1[0],mem[0],ymm1[1],mem[1],ymm1[4],mem[4],ymm1[5],mem[5] -; XOP-NEXT: vshufps {{.*#+}} ymm1 = ymm1[0,1],ymm2[2,0],ymm1[4,5],ymm2[6,4] -; XOP-NEXT: vmovdqa 16(%rdi), %xmm2 -; XOP-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm2[0],mem[0],xmm2[1],mem[1] -; XOP-NEXT: vunpcklpd {{.*#+}} ymm0 = ymm0[0],mem[0],ymm0[2],mem[2] -; XOP-NEXT: vshufps {{.*#+}} ymm0 = ymm0[0,1,2,0,4,5,6,4] +; XOP-NEXT: vmovups 128(%rdi), %ymm0 +; XOP-NEXT: vmovups 224(%rdi), %ymm1 +; XOP-NEXT: vunpcklpd {{.*#+}} ymm1 = ymm1[0],mem[0],ymm1[2],mem[2] +; XOP-NEXT: vunpcklps {{.*#+}} ymm0 = ymm0[0],mem[0],ymm0[1],mem[1],ymm0[4],mem[4],ymm0[5],mem[5] +; XOP-NEXT: vshufps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,0],ymm0[4,5],ymm1[6,4] +; XOP-NEXT: vmovaps 112(%rdi), %xmm1 +; XOP-NEXT: vunpcklpd {{.*#+}} xmm1 = xmm1[0],mem[0] +; XOP-NEXT: vmovaps 16(%rdi), %xmm2 +; XOP-NEXT: vunpcklps {{.*#+}} xmm2 = xmm2[0],mem[0],xmm2[1],mem[1] +; XOP-NEXT: vshufps {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,0] +; XOP-NEXT: vpsrld $16, %xmm1, %xmm1 ; XOP-NEXT: vextractf128 $1, %ymm0, %xmm0 -; XOP-NEXT: vpblendw {{.*#+}} xmm0 = xmm2[0,1,2,3],xmm0[4,5,6,7] ; XOP-NEXT: vpsrld $16, %xmm0, %xmm0 -; XOP-NEXT: vextractf128 $1, %ymm1, %xmm1 -; XOP-NEXT: vpsrld $16, %xmm1, %xmm1 -; XOP-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm1 -; XOP-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[0,1,1,3] -; XOP-NEXT: vpshufd {{.*#+}} xmm3 = xmm0[2,1,3,3] +; XOP-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0 +; XOP-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[0,1,1,3] +; XOP-NEXT: vpshufd {{.*#+}} xmm3 = xmm1[2,1,3,3] ; XOP-NEXT: vinsertf128 $1, %xmm3, %ymm2, %ymm2 -; XOP-NEXT: vshufps {{.*#+}} ymm3 = ymm1[3,3,3,3,7,7,7,7] -; XOP-NEXT: vshufpd {{.*#+}} ymm4 = ymm1[0,0,3,2] -; XOP-NEXT: vmovshdup {{.*#+}} ymm5 = ymm1[1,1,3,3,5,5,7,7] -; XOP-NEXT: vpshufd {{.*#+}} xmm6 = xmm0[3,3,3,3] -; XOP-NEXT: vpshufd {{.*#+}} xmm7 = xmm0[1,1,1,1] -; XOP-NEXT: vmovdqa %xmm0, 16(%rsi) +; XOP-NEXT: vshufps {{.*#+}} ymm3 = ymm0[3,3,3,3,7,7,7,7] +; XOP-NEXT: vshufpd {{.*#+}} ymm4 = ymm0[0,0,3,2] +; XOP-NEXT: vmovshdup {{.*#+}} ymm5 = ymm0[1,1,3,3,5,5,7,7] +; XOP-NEXT: vpshufd {{.*#+}} xmm6 = xmm1[3,3,3,3] +; XOP-NEXT: vpshufd {{.*#+}} xmm7 = xmm1[1,1,1,1] +; XOP-NEXT: vmovdqa %xmm1, 16(%rsi) ; XOP-NEXT: vmovdqa %xmm7, 48(%rsi) ; XOP-NEXT: vmovdqa %xmm6, 112(%rsi) -; XOP-NEXT: vmovups %ymm1, 128(%rsi) +; XOP-NEXT: vmovups %ymm0, 128(%rsi) ; XOP-NEXT: vmovups %ymm5, 160(%rsi) ; XOP-NEXT: vmovupd %ymm4, 192(%rsi) ; XOP-NEXT: vmovupd %ymm3, 224(%rsi) diff --git a/llvm/test/CodeGen/X86/pr116153.ll b/llvm/test/CodeGen/X86/pr116153.ll new file mode 100644 index 000000000000..5c9c2c76131d --- /dev/null +++ b/llvm/test/CodeGen/X86/pr116153.ll @@ -0,0 +1,50 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5 +; RUN: llc < %s -mtriple=x86_64-- | FileCheck %s + +define void @_test_func(<16 x half> %0) #0 { +; CHECK-LABEL: _test_func: +; CHECK: # %bb.0: +; CHECK-NEXT: vpshuflw {{.*#+}} xmm1 = xmm0[3,3,3,3,4,5,6,7] +; CHECK-NEXT: vcvtph2ps %xmm1, %xmm1 +; CHECK-NEXT: xorl %eax, %eax +; CHECK-NEXT: vucomiss %xmm1, %xmm1 +; CHECK-NEXT: movl $65535, %ecx # imm = 0xFFFF +; CHECK-NEXT: movl $0, %edx +; CHECK-NEXT: cmovnpl %ecx, %edx +; CHECK-NEXT: vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3] +; CHECK-NEXT: vcvtph2ps %xmm1, %xmm1 +; CHECK-NEXT: vucomiss %xmm1, %xmm1 +; CHECK-NEXT: movl $0, %esi +; CHECK-NEXT: cmovnpl %ecx, %esi +; CHECK-NEXT: vpshuflw {{.*#+}} xmm1 = xmm0[1,1,1,1,4,5,6,7] +; CHECK-NEXT: vcvtph2ps %xmm1, %xmm1 +; CHECK-NEXT: vucomiss %xmm1, %xmm1 +; CHECK-NEXT: movl $0, %edi +; CHECK-NEXT: cmovnpl %ecx, %edi +; CHECK-NEXT: vcvtph2ps %xmm0, %xmm0 +; CHECK-NEXT: vucomiss %xmm0, %xmm0 +; CHECK-NEXT: cmovnpl %ecx, %eax +; CHECK-NEXT: vmovd %eax, %xmm0 +; CHECK-NEXT: vpinsrw $1, %edi, %xmm0, %xmm0 +; CHECK-NEXT: vpinsrw $2, %esi, %xmm0, %xmm0 +; CHECK-NEXT: vpinsrw $3, %edx, %xmm0, %xmm0 +; CHECK-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3] +; CHECK-NEXT: vxorps %xmm1, %xmm1, %xmm1 +; CHECK-NEXT: vpackssdw %xmm1, %xmm0, %xmm0 +; CHECK-NEXT: vmovdqu %xmm1, 16 +; CHECK-NEXT: vmovdqu %xmm0, 0 +; CHECK-NEXT: vzeroupper +; CHECK-NEXT: retq + %2 = fcmp ord <16 x half> %0, zeroinitializer + %3 = sext <16 x i1> %2 to <16 x i32> + %4 = shufflevector <16 x i32> %3, <16 x i32> zeroinitializer, <4 x i32> <i32 0, i32 1, i32 2, i32 3> + %5 = tail call <8 x i16> @llvm.x86.sse2.packssdw.128(<4 x i32> %4, <4 x i32> zeroinitializer) + %6 = shufflevector <8 x i16> %5, <8 x i16> zeroinitializer, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15> + %7 = bitcast <16 x i16> %6 to <32 x i8> + store <32 x i8> %7, ptr null, align 1 + ret void +} + +declare <8 x i16> @llvm.x86.sse2.packssdw.128(<4 x i32>, <4 x i32>) + +attributes #0 = { "target-features"="+aes,+avx,+avx2,+avx512f,+avx512vnni,+cmov,+crc32,+cx16,+cx8,+evex512,+f16c,+fma,+fxsr,+mmx,+pclmul,+popcnt,+prfchw,+sse,+sse2,+sse3,+sse4.1,+sse4.2,+ssse3,+x87,+xsave" } diff --git a/llvm/test/CodeGen/X86/pr118934.ll b/llvm/test/CodeGen/X86/pr118934.ll new file mode 100644 index 000000000000..b68ed2643d8d --- /dev/null +++ b/llvm/test/CodeGen/X86/pr118934.ll @@ -0,0 +1,45 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc < %s -mtriple=i686-unknown | FileCheck %s --check-prefix=X86 +; RUN: llc < %s -mtriple=x86_64-unknown | FileCheck %s --check-prefix=X64 + +define void @PR118934(i1 %b, ptr %f, ptr %k) { +; X86-LABEL: PR118934: +; X86: # %bb.0: # %entry +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: movzbl {{[0-9]+}}(%esp), %edx +; X86-NEXT: andb $1, %dl +; X86-NEXT: addb %dl, %dl +; X86-NEXT: addb $-2, %dl +; X86-NEXT: movsbl %dl, %edx +; X86-NEXT: addl $-6, %edx +; X86-NEXT: addl $6, %edx +; X86-NEXT: movl %edx, (%ecx) +; X86-NEXT: addl %edx, %edx +; X86-NEXT: movl %edx, (%eax) +; +; X64-LABEL: PR118934: +; X64: # %bb.0: # %entry +; X64-NEXT: andb $1, %dil +; X64-NEXT: addb %dil, %dil +; X64-NEXT: addb $-2, %dil +; X64-NEXT: movsbl %dil, %eax +; X64-NEXT: addl $-6, %eax +; X64-NEXT: addl $6, %eax +; X64-NEXT: movl %eax, (%rsi) +; X64-NEXT: addl %eax, %eax +; X64-NEXT: movl %eax, (%rdx) +entry: + %0 = select i1 %b, i8 0, i8 -2 + br label %for.end + +for.end: + %n.i.i = select i1 poison, i32 6, i32 7 + %narrow = add nsw i8 %0, -6 + %add2.i = sext i8 %narrow to i32 + %conv5.i = add nsw i32 %n.i.i, %add2.i + store i32 %conv5.i, ptr %f, align 4 + %add = shl nsw i32 %conv5.i, 1 + store i32 %add, ptr %k, align 4 + unreachable +} diff --git a/llvm/test/CodeGen/X86/pr119158.ll b/llvm/test/CodeGen/X86/pr119158.ll new file mode 100644 index 000000000000..ca31df802c91 --- /dev/null +++ b/llvm/test/CodeGen/X86/pr119158.ll @@ -0,0 +1,40 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5 +; RUN: llc < %s -mtriple=x86_64-- | FileCheck %s + +define dso_local void @foo() #1 { +; CHECK-LABEL: foo: +; CHECK: # %bb.0: # %newFuncRoot +; CHECK-NEXT: vpmovzxbd {{.*#+}} ymm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero +; CHECK-NEXT: vpbroadcastd {{.*#+}} ymm1 = [64,64,64,64,64,64,64,64] +; CHECK-NEXT: vpdpwssd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm0, %ymm1 +; CHECK-NEXT: vpsrld $7, %ymm1, %ymm0 +; CHECK-NEXT: vpackusdw %ymm0, %ymm0, %ymm0 +; CHECK-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3] +; CHECK-NEXT: vmovdqu %ymm0, (%rax) +; CHECK-NEXT: vzeroupper +; CHECK-NEXT: retq +newFuncRoot: + br label %loop + +loop: ; preds = %newFuncRoot, %loop + %0 = load <16 x i8>, ptr poison, align 1 + %1 = zext <16 x i8> %0 to <16 x i32> + %2 = mul nuw nsw <16 x i32> %1, splat (i32 18) + %3 = add nuw nsw <16 x i32> zeroinitializer, splat (i32 64) + %4 = add nuw nsw <16 x i32> %3, zeroinitializer + %5 = add nuw nsw <16 x i32> %4, %2 + %6 = sub nsw <16 x i32> %5, zeroinitializer + %7 = ashr <16 x i32> %6, splat (i32 7) + %8 = tail call <16 x i32> @llvm.smin.v16i32(<16 x i32> %7, <16 x i32> splat (i32 255)) + %9 = tail call <16 x i32> @llvm.smax.v16i32(<16 x i32> %8, <16 x i32> zeroinitializer) + %10 = trunc <16 x i32> %9 to <16 x i8> + %11 = shufflevector <16 x i8> %10, <16 x i8> poison, <32 x i32> <i32 0, i32 16, i32 1, i32 17, i32 2, i32 18, i32 3, i32 19, i32 4, i32 20, i32 5, i32 21, i32 6, i32 22, i32 7, i32 23, i32 8, i32 24, i32 9, i32 25, i32 10, i32 26, i32 11, i32 27, i32 12, i32 28, i32 13, i32 29, i32 14, i32 30, i32 15, i32 31> + store <32 x i8> %11, ptr poison, align 1 + br i1 poison, label %.exitStub, label %loop + +.exitStub: ; preds = %loop + ret void +} + +attributes #0 = { nocallback nofree nosync nounwind speculatable willreturn memory(none) } +attributes #1 = { "min-legal-vector-width"="0" "target-cpu"="tigerlake" "target-features"="+adx,+aes,+avx,+avx2,+avx512bitalg,+avx512bw,+avx512cd,+avx512dq,+avx512f,+avx512ifma,+avx512vbmi,+avx512vbmi2,+avx512vl,+avx512vnni,+avx512vp2intersect,+avx512vpopcntdq,+bmi,+bmi2,+clflushopt,+clwb,+cmov,+crc32,+cx16,+cx8,+evex512,+f16c,+fma,+fsgsbase,+fxsr,+gfni,+invpcid,+kl,+lzcnt,+mmx,+movbe,+movdir64b,+movdiri,+pclmul,+pku,+popcnt,+prfchw,+rdpid,+rdrnd,+rdseed,+sahf,+sgx,+sha,+shstk,+sse,+sse2,+sse3,+sse4.1,+sse4.2,+ssse3,+vaes,+vpclmulqdq,+widekl,+x87,+xsave,+xsavec,+xsaveopt,+xsaves" } diff --git a/llvm/test/CodeGen/X86/pr120093.ll b/llvm/test/CodeGen/X86/pr120093.ll new file mode 100644 index 000000000000..99bf2218c553 --- /dev/null +++ b/llvm/test/CodeGen/X86/pr120093.ll @@ -0,0 +1,31 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5 +; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu | FileCheck %s + +define double @PR120093() { +; CHECK-LABEL: PR120093: +; CHECK: # %bb.0: # %bb +; CHECK-NEXT: xorpd %xmm0, %xmm0 +; CHECK-NEXT: xorpd %xmm1, %xmm1 +; CHECK-NEXT: movhpd {{.*#+}} xmm1 = xmm1[0],mem[0] +; CHECK-NEXT: cmpnltpd %xmm1, %xmm0 +; CHECK-NEXT: movmskpd %xmm0, %eax +; CHECK-NEXT: cmpl $3, %eax +; CHECK-NEXT: jne .LBB0_2 +; CHECK-NEXT: # %bb.1: # %bb2 +; CHECK-NEXT: xorpd %xmm0, %xmm0 +; CHECK-NEXT: retq +; CHECK-NEXT: .LBB0_2: # %bb3 +bb: + %insertelement = insertelement <2 x double> <double poison, double 0xFFFFFFFFFFFFFFFF>, double 0.000000e+00, i64 0 + %fcmp = fcmp uge <2 x double> zeroinitializer, %insertelement + %extractelement = extractelement <2 x i1> %fcmp, i64 0 + %extractelement1 = extractelement <2 x i1> %fcmp, i64 1 + %select = select i1 %extractelement, i1 %extractelement1, i1 false + br i1 %select, label %bb2, label %bb3 + +bb2: ; preds = %bb + ret double 0.000000e+00 + +bb3: ; preds = %bb + unreachable +} diff --git a/llvm/test/CodeGen/X86/pr120906.ll b/llvm/test/CodeGen/X86/pr120906.ll new file mode 100644 index 000000000000..f5f6331bf3bf --- /dev/null +++ b/llvm/test/CodeGen/X86/pr120906.ll @@ -0,0 +1,40 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5 +; RUN: llc < %s -mtriple=x86_64-- | FileCheck %s + +define i32 @PR120906(ptr %p) { +; CHECK-LABEL: PR120906: +; CHECK: # %bb.0: +; CHECK-NEXT: movl $564341309, (%rdi) # imm = 0x21A32A3D +; CHECK-NEXT: pxor %xmm0, %xmm0 +; CHECK-NEXT: pxor %xmm1, %xmm1 +; CHECK-NEXT: paddb %xmm1, %xmm1 +; CHECK-NEXT: paddb %xmm1, %xmm1 +; CHECK-NEXT: pxor %xmm2, %xmm2 +; CHECK-NEXT: pcmpgtb %xmm1, %xmm2 +; CHECK-NEXT: movdqa {{.*#+}} xmm1 = [11,11,11,11,u,u,u,u,u,u,u,u,u,u,u,u] +; CHECK-NEXT: movdqa %xmm1, %xmm3 +; CHECK-NEXT: paddb %xmm1, %xmm3 +; CHECK-NEXT: pand %xmm2, %xmm3 +; CHECK-NEXT: pandn %xmm1, %xmm2 +; CHECK-NEXT: por %xmm1, %xmm2 +; CHECK-NEXT: por %xmm3, %xmm2 +; CHECK-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7] +; CHECK-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3] +; CHECK-NEXT: pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3] +; CHECK-NEXT: por %xmm2, %xmm0 +; CHECK-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1] +; CHECK-NEXT: por %xmm0, %xmm1 +; CHECK-NEXT: movd %xmm1, %eax +; CHECK-NEXT: retq + store i32 564341309, ptr %p, align 4 + %load = load i32, ptr %p, align 4 + %broadcast.splatinsert.1 = insertelement <4 x i32> zeroinitializer, i32 %load, i64 0 + %broadcast.splat.1 = shufflevector <4 x i32> %broadcast.splatinsert.1, <4 x i32> zeroinitializer, <4 x i32> zeroinitializer + %icmp = icmp ugt <4 x i32> %broadcast.splat.1, splat (i32 -9) + %zext8 = zext <4 x i1> %icmp to <4 x i8> + %shl = shl <4 x i8> splat (i8 11), %zext8 + %or = or <4 x i8> %shl, splat (i8 11) + %zext32 = zext <4 x i8> %or to <4 x i32> + %rdx = tail call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> %zext32) + ret i32 %rdx +} diff --git a/llvm/test/CodeGen/X86/pr33349.ll b/llvm/test/CodeGen/X86/pr33349.ll index 83d3a3357226..c879cb9867ab 100644 --- a/llvm/test/CodeGen/X86/pr33349.ll +++ b/llvm/test/CodeGen/X86/pr33349.ll @@ -17,23 +17,23 @@ target triple = "x86_64-unknown-linux-gnu" ; KNL-NEXT: fldz ; KNL-NEXT: fld %st(0) ; KNL-NEXT: fcmovne %st(2), %st -; KNL-NEXT: testb $2, %al -; KNL-NEXT: fld %st(1) -; KNL-NEXT: fcmovne %st(3), %st ; KNL-NEXT: kmovw %k0, %eax ; KNL-NEXT: testb $1, %al +; KNL-NEXT: fld %st(1) +; KNL-NEXT: fcmovne %st(3), %st +; KNL-NEXT: testb $2, %al ; KNL-NEXT: fld %st(2) ; KNL-NEXT: fcmovne %st(4), %st -; KNL-NEXT: testb $2, %al +; KNL-NEXT: testb $8, %al ; KNL-NEXT: fxch %st(3) ; KNL-NEXT: fcmovne %st(4), %st ; KNL-NEXT: fstp %st(4) ; KNL-NEXT: fxch %st(3) +; KNL-NEXT: fstpt 30(%rdi) +; KNL-NEXT: fxch %st(1) ; KNL-NEXT: fstpt 10(%rdi) ; KNL-NEXT: fxch %st(1) ; KNL-NEXT: fstpt (%rdi) -; KNL-NEXT: fxch %st(1) -; KNL-NEXT: fstpt 30(%rdi) ; KNL-NEXT: fstpt 20(%rdi) ; KNL-NEXT: vzeroupper ; KNL-NEXT: retq @@ -49,23 +49,23 @@ target triple = "x86_64-unknown-linux-gnu" ; SKX-NEXT: fldz ; SKX-NEXT: fld %st(0) ; SKX-NEXT: fcmovne %st(2), %st -; SKX-NEXT: testb $2, %al -; SKX-NEXT: fld %st(1) -; SKX-NEXT: fcmovne %st(3), %st ; SKX-NEXT: kmovd %k0, %eax ; SKX-NEXT: testb $1, %al +; SKX-NEXT: fld %st(1) +; SKX-NEXT: fcmovne %st(3), %st +; SKX-NEXT: testb $2, %al ; SKX-NEXT: fld %st(2) ; SKX-NEXT: fcmovne %st(4), %st -; SKX-NEXT: testb $2, %al +; SKX-NEXT: testb $8, %al ; SKX-NEXT: fxch %st(3) ; SKX-NEXT: fcmovne %st(4), %st ; SKX-NEXT: fstp %st(4) ; SKX-NEXT: fxch %st(3) +; SKX-NEXT: fstpt 30(%rdi) +; SKX-NEXT: fxch %st(1) ; SKX-NEXT: fstpt 10(%rdi) ; SKX-NEXT: fxch %st(1) ; SKX-NEXT: fstpt (%rdi) -; SKX-NEXT: fxch %st(1) -; SKX-NEXT: fstpt 30(%rdi) ; SKX-NEXT: fstpt 20(%rdi) ; SKX-NEXT: retq bb: diff --git a/llvm/test/CodeGen/X86/pr34177.ll b/llvm/test/CodeGen/X86/pr34177.ll index 29922c2ac1a7..5b2431eb2149 100644 --- a/llvm/test/CodeGen/X86/pr34177.ll +++ b/llvm/test/CodeGen/X86/pr34177.ll @@ -51,18 +51,18 @@ define void @test(<4 x i64> %a, <4 x x86_fp80> %b, ptr %c) local_unnamed_addr { ; AVX512VL-NEXT: vpcmpeqq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %k0 ; AVX512VL-NEXT: kshiftrb $2, %k0, %k1 ; AVX512VL-NEXT: kmovd %k0, %eax -; AVX512VL-NEXT: testb $2, %al +; AVX512VL-NEXT: testb $8, %al ; AVX512VL-NEXT: fld1 ; AVX512VL-NEXT: fldz ; AVX512VL-NEXT: fld %st(0) ; AVX512VL-NEXT: fcmovne %st(2), %st -; AVX512VL-NEXT: testb $1, %al +; AVX512VL-NEXT: testb $2, %al ; AVX512VL-NEXT: fld %st(1) ; AVX512VL-NEXT: fcmovne %st(3), %st -; AVX512VL-NEXT: kmovd %k1, %eax -; AVX512VL-NEXT: testb $2, %al +; AVX512VL-NEXT: testb $1, %al ; AVX512VL-NEXT: fld %st(2) ; AVX512VL-NEXT: fcmovne %st(4), %st +; AVX512VL-NEXT: kmovd %k1, %eax ; AVX512VL-NEXT: testb $1, %al ; AVX512VL-NEXT: fxch %st(3) ; AVX512VL-NEXT: fcmovne %st(4), %st @@ -77,12 +77,12 @@ define void @test(<4 x i64> %a, <4 x x86_fp80> %b, ptr %c) local_unnamed_addr { ; AVX512VL-NEXT: fstpt 10(%rdi) ; AVX512VL-NEXT: fxch %st(1) ; AVX512VL-NEXT: fadd %st, %st(0) +; AVX512VL-NEXT: fstpt 60(%rdi) +; AVX512VL-NEXT: fadd %st, %st(0) ; AVX512VL-NEXT: fstpt 20(%rdi) ; AVX512VL-NEXT: fadd %st, %st(0) ; AVX512VL-NEXT: fstpt (%rdi) ; AVX512VL-NEXT: fadd %st, %st(0) -; AVX512VL-NEXT: fstpt 60(%rdi) -; AVX512VL-NEXT: fadd %st, %st(0) ; AVX512VL-NEXT: fstpt 40(%rdi) %1 = icmp eq <4 x i64> <i64 0, i64 1, i64 2, i64 3>, %a %2 = select <4 x i1> %1, <4 x x86_fp80> <x86_fp80 0xK3FFF8000000000000000, x86_fp80 0xK3FFF8000000000000000, x86_fp80 0xK3FFF8000000000000000, x86_fp80 0xK3FFF8000000000000000>, <4 x x86_fp80> zeroinitializer diff --git a/llvm/test/CodeGen/X86/pr35763.ll b/llvm/test/CodeGen/X86/pr35763.ll index 9d2ee84cf675..c10a24cffe5f 100644 --- a/llvm/test/CodeGen/X86/pr35763.ll +++ b/llvm/test/CodeGen/X86/pr35763.ll @@ -14,15 +14,7 @@ define dso_local void @PR35763() { ; CHECK-NEXT: movzwl z+2(%rip), %ecx ; CHECK-NEXT: orl %eax, %ecx ; CHECK-NEXT: movq %rcx, tf_3_var_136(%rip) -; CHECK-NEXT: movl z+6(%rip), %eax -; CHECK-NEXT: movzbl z+10(%rip), %ecx -; CHECK-NEXT: shlq $32, %rcx -; CHECK-NEXT: orq %rax, %rcx -; CHECK-NEXT: movabsq $1090921758719, %rax # imm = 0xFE0000FFFF -; CHECK-NEXT: andq %rcx, %rax -; CHECK-NEXT: movl %eax, z+6(%rip) -; CHECK-NEXT: shrq $32, %rax -; CHECK-NEXT: movb %al, z+10(%rip) +; CHECK-NEXT: andl $-33554177, z+7(%rip) # imm = 0xFE0000FF ; CHECK-NEXT: retq entry: %0 = load i16, ptr @z, align 8 diff --git a/llvm/test/CodeGen/X86/sincos-stack-args.ll b/llvm/test/CodeGen/X86/sincos-stack-args.ll index 9fb3a6769fda..fe1533083333 100644 --- a/llvm/test/CodeGen/X86/sincos-stack-args.ll +++ b/llvm/test/CodeGen/X86/sincos-stack-args.ll @@ -4,8 +4,11 @@ declare double @g(double, double) -define double @f(double %a) { -; CHECK-LABEL: f: +; Though not visible within the IR, this will lower to an FSINCOS node, with +; store users, that are within a (callseq_start, callseq_end) pair. In this +; case, the stores cannot be folded into the sincos call. +define double @negative_sincos_with_stores_within_call_sequence(double %a) { +; CHECK-LABEL: negative_sincos_with_stores_within_call_sequence: ; CHECK: # %bb.0: # %entry ; CHECK-NEXT: subl $44, %esp ; CHECK-NEXT: .cfi_def_cfa_offset 48 diff --git a/llvm/test/CodeGen/X86/stack-frame-layout-remarks.ll b/llvm/test/CodeGen/X86/stack-frame-layout-remarks.ll index d8ce5b041042..cfc493af6670 100644 --- a/llvm/test/CodeGen/X86/stack-frame-layout-remarks.ll +++ b/llvm/test/CodeGen/X86/stack-frame-layout-remarks.ll @@ -7,10 +7,10 @@ ; RUN: llc -mcpu=corei7 -O0 -pass-remarks-analysis=stack-frame-layout < %s 2>&1 >/dev/null | FileCheck %s --check-prefix=NO_COLORING ; check more complex cases -; RUN: llc %s -pass-remarks-analysis=stack-frame-layout -o /dev/null --march=x86 -mcpu=i386 2>&1 | FileCheck %s --check-prefix=BOTH --check-prefix=DEBUG +; RUN: llc %s -pass-remarks-analysis=stack-frame-layout -o /dev/null -mtriple=i686-unknown-linux 2>&1 | FileCheck %s --check-prefix=BOTH --check-prefix=DEBUG ; check output without debug info -; RUN: opt %s -passes=strip -S | llc -pass-remarks-analysis=stack-frame-layout -o /dev/null --march=x86 -mcpu=i386 2>&1 | FileCheck %s --check-prefix=BOTH --check-prefix=STRIPPED +; RUN: opt %s -passes=strip -S | llc -pass-remarks-analysis=stack-frame-layout -o /dev/null -mtriple=i686-unknown-linux 2>&1 | FileCheck %s --check-prefix=BOTH --check-prefix=STRIPPED target triple = "x86_64-unknown-linux-gnu" diff --git a/llvm/test/CodeGen/X86/stackmap-args.ll b/llvm/test/CodeGen/X86/stackmap-args.ll new file mode 100644 index 000000000000..622c41c06fb9 --- /dev/null +++ b/llvm/test/CodeGen/X86/stackmap-args.ll @@ -0,0 +1,22 @@ +; RUN: not llc -mtriple=x86_64-apple-darwin -mcpu=corei7 < %s 2>&1 | FileCheck %s +; Tests error when we pass non-immediate parameters to @llvm.experiment.stackmap + +define void @first_arg() { +; CHECK: immarg operand has non-immediate parameter +entry: + ; First operand should be immediate + %id = add i64 0, 0 + call void (i64, i32, ...) @llvm.experimental.stackmap(i64 %id, i32 0) + ret void +} + +define void @second_arg() { +; CHECK: immarg operand has non-immediate parameter +entry: + ; Second operand should be immediate + %numShadowByte = add i32 0, 0 + call void (i64, i32, ...) @llvm.experimental.stackmap(i64 1, i32 %numShadowByte) + ret void +} + +declare void @llvm.experimental.stackmap(i64, i32, ...)
\ No newline at end of file diff --git a/llvm/test/CodeGen/X86/stackmap-undef-operand-anyregcc.mir b/llvm/test/CodeGen/X86/stackmap-undef-operand-anyregcc.mir new file mode 100644 index 000000000000..3c6390ba2379 --- /dev/null +++ b/llvm/test/CodeGen/X86/stackmap-undef-operand-anyregcc.mir @@ -0,0 +1,77 @@ +# RUN: llc -mtriple=x86_64-apple-darwin -start-after=virtregrewriter -o - %s | FileCheck %s + +# Check there's no assertion for anyregcc with an undef operand to a stackmap. + +# CHECK: __LLVM_StackMaps: +# CHECK-NEXT: .byte 3 +# CHECK-NEXT: .byte 0 +# CHECK-NEXT: .short 0 +# CHECK-NEXT: .long 1 +# CHECK-NEXT: .long 0 +# CHECK-NEXT: .long 1 +# CHECK-NEXT: .quad _undef_anyregcc_patchpoint +# CHECK-NEXT: .quad 8 +# CHECK-NEXT: .quad 1 +# CHECK-NEXT: .quad 12 +# CHECK-NEXT: .long Ltmp0-_undef_anyregcc_patchpoint +# CHECK-NEXT: .short 0 +# CHECK-NEXT: .short 2 +# CHECK-NEXT: .byte 1 +# CHECK-NEXT: .byte 0 +# CHECK-NEXT: .short 8 +# CHECK-NEXT: .short 0 +# CHECK-NEXT: .short 0 +# CHECK-NEXT: .long 0 +# CHECK-NEXT: .byte 1 +# CHECK-NEXT: .byte 0 +# CHECK-NEXT: .short 8 +# CHECK-NEXT: .short 0 +# CHECK-NEXT: .short 0 +# CHECK-NEXT: .long 0 +# CHECK-NEXT: .p2align 3, 0x0 +# CHECK-NEXT: .short 0 +# CHECK-NEXT: .short 7 +# CHECK-NEXT: .short 0 +# CHECK-NEXT: .byte 0 +# CHECK-NEXT: .byte 8 +# CHECK-NEXT: .short 3 +# CHECK-NEXT: .byte 0 +# CHECK-NEXT: .byte 8 +# CHECK-NEXT: .short 7 +# CHECK-NEXT: .byte 0 +# CHECK-NEXT: .byte 8 +# CHECK-NEXT: .short 12 +# CHECK-NEXT: .byte 0 +# CHECK-NEXT: .byte 8 +# CHECK-NEXT: .short 13 +# CHECK-NEXT: .byte 0 +# CHECK-NEXT: .byte 8 +# CHECK-NEXT: .short 14 +# CHECK-NEXT: .byte 0 +# CHECK-NEXT: .byte 8 +# CHECK-NEXT: .short 15 +# CHECK-NEXT: .byte 0 +# CHECK-NEXT: .byte 8 +# CHECK-NEXT: .p2align 3 +--- +name: undef_anyregcc_patchpoint +tracksRegLiveness: true +frameInfo: + hasPatchPoint: true + hasCalls: true + adjustsStack: true +fixedStack: + - { id: 0, type: default, offset: 72, size: 8, alignment: 8, stack-id: default, + isImmutable: true, isAliased: false, callee-saved-register: '', callee-saved-restored: true, + debug-info-variable: '', debug-info-expression: '', debug-info-location: '' } +body: | + bb.0: + liveins: $rcx, $rdi, $rdx, $rsi, $r8, $r9 + + ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp + dead renamable $rax = MOV64rm %fixed-stack.0, 1, $noreg, 0, $noreg :: (load (s64) from %fixed-stack.0) + renamable $rax = PATCHPOINT 12, 15, 0, 1, 13, undef renamable $rax, csr_64_allregs, implicit-def dead early-clobber $r11, implicit-def $rsp, implicit-def $ssp + ADJCALLSTACKUP64 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp + RET 0, $rax + +... diff --git a/llvm/test/CodeGen/X86/statepoint-fixup-undef.mir b/llvm/test/CodeGen/X86/statepoint-fixup-undef.mir index 4a18351bde49..edb0d517d5d5 100644 --- a/llvm/test/CodeGen/X86/statepoint-fixup-undef.mir +++ b/llvm/test/CodeGen/X86/statepoint-fixup-undef.mir @@ -181,13 +181,13 @@ body: | ; STACKMAP-NEXT: .short 3 ; STACKMAP-NEXT: .short 0 ; STACKMAP-NEXT: .long 0 - ; This is a constant 0xFEFEFEFE we are looking for - ; STACKMAP-NEXT: .byte 4 + ; This is entry we're looking for + ; STACKMAP-NEXT: .byte 1 ; STACKMAP-NEXT: .byte 0 - ; STACKMAP-NEXT: .short 8 + ; STACKMAP-NEXT: .short 4 ; STACKMAP-NEXT: .short 0 ; STACKMAP-NEXT: .short 0 - ; STACKMAP-NEXT: .long -16843010 + ; STACKMAP-NEXT: .long 0 ; STACKMAP-NEXT: .byte 4 ; STACKMAP-NEXT: .byte 0 ; STACKMAP-NEXT: .short 8 diff --git a/llvm/test/CodeGen/X86/store_op_load_fold.ll b/llvm/test/CodeGen/X86/store_op_load_fold.ll index bd7068535eab..0309fa7bb031 100644 --- a/llvm/test/CodeGen/X86/store_op_load_fold.ll +++ b/llvm/test/CodeGen/X86/store_op_load_fold.ll @@ -23,7 +23,7 @@ define void @test2() nounwind uwtable ssp { ; CHECK-LABEL: test2: ; CHECK: ## %bb.0: ; CHECK-NEXT: movl L_s2$non_lazy_ptr, %eax -; CHECK-NEXT: andl $-262144, 20(%eax) ## imm = 0xFFFC0000 +; CHECK-NEXT: andl $-67108609, 19(%eax) ## imm = 0xFC0000FF ; CHECK-NEXT: retl %bf.load35 = load i56, ptr getelementptr inbounds (%struct.S2, ptr @s2, i32 0, i32 5), align 16 %bf.clear36 = and i56 %bf.load35, -1125895611875329 diff --git a/llvm/test/CodeGen/X86/urem-seteq-illegal-types.ll b/llvm/test/CodeGen/X86/urem-seteq-illegal-types.ll index b4e91da920a2..28ac4496acb9 100644 --- a/llvm/test/CodeGen/X86/urem-seteq-illegal-types.ll +++ b/llvm/test/CodeGen/X86/urem-seteq-illegal-types.ll @@ -245,7 +245,7 @@ define <3 x i1> @test_urem_vec(<3 x i11> %X) nounwind { ; AVX512VL-NEXT: vpbroadcastd {{.*#+}} xmm2 = [2047,2047,2047,2047] ; AVX512VL-NEXT: vpand %xmm2, %xmm0, %xmm0 ; AVX512VL-NEXT: vpsrlvd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 -; AVX512VL-NEXT: vpternlogd $200, %xmm1, %xmm2, %xmm0 +; AVX512VL-NEXT: vpternlogd {{.*#+}} xmm0 = xmm2 & (xmm0 | xmm1) ; AVX512VL-NEXT: vpcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %k0 ; AVX512VL-NEXT: kshiftrw $1, %k0, %k1 ; AVX512VL-NEXT: kmovw %k1, %edx diff --git a/llvm/test/CodeGen/X86/vec-strict-cmp-128.ll b/llvm/test/CodeGen/X86/vec-strict-cmp-128.ll index 66f91dbe2d63..209d6a5a6710 100644 --- a/llvm/test/CodeGen/X86/vec-strict-cmp-128.ll +++ b/llvm/test/CodeGen/X86/vec-strict-cmp-128.ll @@ -6087,9 +6087,106 @@ define <2 x double> @test_v4f64_ogt2_s(<2 x double> %a, <2 x double> %b) #0 { ret <2 x double> %res } +define <2 x float> @test_v2f32_ogt2_s(<2 x float> %a, <2 x float> %b) #0 { +; SSE-32-LABEL: test_v2f32_ogt2_s: +; SSE-32: # %bb.0: +; SSE-32-NEXT: maxps %xmm1, %xmm0 +; SSE-32-NEXT: retl +; +; SSE-64-LABEL: test_v2f32_ogt2_s: +; SSE-64: # %bb.0: +; SSE-64-NEXT: maxps %xmm1, %xmm0 +; SSE-64-NEXT: retq +; +; AVX-32-LABEL: test_v2f32_ogt2_s: +; AVX-32: # %bb.0: +; AVX-32-NEXT: vmaxps %xmm1, %xmm0, %xmm0 +; AVX-32-NEXT: retl +; +; AVX-64-LABEL: test_v2f32_ogt2_s: +; AVX-64: # %bb.0: +; AVX-64-NEXT: vmaxps %xmm1, %xmm0, %xmm0 +; AVX-64-NEXT: retq +; +; AVX512-32-LABEL: test_v2f32_ogt2_s: +; AVX512-32: # %bb.0: +; AVX512-32-NEXT: vmaxps %xmm1, %xmm0, %xmm0 +; AVX512-32-NEXT: retl +; +; AVX512-64-LABEL: test_v2f32_ogt2_s: +; AVX512-64: # %bb.0: +; AVX512-64-NEXT: vmaxps %xmm1, %xmm0, %xmm0 +; AVX512-64-NEXT: retq +; +; AVX512F-32-LABEL: test_v2f32_ogt2_s: +; AVX512F-32: # %bb.0: +; AVX512F-32-NEXT: vmaxps %xmm1, %xmm0, %xmm0 +; AVX512F-32-NEXT: retl +; +; AVX512F-64-LABEL: test_v2f32_ogt2_s: +; AVX512F-64: # %bb.0: +; AVX512F-64-NEXT: vmaxps %xmm1, %xmm0, %xmm0 +; AVX512F-64-NEXT: retq + %cond = call <2 x i1> @llvm.experimental.constrained.fcmps.v2f32( + <2 x float> %a, <2 x float> %b, metadata !"ogt", + metadata !"fpexcept.strict") + %res = select <2 x i1> %cond, <2 x float> %a, <2 x float> %b + ret <2 x float> %res +} + +define <2 x float> @test_v2f32_ule2_s(<2 x float> %a, <2 x float> %b) #0 { +; SSE-32-LABEL: test_v2f32_ule2_s: +; SSE-32: # %bb.0: +; SSE-32-NEXT: minps %xmm0, %xmm1 +; SSE-32-NEXT: movaps %xmm1, %xmm0 +; SSE-32-NEXT: retl +; +; SSE-64-LABEL: test_v2f32_ule2_s: +; SSE-64: # %bb.0: +; SSE-64-NEXT: minps %xmm0, %xmm1 +; SSE-64-NEXT: movaps %xmm1, %xmm0 +; SSE-64-NEXT: retq +; +; AVX-32-LABEL: test_v2f32_ule2_s: +; AVX-32: # %bb.0: +; AVX-32-NEXT: vminps %xmm0, %xmm1, %xmm0 +; AVX-32-NEXT: retl +; +; AVX-64-LABEL: test_v2f32_ule2_s: +; AVX-64: # %bb.0: +; AVX-64-NEXT: vminps %xmm0, %xmm1, %xmm0 +; AVX-64-NEXT: retq +; +; AVX512-32-LABEL: test_v2f32_ule2_s: +; AVX512-32: # %bb.0: +; AVX512-32-NEXT: vminps %xmm0, %xmm1, %xmm0 +; AVX512-32-NEXT: retl +; +; AVX512-64-LABEL: test_v2f32_ule2_s: +; AVX512-64: # %bb.0: +; AVX512-64-NEXT: vminps %xmm0, %xmm1, %xmm0 +; AVX512-64-NEXT: retq +; +; AVX512F-32-LABEL: test_v2f32_ule2_s: +; AVX512F-32: # %bb.0: +; AVX512F-32-NEXT: vminps %xmm0, %xmm1, %xmm0 +; AVX512F-32-NEXT: retl +; +; AVX512F-64-LABEL: test_v2f32_ule2_s: +; AVX512F-64: # %bb.0: +; AVX512F-64-NEXT: vminps %xmm0, %xmm1, %xmm0 +; AVX512F-64-NEXT: retq + %cond = call <2 x i1> @llvm.experimental.constrained.fcmps.v2f32( + <2 x float> %a, <2 x float> %b, metadata !"ule", + metadata !"fpexcept.strict") + %res = select <2 x i1> %cond, <2 x float> %a, <2 x float> %b + ret <2 x float> %res +} + attributes #0 = { strictfp nounwind } declare <4 x i1> @llvm.experimental.constrained.fcmp.v4f32(<4 x float>, <4 x float>, metadata, metadata) declare <2 x i1> @llvm.experimental.constrained.fcmp.v2f64(<2 x double>, <2 x double>, metadata, metadata) +declare <2 x i1> @llvm.experimental.constrained.fcmps.v2f32(<2 x float>, <2 x float>, metadata, metadata) declare <4 x i1> @llvm.experimental.constrained.fcmps.v4f32(<4 x float>, <4 x float>, metadata, metadata) declare <2 x i1> @llvm.experimental.constrained.fcmps.v2f64(<2 x double>, <2 x double>, metadata, metadata) diff --git a/llvm/test/CodeGen/X86/vec_shift6.ll b/llvm/test/CodeGen/X86/vec_shift6.ll index 59bc3940fcb3..71e659c681d1 100644 --- a/llvm/test/CodeGen/X86/vec_shift6.ll +++ b/llvm/test/CodeGen/X86/vec_shift6.ll @@ -22,15 +22,27 @@ define <8 x i16> @test1(<8 x i16> %a) { ret <8 x i16> %shl } +; Only two legal shift amounts, so we can lower to shuffle(psllw(),psllw()) + define <8 x i16> @test2(<8 x i16> %a) { -; SSE-LABEL: test2: -; SSE: # %bb.0: -; SSE-NEXT: pmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 # [1,u,1,1,2,u,u,2] -; SSE-NEXT: retq +; SSE2-LABEL: test2: +; SSE2: # %bb.0: +; SSE2-NEXT: movdqa %xmm0, %xmm1 +; SSE2-NEXT: paddw %xmm0, %xmm1 +; SSE2-NEXT: shufps {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3] +; SSE2-NEXT: retq +; +; SSE41-LABEL: test2: +; SSE41: # %bb.0: +; SSE41-NEXT: movdqa %xmm0, %xmm1 +; SSE41-NEXT: paddw %xmm0, %xmm1 +; SSE41-NEXT: pblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm1[4,5,6,7] +; SSE41-NEXT: retq ; ; AVX-LABEL: test2: ; AVX: # %bb.0: -; AVX-NEXT: vpmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 # [1,u,1,1,2,u,u,2] +; AVX-NEXT: vpaddw %xmm0, %xmm0, %xmm1 +; AVX-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3] ; AVX-NEXT: retq %shl = shl <8 x i16> %a, <i16 0, i16 undef, i16 0, i16 0, i16 1, i16 undef, i16 -1, i16 1> ret <8 x i16> %shl @@ -43,17 +55,18 @@ define <8 x i16> @test2(<8 x i16> %a) { define <4 x i32> @test3(<4 x i32> %a) { ; SSE2-LABEL: test3: ; SSE2: # %bb.0: -; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,3,3] -; SSE2-NEXT: pmuludq %xmm0, %xmm1 -; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3] -; SSE2-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 -; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] -; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] +; SSE2-NEXT: movdqa %xmm0, %xmm1 +; SSE2-NEXT: paddd %xmm0, %xmm1 +; SSE2-NEXT: pslld $2, %xmm0 +; SSE2-NEXT: movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1] ; SSE2-NEXT: retq ; ; SSE41-LABEL: test3: ; SSE41: # %bb.0: -; SSE41-NEXT: pmulld {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 +; SSE41-NEXT: movdqa %xmm0, %xmm1 +; SSE41-NEXT: pslld $2, %xmm1 +; SSE41-NEXT: paddd %xmm0, %xmm0 +; SSE41-NEXT: pblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm1[4,5,6,7] ; SSE41-NEXT: retq ; ; AVX-LABEL: test3: @@ -68,14 +81,14 @@ define <4 x i32> @test4(<4 x i32> %a) { ; SSE2-LABEL: test4: ; SSE2: # %bb.0: ; SSE2-NEXT: movdqa %xmm0, %xmm1 -; SSE2-NEXT: pslld $1, %xmm1 +; SSE2-NEXT: paddd %xmm0, %xmm1 ; SSE2-NEXT: shufps {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3] ; SSE2-NEXT: retq ; ; SSE41-LABEL: test4: ; SSE41: # %bb.0: ; SSE41-NEXT: movdqa %xmm0, %xmm1 -; SSE41-NEXT: pslld $1, %xmm1 +; SSE41-NEXT: paddd %xmm0, %xmm1 ; SSE41-NEXT: pblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm1[4,5,6,7] ; SSE41-NEXT: retq ; diff --git a/llvm/test/CodeGen/X86/vec_smulo.ll b/llvm/test/CodeGen/X86/vec_smulo.ll index 22b5246443fa..7e081310c35b 100644 --- a/llvm/test/CodeGen/X86/vec_smulo.ll +++ b/llvm/test/CodeGen/X86/vec_smulo.ll @@ -2668,11 +2668,11 @@ define <64 x i32> @smulo_v64i8(<64 x i8> %a0, <64 x i8> %a1, ptr %p2) nounwind { ; AVX512BW-NEXT: vpmovm2b %k0, %zmm0 ; AVX512BW-NEXT: vpcmpneqb %zmm1, %zmm0, %k1 ; AVX512BW-NEXT: vpternlogd {{.*#+}} zmm0 {%k1} {z} = -1 -; AVX512BW-NEXT: kshiftrd $16, %k1, %k2 +; AVX512BW-NEXT: kshiftrq $16, %k1, %k2 ; AVX512BW-NEXT: vpternlogd {{.*#+}} zmm1 {%k2} {z} = -1 -; AVX512BW-NEXT: kshiftrq $32, %k1, %k1 -; AVX512BW-NEXT: vpternlogd {{.*#+}} zmm2 {%k1} {z} = -1 -; AVX512BW-NEXT: kshiftrd $16, %k1, %k1 +; AVX512BW-NEXT: kshiftrq $32, %k1, %k2 +; AVX512BW-NEXT: vpternlogd {{.*#+}} zmm2 {%k2} {z} = -1 +; AVX512BW-NEXT: kshiftrq $48, %k1, %k1 ; AVX512BW-NEXT: vpternlogd {{.*#+}} zmm3 {%k1} {z} = -1 ; AVX512BW-NEXT: vmovdqa64 %zmm4, (%rdi) ; AVX512BW-NEXT: retq diff --git a/llvm/test/CodeGen/X86/vec_umulo.ll b/llvm/test/CodeGen/X86/vec_umulo.ll index 4d7d2573183e..68c6ca93576b 100644 --- a/llvm/test/CodeGen/X86/vec_umulo.ll +++ b/llvm/test/CodeGen/X86/vec_umulo.ll @@ -2329,11 +2329,11 @@ define <64 x i32> @umulo_v64i8(<64 x i8> %a0, <64 x i8> %a1, ptr %p2) nounwind { ; AVX512BW-NEXT: vpackuswb %zmm1, %zmm0, %zmm0 ; AVX512BW-NEXT: vptestmb %zmm0, %zmm0, %k1 ; AVX512BW-NEXT: vpternlogd {{.*#+}} zmm0 {%k1} {z} = -1 -; AVX512BW-NEXT: kshiftrd $16, %k1, %k2 +; AVX512BW-NEXT: kshiftrq $16, %k1, %k2 ; AVX512BW-NEXT: vpternlogd {{.*#+}} zmm1 {%k2} {z} = -1 -; AVX512BW-NEXT: kshiftrq $32, %k1, %k1 -; AVX512BW-NEXT: vpternlogd {{.*#+}} zmm2 {%k1} {z} = -1 -; AVX512BW-NEXT: kshiftrd $16, %k1, %k1 +; AVX512BW-NEXT: kshiftrq $32, %k1, %k2 +; AVX512BW-NEXT: vpternlogd {{.*#+}} zmm2 {%k2} {z} = -1 +; AVX512BW-NEXT: kshiftrq $48, %k1, %k1 ; AVX512BW-NEXT: vpternlogd {{.*#+}} zmm3 {%k1} {z} = -1 ; AVX512BW-NEXT: vmovdqa64 %zmm4, (%rdi) ; AVX512BW-NEXT: retq diff --git a/llvm/test/CodeGen/X86/vector-compress.ll b/llvm/test/CodeGen/X86/vector-compress.ll index f8c076db65de..17b98b5ebcae 100644 --- a/llvm/test/CodeGen/X86/vector-compress.ll +++ b/llvm/test/CodeGen/X86/vector-compress.ll @@ -840,12 +840,12 @@ define <64 x i32> @test_compress_large(<64 x i1> %mask, <64 x i32> %vec, <64 x i ; AVX512VL-NEXT: subq $576, %rsp # imm = 0x240 ; AVX512VL-NEXT: vpsllw $7, %zmm0, %zmm0 ; AVX512VL-NEXT: vpmovb2m %zmm0, %k1 +; AVX512VL-NEXT: kshiftrq $48, %k1, %k3 ; AVX512VL-NEXT: kshiftrq $32, %k1, %k4 -; AVX512VL-NEXT: kshiftrd $16, %k4, %k3 -; AVX512VL-NEXT: kshiftrd $16, %k1, %k2 +; AVX512VL-NEXT: kshiftrq $16, %k1, %k2 ; AVX512VL-NEXT: vpcompressd %zmm1, %zmm0 {%k1} {z} ; AVX512VL-NEXT: vmovdqa64 %zmm0, (%rsp) -; AVX512VL-NEXT: kshiftrw $8, %k1, %k0 +; AVX512VL-NEXT: kshiftrq $8, %k1, %k0 ; AVX512VL-NEXT: kxorw %k0, %k1, %k0 ; AVX512VL-NEXT: kshiftrw $4, %k0, %k5 ; AVX512VL-NEXT: kxorw %k5, %k0, %k0 @@ -859,7 +859,7 @@ define <64 x i32> @test_compress_large(<64 x i1> %mask, <64 x i32> %vec, <64 x i ; AVX512VL-NEXT: vmovdqa64 %zmm0, (%rsp,%rax,4) ; AVX512VL-NEXT: vpcompressd %zmm3, %zmm0 {%k4} {z} ; AVX512VL-NEXT: vmovdqa64 %zmm0, {{[0-9]+}}(%rsp) -; AVX512VL-NEXT: kshiftrw $8, %k4, %k0 +; AVX512VL-NEXT: kshiftrq $40, %k1, %k0 ; AVX512VL-NEXT: kxorw %k0, %k4, %k0 ; AVX512VL-NEXT: kshiftrw $4, %k0, %k4 ; AVX512VL-NEXT: kxorw %k4, %k0, %k0 diff --git a/llvm/test/CodeGen/X86/vector-fshl-sub128.ll b/llvm/test/CodeGen/X86/vector-fshl-sub128.ll index d8e45ed9151d..eb4d84b8d7dd 100644 --- a/llvm/test/CodeGen/X86/vector-fshl-sub128.ll +++ b/llvm/test/CodeGen/X86/vector-fshl-sub128.ll @@ -337,7 +337,7 @@ define <2 x i32> @constant_funnnel_v2i32(<2 x i32> %x, <2 x i32> %y) nounwind { ; SSE41-NEXT: movdqa %xmm1, %xmm2 ; SSE41-NEXT: psrld $27, %xmm2 ; SSE41-NEXT: psrld $28, %xmm1 -; SSE41-NEXT: pblendw {{.*#+}} xmm2 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7] +; SSE41-NEXT: pblendw {{.*#+}} xmm2 = xmm1[0,1],xmm2[2,3],xmm1[4,5,6,7] ; SSE41-NEXT: pmulld {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 ; SSE41-NEXT: por %xmm2, %xmm0 ; SSE41-NEXT: retq @@ -346,7 +346,7 @@ define <2 x i32> @constant_funnnel_v2i32(<2 x i32> %x, <2 x i32> %y) nounwind { ; AVX1: # %bb.0: ; AVX1-NEXT: vpsrld $27, %xmm1, %xmm2 ; AVX1-NEXT: vpsrld $28, %xmm1, %xmm1 -; AVX1-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7] +; AVX1-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5,6,7] ; AVX1-NEXT: vpmulld {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 ; AVX1-NEXT: vpor %xmm1, %xmm0, %xmm0 ; AVX1-NEXT: retq diff --git a/llvm/test/CodeGen/X86/vector-fshr-sub128.ll b/llvm/test/CodeGen/X86/vector-fshr-sub128.ll index a6067a960fc0..58dc17988b64 100644 --- a/llvm/test/CodeGen/X86/vector-fshr-sub128.ll +++ b/llvm/test/CodeGen/X86/vector-fshr-sub128.ll @@ -379,16 +379,11 @@ define <2 x i32> @constant_funnnel_v2i32(<2 x i32> %x, <2 x i32> %y) nounwind { ; SSE2-NEXT: psrld $4, %xmm3 ; SSE2-NEXT: punpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm2[0] ; SSE2-NEXT: shufps {{.*#+}} xmm3 = xmm3[0,3],xmm1[2,3] -; SSE2-NEXT: movl $268435456, %eax # imm = 0x10000000 -; SSE2-NEXT: movd %eax, %xmm1 -; SSE2-NEXT: pmuludq %xmm0, %xmm1 -; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3] -; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] -; SSE2-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 -; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] -; SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1] -; SSE2-NEXT: por %xmm3, %xmm1 -; SSE2-NEXT: movdqa %xmm1, %xmm0 +; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1] +; SSE2-NEXT: pslld $28, %xmm0 +; SSE2-NEXT: pslld $27, %xmm1 +; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] +; SSE2-NEXT: por %xmm3, %xmm0 ; SSE2-NEXT: retq ; ; SSE41-LABEL: constant_funnnel_v2i32: @@ -400,7 +395,10 @@ define <2 x i32> @constant_funnnel_v2i32(<2 x i32> %x, <2 x i32> %y) nounwind { ; SSE41-NEXT: psrld $4, %xmm3 ; SSE41-NEXT: pblendw {{.*#+}} xmm3 = xmm3[0,1,2,3],xmm1[4,5,6,7] ; SSE41-NEXT: pblendw {{.*#+}} xmm3 = xmm3[0,1],xmm2[2,3],xmm3[4,5],xmm2[6,7] -; SSE41-NEXT: pmulld {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 +; SSE41-NEXT: movdqa %xmm0, %xmm1 +; SSE41-NEXT: pslld $27, %xmm1 +; SSE41-NEXT: pslld $28, %xmm0 +; SSE41-NEXT: pblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3],xmm0[4,5,6,7] ; SSE41-NEXT: por %xmm3, %xmm0 ; SSE41-NEXT: retq ; @@ -411,7 +409,9 @@ define <2 x i32> @constant_funnnel_v2i32(<2 x i32> %x, <2 x i32> %y) nounwind { ; AVX1-NEXT: vpsrld $4, %xmm1, %xmm3 ; AVX1-NEXT: vpblendw {{.*#+}} xmm1 = xmm3[0,1,2,3],xmm1[4,5,6,7] ; AVX1-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7] -; AVX1-NEXT: vpmulld {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 +; AVX1-NEXT: vpslld $27, %xmm0, %xmm2 +; AVX1-NEXT: vpslld $28, %xmm0, %xmm0 +; AVX1-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3],xmm0[4,5,6,7] ; AVX1-NEXT: vpor %xmm1, %xmm0, %xmm0 ; AVX1-NEXT: retq ; @@ -482,22 +482,17 @@ define <2 x i32> @constant_funnnel_v2i32(<2 x i32> %x, <2 x i32> %y) nounwind { ; ; X86-SSE2-LABEL: constant_funnnel_v2i32: ; X86-SSE2: # %bb.0: -; X86-SSE2-NEXT: movdqa %xmm1, %xmm3 -; X86-SSE2-NEXT: psrld $5, %xmm3 ; X86-SSE2-NEXT: movdqa %xmm1, %xmm2 -; X86-SSE2-NEXT: psrld $4, %xmm2 -; X86-SSE2-NEXT: punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0] -; X86-SSE2-NEXT: shufps {{.*#+}} xmm2 = xmm2[0,3],xmm1[2,3] -; X86-SSE2-NEXT: movl $268435456, %eax # imm = 0x10000000 -; X86-SSE2-NEXT: movd %eax, %xmm1 -; X86-SSE2-NEXT: pmuludq %xmm0, %xmm1 -; X86-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3] -; X86-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] -; X86-SSE2-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0 -; X86-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] -; X86-SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1] -; X86-SSE2-NEXT: por %xmm2, %xmm1 -; X86-SSE2-NEXT: movdqa %xmm1, %xmm0 +; X86-SSE2-NEXT: psrld $5, %xmm2 +; X86-SSE2-NEXT: movdqa %xmm1, %xmm3 +; X86-SSE2-NEXT: psrld $4, %xmm3 +; X86-SSE2-NEXT: punpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm2[0] +; X86-SSE2-NEXT: shufps {{.*#+}} xmm3 = xmm3[0,3],xmm1[2,3] +; X86-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1] +; X86-SSE2-NEXT: pslld $28, %xmm0 +; X86-SSE2-NEXT: pslld $27, %xmm1 +; X86-SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] +; X86-SSE2-NEXT: por %xmm3, %xmm0 ; X86-SSE2-NEXT: retl %res = call <2 x i32> @llvm.fshr.v2i32(<2 x i32> %x, <2 x i32> %y, <2 x i32> <i32 4, i32 5>) ret <2 x i32> %res diff --git a/llvm/test/CodeGen/X86/vector-interleaved-load-i32-stride-7.ll b/llvm/test/CodeGen/X86/vector-interleaved-load-i32-stride-7.ll index e03e19fc6d16..d806253ef23a 100644 --- a/llvm/test/CodeGen/X86/vector-interleaved-load-i32-stride-7.ll +++ b/llvm/test/CodeGen/X86/vector-interleaved-load-i32-stride-7.ll @@ -69,11 +69,11 @@ define void @load_i32_stride7_vf2(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr ; AVX-NEXT: vshufps {{.*#+}} xmm4 = xmm4[2,3,2,3] ; AVX-NEXT: vblendps {{.*#+}} xmm3 = xmm3[0],xmm4[1],xmm3[2,3] ; AVX-NEXT: vshufps {{.*#+}} ymm4 = ymm1[0,0],ymm0[1,0],ymm1[4,4],ymm0[5,4] -; AVX-NEXT: vshufps {{.*#+}} ymm4 = ymm4[2,0,2,3,6,4,6,7] ; AVX-NEXT: vextractf128 $1, %ymm4, %xmm4 +; AVX-NEXT: vshufps {{.*#+}} xmm4 = xmm4[2,0,2,3] ; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm1[1,0],ymm0[2,0],ymm1[5,4],ymm0[6,4] -; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm0[2,0,2,3,6,4,6,7] ; AVX-NEXT: vextractf128 $1, %ymm0, %xmm0 +; AVX-NEXT: vshufps {{.*#+}} xmm0 = xmm0[2,0,2,3] ; AVX-NEXT: vmovlps %xmm5, (%rsi) ; AVX-NEXT: vmovlps %xmm6, (%rdx) ; AVX-NEXT: vmovlps %xmm7, (%rcx) @@ -104,8 +104,8 @@ define void @load_i32_stride7_vf2(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr ; AVX2-NEXT: vblendps {{.*#+}} ymm7 = ymm1[0,1,2,3],ymm0[4,5,6,7] ; AVX2-NEXT: vpermps %ymm7, %ymm4, %ymm4 ; AVX2-NEXT: vblendps {{.*#+}} ymm7 = ymm1[0],ymm0[1],ymm1[2,3,4],ymm0[5],ymm1[6,7] -; AVX2-NEXT: vshufps {{.*#+}} ymm7 = ymm7[1,0,2,3,5,4,6,7] ; AVX2-NEXT: vextractf128 $1, %ymm7, %xmm7 +; AVX2-NEXT: vshufps {{.*#+}} xmm7 = xmm7[1,0,2,3] ; AVX2-NEXT: vshufps {{.*#+}} ymm0 = ymm0[2,3,2,3,6,7,6,7] ; AVX2-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2,3,4],ymm1[5],ymm0[6,7] ; AVX2-NEXT: vextractf128 $1, %ymm0, %xmm0 @@ -139,8 +139,8 @@ define void @load_i32_stride7_vf2(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr ; AVX2-FP-NEXT: vblendps {{.*#+}} ymm7 = ymm1[0,1,2,3],ymm0[4,5,6,7] ; AVX2-FP-NEXT: vpermps %ymm7, %ymm4, %ymm4 ; AVX2-FP-NEXT: vblendps {{.*#+}} ymm7 = ymm1[0],ymm0[1],ymm1[2,3,4],ymm0[5],ymm1[6,7] -; AVX2-FP-NEXT: vshufps {{.*#+}} ymm7 = ymm7[1,0,2,3,5,4,6,7] ; AVX2-FP-NEXT: vextractf128 $1, %ymm7, %xmm7 +; AVX2-FP-NEXT: vshufps {{.*#+}} xmm7 = xmm7[1,0,2,3] ; AVX2-FP-NEXT: vshufps {{.*#+}} ymm0 = ymm0[2,3,2,3,6,7,6,7] ; AVX2-FP-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2,3,4],ymm1[5],ymm0[6,7] ; AVX2-FP-NEXT: vextractf128 $1, %ymm0, %xmm0 @@ -174,8 +174,8 @@ define void @load_i32_stride7_vf2(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr ; AVX2-FCP-NEXT: vblendps {{.*#+}} ymm7 = ymm1[0,1,2,3],ymm0[4,5,6,7] ; AVX2-FCP-NEXT: vpermps %ymm7, %ymm4, %ymm4 ; AVX2-FCP-NEXT: vblendps {{.*#+}} ymm7 = ymm1[0],ymm0[1],ymm1[2,3,4],ymm0[5],ymm1[6,7] -; AVX2-FCP-NEXT: vshufps {{.*#+}} ymm7 = ymm7[1,0,2,3,5,4,6,7] ; AVX2-FCP-NEXT: vextractf128 $1, %ymm7, %xmm7 +; AVX2-FCP-NEXT: vshufps {{.*#+}} xmm7 = xmm7[1,0,2,3] ; AVX2-FCP-NEXT: vshufps {{.*#+}} ymm0 = ymm0[2,3,2,3,6,7,6,7] ; AVX2-FCP-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2,3,4],ymm1[5],ymm0[6,7] ; AVX2-FCP-NEXT: vextractf128 $1, %ymm0, %xmm0 @@ -208,8 +208,8 @@ define void @load_i32_stride7_vf2(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr ; AVX512-NEXT: vmovdqa (%rdi), %ymm6 ; AVX512-NEXT: vpermi2d %ymm5, %ymm6, %ymm1 ; AVX512-NEXT: vpblendd {{.*#+}} ymm7 = ymm5[0],ymm6[1],ymm5[2,3,4],ymm6[5],ymm5[6,7] -; AVX512-NEXT: vpshufd {{.*#+}} ymm7 = ymm7[1,0,2,3,5,4,6,7] ; AVX512-NEXT: vextracti128 $1, %ymm7, %xmm7 +; AVX512-NEXT: vpshufd {{.*#+}} xmm7 = xmm7[1,0,2,3] ; AVX512-NEXT: vpshufd {{.*#+}} ymm6 = ymm6[2,3,2,3,6,7,6,7] ; AVX512-NEXT: vpblendd {{.*#+}} ymm5 = ymm6[0],ymm5[1],ymm6[2,3,4],ymm5[5],ymm6[6,7] ; AVX512-NEXT: vextracti128 $1, %ymm5, %xmm5 @@ -273,8 +273,8 @@ define void @load_i32_stride7_vf2(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr ; AVX512DQ-NEXT: vmovdqa (%rdi), %ymm6 ; AVX512DQ-NEXT: vpermi2d %ymm5, %ymm6, %ymm1 ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm7 = ymm5[0],ymm6[1],ymm5[2,3,4],ymm6[5],ymm5[6,7] -; AVX512DQ-NEXT: vpshufd {{.*#+}} ymm7 = ymm7[1,0,2,3,5,4,6,7] ; AVX512DQ-NEXT: vextracti128 $1, %ymm7, %xmm7 +; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm7 = xmm7[1,0,2,3] ; AVX512DQ-NEXT: vpshufd {{.*#+}} ymm6 = ymm6[2,3,2,3,6,7,6,7] ; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm5 = ymm6[0],ymm5[1],ymm6[2,3,4],ymm5[5],ymm6[6,7] ; AVX512DQ-NEXT: vextracti128 $1, %ymm5, %xmm5 @@ -338,8 +338,8 @@ define void @load_i32_stride7_vf2(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr ; AVX512BW-NEXT: vmovdqa (%rdi), %ymm6 ; AVX512BW-NEXT: vpermi2d %ymm5, %ymm6, %ymm1 ; AVX512BW-NEXT: vpblendd {{.*#+}} ymm7 = ymm5[0],ymm6[1],ymm5[2,3,4],ymm6[5],ymm5[6,7] -; AVX512BW-NEXT: vpshufd {{.*#+}} ymm7 = ymm7[1,0,2,3,5,4,6,7] ; AVX512BW-NEXT: vextracti128 $1, %ymm7, %xmm7 +; AVX512BW-NEXT: vpshufd {{.*#+}} xmm7 = xmm7[1,0,2,3] ; AVX512BW-NEXT: vpshufd {{.*#+}} ymm6 = ymm6[2,3,2,3,6,7,6,7] ; AVX512BW-NEXT: vpblendd {{.*#+}} ymm5 = ymm6[0],ymm5[1],ymm6[2,3,4],ymm5[5],ymm6[6,7] ; AVX512BW-NEXT: vextracti128 $1, %ymm5, %xmm5 @@ -403,8 +403,8 @@ define void @load_i32_stride7_vf2(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr ; AVX512DQ-BW-NEXT: vmovdqa (%rdi), %ymm6 ; AVX512DQ-BW-NEXT: vpermi2d %ymm5, %ymm6, %ymm1 ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm7 = ymm5[0],ymm6[1],ymm5[2,3,4],ymm6[5],ymm5[6,7] -; AVX512DQ-BW-NEXT: vpshufd {{.*#+}} ymm7 = ymm7[1,0,2,3,5,4,6,7] ; AVX512DQ-BW-NEXT: vextracti128 $1, %ymm7, %xmm7 +; AVX512DQ-BW-NEXT: vpshufd {{.*#+}} xmm7 = xmm7[1,0,2,3] ; AVX512DQ-BW-NEXT: vpshufd {{.*#+}} ymm6 = ymm6[2,3,2,3,6,7,6,7] ; AVX512DQ-BW-NEXT: vpblendd {{.*#+}} ymm5 = ymm6[0],ymm5[1],ymm6[2,3,4],ymm5[5],ymm6[6,7] ; AVX512DQ-BW-NEXT: vextracti128 $1, %ymm5, %xmm5 @@ -562,15 +562,13 @@ define void @load_i32_stride7_vf4(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr ; AVX-NEXT: vblendps {{.*#+}} xmm4 = xmm4[0,1],xmm10[2,3] ; AVX-NEXT: vblendps {{.*#+}} xmm5 = xmm9[0,1,2],xmm5[3] ; AVX-NEXT: vshufps {{.*#+}} ymm10 = ymm0[0,0],ymm1[1,0],ymm0[4,4],ymm1[5,4] -; AVX-NEXT: vshufps {{.*#+}} ymm10 = ymm10[2,0,2,3,6,4,6,7] ; AVX-NEXT: vextractf128 $1, %ymm10, %xmm10 -; AVX-NEXT: vshufps {{.*#+}} xmm5 = xmm10[0,1],xmm5[3,2] +; AVX-NEXT: vshufps {{.*#+}} xmm5 = xmm10[2,0],xmm5[3,2] ; AVX-NEXT: vshufps {{.*#+}} xmm6 = xmm6[0,1,0,1] ; AVX-NEXT: vblendps {{.*#+}} xmm6 = xmm6[0,1,2],xmm9[3] ; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm0[1,0],ymm1[2,0],ymm0[5,4],ymm1[6,4] -; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm0[2,0,2,3,6,4,6,7] ; AVX-NEXT: vextractf128 $1, %ymm0, %xmm0 -; AVX-NEXT: vblendps {{.*#+}} xmm0 = xmm0[0,1],xmm6[2,3] +; AVX-NEXT: vshufps {{.*#+}} xmm0 = xmm0[2,0],xmm6[2,3] ; AVX-NEXT: vmovaps %xmm2, (%rsi) ; AVX-NEXT: vmovaps %xmm7, (%rdx) ; AVX-NEXT: vmovaps %xmm8, (%rcx) @@ -1154,8 +1152,8 @@ define void @load_i32_stride7_vf8(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr ; AVX-LABEL: load_i32_stride7_vf8: ; AVX: # %bb.0: ; AVX-NEXT: vmovaps 160(%rdi), %ymm4 -; AVX-NEXT: vmovaps 128(%rdi), %ymm8 -; AVX-NEXT: vmovaps 64(%rdi), %ymm11 +; AVX-NEXT: vmovaps 128(%rdi), %ymm7 +; AVX-NEXT: vmovaps 64(%rdi), %ymm10 ; AVX-NEXT: vmovaps 32(%rdi), %ymm0 ; AVX-NEXT: vmovaps (%rdi), %ymm1 ; AVX-NEXT: vmovaps 96(%rdi), %ymm12 @@ -1163,88 +1161,86 @@ define void @load_i32_stride7_vf8(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr ; AVX-NEXT: vunpcklpd {{.*#+}} ymm3 = ymm12[0],ymm2[0],ymm12[2],ymm2[2] ; AVX-NEXT: vblendps {{.*#+}} ymm5 = ymm1[0,1,2,3,4,5],ymm0[6],ymm1[7] ; AVX-NEXT: vextractf128 $1, %ymm5, %xmm5 -; AVX-NEXT: vmovaps (%rdi), %xmm13 +; AVX-NEXT: vmovaps (%rdi), %xmm14 ; AVX-NEXT: vmovaps 32(%rdi), %xmm9 -; AVX-NEXT: vblendps {{.*#+}} xmm5 = xmm13[0,1],xmm5[2,3] +; AVX-NEXT: vblendps {{.*#+}} xmm5 = xmm14[0,1],xmm5[2,3] ; AVX-NEXT: vshufps {{.*#+}} xmm5 = xmm5[0,3,2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm3 = ymm5[0,1,2],ymm3[3,4,5,6,7] -; AVX-NEXT: vmovaps 160(%rdi), %xmm5 -; AVX-NEXT: vmovaps 128(%rdi), %xmm6 -; AVX-NEXT: vunpckhpd {{.*#+}} xmm7 = xmm6[1],xmm5[1] -; AVX-NEXT: vmovaps 192(%rdi), %xmm10 -; AVX-NEXT: vinsertps {{.*#+}} xmm7 = zero,xmm7[1,2],xmm10[1] -; AVX-NEXT: vinsertf128 $1, %xmm7, %ymm0, %ymm7 -; AVX-NEXT: vblendps {{.*#+}} ymm3 = ymm3[0,1,2,3,4],ymm7[5,6,7] -; AVX-NEXT: vshufps {{.*#+}} ymm7 = ymm12[1,1],ymm11[2,2],ymm12[5,5],ymm11[6,6] -; AVX-NEXT: vperm2f128 {{.*#+}} ymm7 = ymm7[2,3,2,3] -; AVX-NEXT: vblendps {{.*#+}} xmm14 = xmm9[0],xmm13[1],xmm9[2,3] -; AVX-NEXT: vshufps {{.*#+}} xmm14 = xmm14[1,0],mem[3,3] -; AVX-NEXT: vblendps {{.*#+}} ymm7 = ymm14[0,1,2],ymm7[3,4,5,6,7] -; AVX-NEXT: vperm2f128 {{.*#+}} ymm14 = ymm8[2,3],ymm4[0,1] -; AVX-NEXT: vshufps {{.*#+}} ymm14 = ymm8[0,0],ymm14[3,3],ymm8[4,4],ymm14[7,7] -; AVX-NEXT: vextractf128 $1, %ymm14, %xmm14 -; AVX-NEXT: vinsertps {{.*#+}} xmm14 = zero,xmm14[1,2],xmm10[2] -; AVX-NEXT: vinsertf128 $1, %xmm14, %ymm0, %ymm14 -; AVX-NEXT: vblendps {{.*#+}} ymm7 = ymm7[0,1,2,3,4],ymm14[5,6,7] -; AVX-NEXT: vshufps {{.*#+}} xmm14 = xmm13[2,3,2,3] -; AVX-NEXT: vblendps {{.*#+}} xmm14 = xmm14[0],xmm9[1],xmm14[2,3] -; AVX-NEXT: vshufps {{.*#+}} ymm15 = ymm2[3,1],ymm11[0,3],ymm2[7,5],ymm11[4,7] +; AVX-NEXT: vblendps {{.*#+}} ymm6 = ymm5[0,1,2],ymm3[3,4,5,6,7] +; AVX-NEXT: vmovaps 160(%rdi), %xmm3 +; AVX-NEXT: vmovaps 128(%rdi), %xmm5 +; AVX-NEXT: vunpckhpd {{.*#+}} xmm8 = xmm5[1],xmm3[1] +; AVX-NEXT: vmovaps 192(%rdi), %xmm11 +; AVX-NEXT: vinsertps {{.*#+}} xmm8 = zero,xmm8[1,2],xmm11[1] +; AVX-NEXT: vinsertf128 $1, %xmm8, %ymm0, %ymm8 +; AVX-NEXT: vblendps {{.*#+}} ymm6 = ymm6[0,1,2,3,4],ymm8[5,6,7] +; AVX-NEXT: vshufps {{.*#+}} ymm8 = ymm12[1,1],ymm10[2,2],ymm12[5,5],ymm10[6,6] +; AVX-NEXT: vperm2f128 {{.*#+}} ymm8 = ymm8[2,3,2,3] +; AVX-NEXT: vblendps {{.*#+}} xmm13 = xmm9[0],xmm14[1],xmm9[2,3] +; AVX-NEXT: vshufps {{.*#+}} xmm13 = xmm13[1,0],mem[3,3] +; AVX-NEXT: vblendps {{.*#+}} ymm8 = ymm13[0,1,2],ymm8[3,4,5,6,7] +; AVX-NEXT: vperm2f128 {{.*#+}} ymm13 = ymm7[2,3],ymm4[0,1] +; AVX-NEXT: vshufps {{.*#+}} ymm13 = ymm7[0,0],ymm13[3,3],ymm7[4,4],ymm13[7,7] +; AVX-NEXT: vextractf128 $1, %ymm13, %xmm13 +; AVX-NEXT: vinsertps {{.*#+}} xmm13 = zero,xmm13[1,2],xmm11[2] +; AVX-NEXT: vinsertf128 $1, %xmm13, %ymm0, %ymm13 +; AVX-NEXT: vblendps {{.*#+}} ymm8 = ymm8[0,1,2,3,4],ymm13[5,6,7] +; AVX-NEXT: vshufps {{.*#+}} xmm13 = xmm14[2,3,2,3] +; AVX-NEXT: vblendps {{.*#+}} xmm13 = xmm13[0],xmm9[1],xmm13[2,3] +; AVX-NEXT: vshufps {{.*#+}} ymm15 = ymm2[3,1],ymm10[0,3],ymm2[7,5],ymm10[4,7] ; AVX-NEXT: vshufps {{.*#+}} ymm15 = ymm12[2,1],ymm15[2,0],ymm12[6,5],ymm15[6,4] -; AVX-NEXT: vblendps {{.*#+}} ymm15 = ymm14[0,1],ymm15[2,3,4,5,6,7] -; AVX-NEXT: vunpcklpd {{.*#+}} ymm14 = ymm8[0],ymm4[0],ymm8[2],ymm4[2] -; AVX-NEXT: vextractf128 $1, %ymm14, %xmm14 -; AVX-NEXT: vblendps {{.*#+}} xmm10 = xmm14[0,1,2],xmm10[3] -; AVX-NEXT: vmovaps 192(%rdi), %ymm14 -; AVX-NEXT: vinsertf128 $1, %xmm10, %ymm0, %ymm10 -; AVX-NEXT: vblendps {{.*#+}} ymm10 = ymm15[0,1,2,3,4],ymm10[5,6,7] +; AVX-NEXT: vblendps {{.*#+}} ymm15 = ymm13[0,1],ymm15[2,3,4,5,6,7] +; AVX-NEXT: vunpcklpd {{.*#+}} ymm13 = ymm7[0],ymm4[0],ymm7[2],ymm4[2] +; AVX-NEXT: vextractf128 $1, %ymm13, %xmm13 +; AVX-NEXT: vblendps {{.*#+}} xmm11 = xmm13[0,1,2],xmm11[3] +; AVX-NEXT: vmovaps 192(%rdi), %ymm13 +; AVX-NEXT: vinsertf128 $1, %xmm11, %ymm0, %ymm11 +; AVX-NEXT: vblendps {{.*#+}} ymm11 = ymm15[0,1,2,3,4],ymm11[5,6,7] ; AVX-NEXT: vmovaps 64(%rdi), %xmm15 -; AVX-NEXT: vshufps {{.*#+}} ymm11 = ymm11[1,0],ymm12[0,0],ymm11[5,4],ymm12[4,4] -; AVX-NEXT: vshufps {{.*#+}} ymm11 = ymm12[3,1],ymm11[0,2],ymm12[7,5],ymm11[4,6] -; AVX-NEXT: vblendps {{.*#+}} xmm12 = xmm9[0,1,2],xmm13[3] +; AVX-NEXT: vshufps {{.*#+}} ymm10 = ymm10[1,0],ymm12[0,0],ymm10[5,4],ymm12[4,4] +; AVX-NEXT: vshufps {{.*#+}} ymm10 = ymm12[3,1],ymm10[0,2],ymm12[7,5],ymm10[4,6] +; AVX-NEXT: vblendps {{.*#+}} xmm12 = xmm9[0,1,2],xmm14[3] ; AVX-NEXT: vshufps {{.*#+}} xmm12 = xmm12[3,2,2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm11 = ymm12[0,1],ymm11[2,3,4,5,6,7] -; AVX-NEXT: vshufps {{.*#+}} ymm12 = ymm14[0,1],ymm4[1,3],ymm14[4,5],ymm4[5,7] -; AVX-NEXT: vshufps {{.*#+}} ymm12 = ymm8[0,2],ymm12[2,0],ymm8[4,6],ymm12[6,4] -; AVX-NEXT: vblendps {{.*#+}} ymm11 = ymm11[0,1,2,3,4],ymm12[5,6,7] -; AVX-NEXT: vshufps {{.*#+}} ymm12 = ymm14[1,0],ymm4[2,0],ymm14[5,4],ymm4[6,4] -; AVX-NEXT: vperm2f128 {{.*#+}} ymm13 = ymm8[2,3,0,1] -; AVX-NEXT: vshufps {{.*#+}} ymm8 = ymm8[3,0],ymm13[0,0],ymm8[7,4],ymm13[4,4] -; AVX-NEXT: vshufps {{.*#+}} ymm8 = ymm8[2,0],ymm12[2,0],ymm8[6,4],ymm12[6,4] +; AVX-NEXT: vblendps {{.*#+}} ymm10 = ymm12[0,1],ymm10[2,3,4,5,6,7] +; AVX-NEXT: vshufps {{.*#+}} ymm12 = ymm13[0,1],ymm4[1,3],ymm13[4,5],ymm4[5,7] +; AVX-NEXT: vshufps {{.*#+}} ymm12 = ymm7[0,2],ymm12[2,0],ymm7[4,6],ymm12[6,4] +; AVX-NEXT: vblendps {{.*#+}} ymm10 = ymm10[0,1,2,3,4],ymm12[5,6,7] +; AVX-NEXT: vshufps {{.*#+}} ymm12 = ymm13[1,0],ymm4[2,0],ymm13[5,4],ymm4[6,4] +; AVX-NEXT: vperm2f128 {{.*#+}} ymm14 = ymm7[2,3,0,1] +; AVX-NEXT: vshufps {{.*#+}} ymm7 = ymm7[3,0],ymm14[0,0],ymm7[7,4],ymm14[4,4] +; AVX-NEXT: vshufps {{.*#+}} ymm7 = ymm7[2,0],ymm12[2,0],ymm7[6,4],ymm12[6,4] ; AVX-NEXT: vmovaps 96(%rdi), %xmm12 -; AVX-NEXT: vshufps {{.*#+}} xmm13 = xmm12[0,1,0,1] -; AVX-NEXT: vblendps {{.*#+}} xmm13 = xmm15[0,1,2],xmm13[3] +; AVX-NEXT: vshufps {{.*#+}} xmm14 = xmm12[0,1,0,1] +; AVX-NEXT: vblendps {{.*#+}} xmm14 = xmm15[0,1,2],xmm14[3] ; AVX-NEXT: vshufps {{.*#+}} xmm9 = xmm9[2,3,2,3] ; AVX-NEXT: vblendps {{.*#+}} xmm9 = mem[0],xmm9[1],mem[2,3] -; AVX-NEXT: vblendps {{.*#+}} xmm9 = xmm9[0,1],xmm13[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm8 = ymm9[0,1,2,3],ymm8[4,5,6,7] -; AVX-NEXT: vshufps {{.*#+}} ymm4 = ymm14[2,1],ymm4[3,3],ymm14[6,5],ymm4[7,7] -; AVX-NEXT: vblendps {{.*#+}} xmm9 = xmm5[0],xmm6[1],xmm5[2,3] +; AVX-NEXT: vblendps {{.*#+}} xmm9 = xmm9[0,1],xmm14[2,3] +; AVX-NEXT: vblendps {{.*#+}} ymm7 = ymm9[0,1,2,3],ymm7[4,5,6,7] +; AVX-NEXT: vshufps {{.*#+}} ymm4 = ymm13[2,1],ymm4[3,3],ymm13[6,5],ymm4[7,7] +; AVX-NEXT: vblendps {{.*#+}} xmm9 = xmm3[0],xmm5[1],xmm3[2,3] ; AVX-NEXT: vinsertf128 $1, %xmm9, %ymm0, %ymm9 ; AVX-NEXT: vshufps {{.*#+}} ymm4 = ymm9[1,0],ymm4[2,0],ymm9[5,4],ymm4[6,4] ; AVX-NEXT: vblendps {{.*#+}} xmm9 = xmm12[0,1,2],xmm15[3] -; AVX-NEXT: vshufps {{.*#+}} ymm13 = ymm0[0,0],ymm1[1,0],ymm0[4,4],ymm1[5,4] -; AVX-NEXT: vshufps {{.*#+}} ymm13 = ymm13[2,0,2,3,6,4,6,7] -; AVX-NEXT: vextractf128 $1, %ymm13, %xmm13 -; AVX-NEXT: vshufps {{.*#+}} xmm9 = xmm13[0,1],xmm9[3,2] +; AVX-NEXT: vshufps {{.*#+}} ymm14 = ymm0[0,0],ymm1[1,0],ymm0[4,4],ymm1[5,4] +; AVX-NEXT: vextractf128 $1, %ymm14, %xmm14 +; AVX-NEXT: vshufps {{.*#+}} xmm9 = xmm14[2,0],xmm9[3,2] ; AVX-NEXT: vblendps {{.*#+}} ymm4 = ymm9[0,1,2,3],ymm4[4,5,6,7] -; AVX-NEXT: vperm2f128 {{.*#+}} ymm9 = ymm14[2,3,0,1] -; AVX-NEXT: vshufps {{.*#+}} ymm9 = ymm14[3,0],ymm9[0,0],ymm14[7,4],ymm9[4,4] -; AVX-NEXT: vshufps {{.*#+}} xmm6 = xmm6[2,3,2,3] -; AVX-NEXT: vblendps {{.*#+}} xmm5 = xmm6[0],xmm5[1],xmm6[2,3] -; AVX-NEXT: vinsertf128 $1, %xmm5, %ymm0, %ymm5 -; AVX-NEXT: vshufps {{.*#+}} ymm5 = ymm5[0,1],ymm9[2,0],ymm5[4,5],ymm9[6,4] ; AVX-NEXT: vshufps {{.*#+}} xmm2 = xmm2[0,1,0,1] ; AVX-NEXT: vblendps {{.*#+}} xmm2 = xmm2[0,1,2],xmm12[3] ; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm0[1,0],ymm1[2,0],ymm0[5,4],ymm1[6,4] -; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm0[2,0,2,3,6,4,6,7] ; AVX-NEXT: vextractf128 $1, %ymm0, %xmm0 -; AVX-NEXT: vblendps {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm5[4,5,6,7] -; AVX-NEXT: vmovaps %ymm3, (%rsi) -; AVX-NEXT: vmovaps %ymm7, (%rdx) -; AVX-NEXT: vmovaps %ymm10, (%rcx) -; AVX-NEXT: vmovaps %ymm11, (%r8) -; AVX-NEXT: vmovaps %ymm8, (%r9) +; AVX-NEXT: vshufps {{.*#+}} xmm0 = xmm0[2,0],xmm2[2,3] +; AVX-NEXT: vperm2f128 {{.*#+}} ymm1 = ymm13[2,3,0,1] +; AVX-NEXT: vshufps {{.*#+}} ymm1 = ymm13[3,0],ymm1[0,0],ymm13[7,4],ymm1[4,4] +; AVX-NEXT: vshufps {{.*#+}} xmm2 = xmm5[2,3,2,3] +; AVX-NEXT: vblendps {{.*#+}} xmm2 = xmm2[0],xmm3[1],xmm2[2,3] +; AVX-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm2 +; AVX-NEXT: vshufps {{.*#+}} ymm1 = ymm2[0,1],ymm1[2,0],ymm2[4,5],ymm1[6,4] +; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7] +; AVX-NEXT: vmovaps %ymm6, (%rsi) +; AVX-NEXT: vmovaps %ymm8, (%rdx) +; AVX-NEXT: vmovaps %ymm11, (%rcx) +; AVX-NEXT: vmovaps %ymm10, (%r8) +; AVX-NEXT: vmovaps %ymm7, (%r9) ; AVX-NEXT: movq {{[0-9]+}}(%rsp), %rax ; AVX-NEXT: vmovaps %ymm4, (%rax) ; AVX-NEXT: movq {{[0-9]+}}(%rsp), %rax @@ -2334,79 +2330,79 @@ define void @load_i32_stride7_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt ; AVX-LABEL: load_i32_stride7_vf16: ; AVX: # %bb.0: ; AVX-NEXT: subq $456, %rsp # imm = 0x1C8 -; AVX-NEXT: vmovaps 256(%rdi), %ymm4 +; AVX-NEXT: vmovaps 32(%rdi), %ymm4 ; AVX-NEXT: vmovups %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vmovaps 224(%rdi), %ymm5 -; AVX-NEXT: vmovups %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vmovaps 320(%rdi), %ymm15 -; AVX-NEXT: vmovaps 32(%rdi), %ymm2 +; AVX-NEXT: vmovaps (%rdi), %ymm6 +; AVX-NEXT: vmovups %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vmovaps 96(%rdi), %ymm15 +; AVX-NEXT: vmovaps 256(%rdi), %ymm2 ; AVX-NEXT: vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vmovaps (%rdi), %ymm1 +; AVX-NEXT: vmovaps 224(%rdi), %ymm1 ; AVX-NEXT: vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vmovaps 96(%rdi), %ymm7 -; AVX-NEXT: vmovaps 80(%rdi), %xmm0 +; AVX-NEXT: vmovaps 320(%rdi), %ymm5 +; AVX-NEXT: vmovaps 304(%rdi), %xmm0 ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vunpcklpd {{.*#+}} ymm0 = ymm7[0],ymm0[0],ymm7[2],ymm0[2] +; AVX-NEXT: vunpcklpd {{.*#+}} ymm0 = ymm5[0],ymm0[0],ymm5[2],ymm0[2] ; AVX-NEXT: vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5],ymm2[6],ymm1[7] ; AVX-NEXT: vextractf128 $1, %ymm1, %xmm1 -; AVX-NEXT: vmovaps (%rdi), %xmm13 +; AVX-NEXT: vmovaps 224(%rdi), %xmm13 ; AVX-NEXT: vblendps {{.*#+}} xmm1 = xmm13[0,1],xmm1[2,3] ; AVX-NEXT: vshufps {{.*#+}} xmm1 = xmm1[0,3,2,3] ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2],ymm0[3,4,5,6,7] -; AVX-NEXT: vmovaps 160(%rdi), %xmm2 +; AVX-NEXT: vmovaps 384(%rdi), %xmm2 ; AVX-NEXT: vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; AVX-NEXT: vmovaps 128(%rdi), %xmm1 +; AVX-NEXT: vmovaps 352(%rdi), %xmm1 ; AVX-NEXT: vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; AVX-NEXT: vunpckhpd {{.*#+}} xmm1 = xmm1[1],xmm2[1] -; AVX-NEXT: vmovaps 192(%rdi), %xmm12 +; AVX-NEXT: vmovaps 416(%rdi), %xmm12 ; AVX-NEXT: vinsertps {{.*#+}} xmm1 = zero,xmm1[1,2],xmm12[1] ; AVX-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm1 ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm1[5,6,7] ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vmovaps 304(%rdi), %xmm0 +; AVX-NEXT: vmovaps 80(%rdi), %xmm0 ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX-NEXT: vunpcklpd {{.*#+}} ymm0 = ymm15[0],ymm0[0],ymm15[2],ymm0[2] -; AVX-NEXT: vblendps {{.*#+}} ymm1 = ymm5[0,1,2,3,4,5],ymm4[6],ymm5[7] +; AVX-NEXT: vblendps {{.*#+}} ymm1 = ymm6[0,1,2,3,4,5],ymm4[6],ymm6[7] ; AVX-NEXT: vextractf128 $1, %ymm1, %xmm1 -; AVX-NEXT: vmovaps 224(%rdi), %xmm10 -; AVX-NEXT: vblendps {{.*#+}} xmm1 = xmm10[0,1],xmm1[2,3] +; AVX-NEXT: vmovaps (%rdi), %xmm9 +; AVX-NEXT: vblendps {{.*#+}} xmm1 = xmm9[0,1],xmm1[2,3] ; AVX-NEXT: vshufps {{.*#+}} xmm1 = xmm1[0,3,2,3] ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2],ymm0[3,4,5,6,7] -; AVX-NEXT: vmovaps 384(%rdi), %xmm2 +; AVX-NEXT: vmovaps 160(%rdi), %xmm2 ; AVX-NEXT: vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; AVX-NEXT: vmovaps 352(%rdi), %xmm1 +; AVX-NEXT: vmovaps 128(%rdi), %xmm1 ; AVX-NEXT: vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; AVX-NEXT: vunpckhpd {{.*#+}} xmm1 = xmm1[1],xmm2[1] -; AVX-NEXT: vmovaps 416(%rdi), %xmm8 +; AVX-NEXT: vmovaps 192(%rdi), %xmm8 ; AVX-NEXT: vinsertps {{.*#+}} xmm1 = zero,xmm1[1,2],xmm8[1] ; AVX-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm1 ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm1[5,6,7] ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vmovaps 64(%rdi), %ymm5 -; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm7[1,1],ymm5[2,2],ymm7[5,5],ymm5[6,6] +; AVX-NEXT: vmovaps 288(%rdi), %ymm6 +; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm5[1,1],ymm6[2,2],ymm5[5,5],ymm6[6,6] ; AVX-NEXT: vperm2f128 {{.*#+}} ymm0 = ymm0[2,3,2,3] -; AVX-NEXT: vmovaps 32(%rdi), %xmm11 +; AVX-NEXT: vmovaps 256(%rdi), %xmm11 ; AVX-NEXT: vblendps {{.*#+}} xmm1 = xmm11[0],xmm13[1],xmm11[2,3] ; AVX-NEXT: vshufps {{.*#+}} xmm1 = xmm1[1,0],mem[3,3] ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2],ymm0[3,4,5,6,7] -; AVX-NEXT: vmovaps 160(%rdi), %ymm6 -; AVX-NEXT: vmovaps 128(%rdi), %ymm1 -; AVX-NEXT: vperm2f128 {{.*#+}} ymm3 = ymm1[2,3],ymm6[0,1] +; AVX-NEXT: vmovaps 384(%rdi), %ymm7 +; AVX-NEXT: vmovaps 352(%rdi), %ymm1 +; AVX-NEXT: vperm2f128 {{.*#+}} ymm3 = ymm1[2,3],ymm7[0,1] ; AVX-NEXT: vshufps {{.*#+}} ymm3 = ymm1[0,0],ymm3[3,3],ymm1[4,4],ymm3[7,7] ; AVX-NEXT: vextractf128 $1, %ymm3, %xmm3 ; AVX-NEXT: vinsertps {{.*#+}} xmm3 = zero,xmm3[1,2],xmm12[2] ; AVX-NEXT: vinsertf128 $1, %xmm3, %ymm0, %ymm3 ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm3[5,6,7] ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vmovaps 288(%rdi), %ymm3 +; AVX-NEXT: vmovaps 64(%rdi), %ymm3 ; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm15[1,1],ymm3[2,2],ymm15[5,5],ymm3[6,6] ; AVX-NEXT: vperm2f128 {{.*#+}} ymm0 = ymm0[2,3,2,3] -; AVX-NEXT: vmovaps 256(%rdi), %xmm9 -; AVX-NEXT: vblendps {{.*#+}} xmm4 = xmm9[0],xmm10[1],xmm9[2,3] +; AVX-NEXT: vmovaps 32(%rdi), %xmm10 +; AVX-NEXT: vblendps {{.*#+}} xmm4 = xmm10[0],xmm9[1],xmm10[2,3] ; AVX-NEXT: vshufps {{.*#+}} xmm4 = xmm4[1,0],mem[3,3] ; AVX-NEXT: vblendps {{.*#+}} ymm2 = ymm4[0,1,2],ymm0[3,4,5,6,7] -; AVX-NEXT: vmovaps 384(%rdi), %ymm4 -; AVX-NEXT: vmovaps 352(%rdi), %ymm0 +; AVX-NEXT: vmovaps 160(%rdi), %ymm4 +; AVX-NEXT: vmovaps 128(%rdi), %ymm0 ; AVX-NEXT: vperm2f128 {{.*#+}} ymm14 = ymm0[2,3],ymm4[0,1] ; AVX-NEXT: vshufps {{.*#+}} ymm14 = ymm0[0,0],ymm14[3,3],ymm0[4,4],ymm14[7,7] ; AVX-NEXT: vextractf128 $1, %ymm14, %xmm14 @@ -2417,17 +2413,17 @@ define void @load_i32_stride7_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt ; AVX-NEXT: vshufps {{.*#+}} xmm2 = xmm13[2,3,2,3] ; AVX-NEXT: vblendps {{.*#+}} xmm2 = xmm2[0],xmm11[1],xmm2[2,3] ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm14 # 32-byte Reload -; AVX-NEXT: vshufps {{.*#+}} ymm14 = ymm14[3,1],ymm5[0,3],ymm14[7,5],ymm5[4,7] -; AVX-NEXT: vshufps {{.*#+}} ymm14 = ymm7[2,1],ymm14[2,0],ymm7[6,5],ymm14[6,4] +; AVX-NEXT: vshufps {{.*#+}} ymm14 = ymm14[3,1],ymm6[0,3],ymm14[7,5],ymm6[4,7] +; AVX-NEXT: vshufps {{.*#+}} ymm14 = ymm5[2,1],ymm14[2,0],ymm5[6,5],ymm14[6,4] ; AVX-NEXT: vblendps {{.*#+}} ymm2 = ymm2[0,1],ymm14[2,3,4,5,6,7] -; AVX-NEXT: vunpcklpd {{.*#+}} ymm14 = ymm1[0],ymm6[0],ymm1[2],ymm6[2] +; AVX-NEXT: vunpcklpd {{.*#+}} ymm14 = ymm1[0],ymm7[0],ymm1[2],ymm7[2] ; AVX-NEXT: vextractf128 $1, %ymm14, %xmm14 ; AVX-NEXT: vblendps {{.*#+}} xmm12 = xmm14[0,1,2],xmm12[3] ; AVX-NEXT: vinsertf128 $1, %xmm12, %ymm0, %ymm12 ; AVX-NEXT: vblendps {{.*#+}} ymm2 = ymm2[0,1,2,3,4],ymm12[5,6,7] ; AVX-NEXT: vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vshufps {{.*#+}} xmm2 = xmm10[2,3,2,3] -; AVX-NEXT: vblendps {{.*#+}} xmm2 = xmm2[0],xmm9[1],xmm2[2,3] +; AVX-NEXT: vshufps {{.*#+}} xmm2 = xmm9[2,3,2,3] +; AVX-NEXT: vblendps {{.*#+}} xmm2 = xmm2[0],xmm10[1],xmm2[2,3] ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm12 # 32-byte Reload ; AVX-NEXT: vshufps {{.*#+}} ymm12 = ymm12[3,1],ymm3[0,3],ymm12[7,5],ymm3[4,7] ; AVX-NEXT: vshufps {{.*#+}} ymm12 = ymm15[2,1],ymm12[2,0],ymm15[6,5],ymm12[6,4] @@ -2438,134 +2434,130 @@ define void @load_i32_stride7_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt ; AVX-NEXT: vinsertf128 $1, %xmm8, %ymm0, %ymm8 ; AVX-NEXT: vblendps {{.*#+}} ymm2 = ymm2[0,1,2,3,4],ymm8[5,6,7] ; AVX-NEXT: vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vshufps {{.*#+}} ymm2 = ymm5[1,0],ymm7[0,0],ymm5[5,4],ymm7[4,4] -; AVX-NEXT: vshufps {{.*#+}} ymm2 = ymm7[3,1],ymm2[0,2],ymm7[7,5],ymm2[4,6] +; AVX-NEXT: vshufps {{.*#+}} ymm2 = ymm6[1,0],ymm5[0,0],ymm6[5,4],ymm5[4,4] +; AVX-NEXT: vshufps {{.*#+}} ymm2 = ymm5[3,1],ymm2[0,2],ymm5[7,5],ymm2[4,6] ; AVX-NEXT: vblendps {{.*#+}} xmm5 = xmm11[0,1,2],xmm13[3] ; AVX-NEXT: vshufps {{.*#+}} xmm5 = xmm5[3,2,2,3] ; AVX-NEXT: vblendps {{.*#+}} ymm5 = ymm5[0,1],ymm2[2,3,4,5,6,7] -; AVX-NEXT: vmovaps 192(%rdi), %ymm2 -; AVX-NEXT: vshufps {{.*#+}} ymm7 = ymm2[0,1],ymm6[1,3],ymm2[4,5],ymm6[5,7] -; AVX-NEXT: vshufps {{.*#+}} ymm7 = ymm1[0,2],ymm7[2,0],ymm1[4,6],ymm7[6,4] -; AVX-NEXT: vblendps {{.*#+}} ymm5 = ymm5[0,1,2,3,4],ymm7[5,6,7] +; AVX-NEXT: vmovaps 416(%rdi), %ymm2 +; AVX-NEXT: vshufps {{.*#+}} ymm6 = ymm2[0,1],ymm7[1,3],ymm2[4,5],ymm7[5,7] +; AVX-NEXT: vshufps {{.*#+}} ymm6 = ymm1[0,2],ymm6[2,0],ymm1[4,6],ymm6[6,4] +; AVX-NEXT: vblendps {{.*#+}} ymm5 = ymm5[0,1,2,3,4],ymm6[5,6,7] ; AVX-NEXT: vmovups %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX-NEXT: vshufps {{.*#+}} ymm3 = ymm3[1,0],ymm15[0,0],ymm3[5,4],ymm15[4,4] ; AVX-NEXT: vshufps {{.*#+}} ymm3 = ymm15[3,1],ymm3[0,2],ymm15[7,5],ymm3[4,6] -; AVX-NEXT: vblendps {{.*#+}} xmm5 = xmm9[0,1,2],xmm10[3] +; AVX-NEXT: vblendps {{.*#+}} xmm5 = xmm10[0,1,2],xmm9[3] ; AVX-NEXT: vshufps {{.*#+}} xmm5 = xmm5[3,2,2,3] ; AVX-NEXT: vblendps {{.*#+}} ymm3 = ymm5[0,1],ymm3[2,3,4,5,6,7] -; AVX-NEXT: vmovaps 416(%rdi), %ymm5 -; AVX-NEXT: vshufps {{.*#+}} ymm8 = ymm5[0,1],ymm4[1,3],ymm5[4,5],ymm4[5,7] -; AVX-NEXT: vshufps {{.*#+}} ymm8 = ymm0[0,2],ymm8[2,0],ymm0[4,6],ymm8[6,4] -; AVX-NEXT: vblendps {{.*#+}} ymm3 = ymm3[0,1,2,3,4],ymm8[5,6,7] +; AVX-NEXT: vmovaps 192(%rdi), %ymm6 +; AVX-NEXT: vshufps {{.*#+}} ymm5 = ymm6[0,1],ymm4[1,3],ymm6[4,5],ymm4[5,7] +; AVX-NEXT: vshufps {{.*#+}} ymm5 = ymm0[0,2],ymm5[2,0],ymm0[4,6],ymm5[6,4] +; AVX-NEXT: vblendps {{.*#+}} ymm3 = ymm3[0,1,2,3,4],ymm5[5,6,7] ; AVX-NEXT: vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX-NEXT: vperm2f128 {{.*#+}} ymm3 = ymm1[2,3,0,1] ; AVX-NEXT: vshufps {{.*#+}} ymm1 = ymm1[3,0],ymm3[0,0],ymm1[7,4],ymm3[4,4] -; AVX-NEXT: vshufps {{.*#+}} ymm3 = ymm2[1,0],ymm6[2,0],ymm2[5,4],ymm6[6,4] +; AVX-NEXT: vshufps {{.*#+}} ymm3 = ymm2[1,0],ymm7[2,0],ymm2[5,4],ymm7[6,4] ; AVX-NEXT: vshufps {{.*#+}} ymm1 = ymm1[2,0],ymm3[2,0],ymm1[6,4],ymm3[6,4] -; AVX-NEXT: vmovaps 64(%rdi), %xmm3 -; AVX-NEXT: vmovaps 96(%rdi), %xmm10 -; AVX-NEXT: vshufps {{.*#+}} xmm8 = xmm10[0,1,0,1] -; AVX-NEXT: vblendps {{.*#+}} xmm8 = xmm3[0,1,2],xmm8[3] -; AVX-NEXT: vshufps {{.*#+}} xmm11 = xmm11[2,3,2,3] -; AVX-NEXT: vblendps {{.*#+}} xmm11 = mem[0],xmm11[1],mem[2,3] -; AVX-NEXT: vblendps {{.*#+}} xmm8 = xmm11[0,1],xmm8[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm1 = ymm8[0,1,2,3],ymm1[4,5,6,7] -; AVX-NEXT: vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vperm2f128 {{.*#+}} ymm8 = ymm0[2,3,0,1] -; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm0[3,0],ymm8[0,0],ymm0[7,4],ymm8[4,4] -; AVX-NEXT: vshufps {{.*#+}} ymm8 = ymm5[1,0],ymm4[2,0],ymm5[5,4],ymm4[6,4] -; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm0[2,0],ymm8[2,0],ymm0[6,4],ymm8[6,4] -; AVX-NEXT: vmovaps 320(%rdi), %xmm8 -; AVX-NEXT: vshufps {{.*#+}} xmm11 = xmm8[0,1,0,1] -; AVX-NEXT: vmovaps 288(%rdi), %xmm12 -; AVX-NEXT: vblendps {{.*#+}} xmm11 = xmm12[0,1,2],xmm11[3] -; AVX-NEXT: vshufps {{.*#+}} xmm9 = xmm9[2,3,2,3] +; AVX-NEXT: vmovaps 320(%rdi), %xmm5 +; AVX-NEXT: vshufps {{.*#+}} xmm3 = xmm5[0,1,0,1] +; AVX-NEXT: vmovaps 288(%rdi), %xmm8 +; AVX-NEXT: vblendps {{.*#+}} xmm3 = xmm8[0,1,2],xmm3[3] +; AVX-NEXT: vshufps {{.*#+}} xmm9 = xmm11[2,3,2,3] ; AVX-NEXT: vblendps {{.*#+}} xmm9 = mem[0],xmm9[1],mem[2,3] -; AVX-NEXT: vblendps {{.*#+}} xmm9 = xmm9[0,1],xmm11[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm9[0,1,2,3],ymm0[4,5,6,7] +; AVX-NEXT: vblendps {{.*#+}} xmm3 = xmm9[0,1],xmm3[2,3] +; AVX-NEXT: vblendps {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7] +; AVX-NEXT: vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vperm2f128 {{.*#+}} ymm3 = ymm0[2,3,0,1] +; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm0[3,0],ymm3[0,0],ymm0[7,4],ymm3[4,4] +; AVX-NEXT: vshufps {{.*#+}} ymm3 = ymm6[1,0],ymm4[2,0],ymm6[5,4],ymm4[6,4] +; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm0[2,0],ymm3[2,0],ymm0[6,4],ymm3[6,4] +; AVX-NEXT: vmovaps 64(%rdi), %xmm11 +; AVX-NEXT: vmovaps 96(%rdi), %xmm9 +; AVX-NEXT: vshufps {{.*#+}} xmm3 = xmm9[0,1,0,1] +; AVX-NEXT: vblendps {{.*#+}} xmm3 = xmm11[0,1,2],xmm3[3] +; AVX-NEXT: vshufps {{.*#+}} xmm10 = xmm10[2,3,2,3] +; AVX-NEXT: vblendps {{.*#+}} xmm10 = mem[0],xmm10[1],mem[2,3] +; AVX-NEXT: vblendps {{.*#+}} xmm3 = xmm10[0,1],xmm3[2,3] +; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm3[0,1,2,3],ymm0[4,5,6,7] ; AVX-NEXT: vmovups %ymm0, (%rsp) # 32-byte Spill -; AVX-NEXT: vshufps {{.*#+}} ymm6 = ymm2[2,1],ymm6[3,3],ymm2[6,5],ymm6[7,7] -; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload +; AVX-NEXT: vshufps {{.*#+}} ymm3 = ymm2[2,1],ymm7[3,3],ymm2[6,5],ymm7[7,7] +; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload ; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload -; AVX-NEXT: vblendps {{.*#+}} xmm9 = xmm1[0],xmm0[1],xmm1[2,3] -; AVX-NEXT: vinsertf128 $1, %xmm9, %ymm0, %ymm9 -; AVX-NEXT: vshufps {{.*#+}} ymm6 = ymm9[1,0],ymm6[2,0],ymm9[5,4],ymm6[6,4] -; AVX-NEXT: vblendps {{.*#+}} xmm3 = xmm10[0,1,2],xmm3[3] +; AVX-NEXT: vblendps {{.*#+}} xmm7 = xmm14[0],xmm0[1],xmm14[2,3] +; AVX-NEXT: vinsertf128 $1, %xmm7, %ymm0, %ymm7 +; AVX-NEXT: vshufps {{.*#+}} ymm3 = ymm7[1,0],ymm3[2,0],ymm7[5,4],ymm3[6,4] +; AVX-NEXT: vblendps {{.*#+}} xmm7 = xmm5[0,1,2],xmm8[3] +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm12 # 32-byte Reload ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm13 # 32-byte Reload -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm14 # 32-byte Reload -; AVX-NEXT: vshufps {{.*#+}} ymm9 = ymm14[0,0],ymm13[1,0],ymm14[4,4],ymm13[5,4] -; AVX-NEXT: vshufps {{.*#+}} ymm9 = ymm9[2,0,2,3,6,4,6,7] -; AVX-NEXT: vextractf128 $1, %ymm9, %xmm9 -; AVX-NEXT: vshufps {{.*#+}} xmm3 = xmm9[0,1],xmm3[3,2] -; AVX-NEXT: vblendps {{.*#+}} ymm15 = ymm3[0,1,2,3],ymm6[4,5,6,7] -; AVX-NEXT: vshufps {{.*#+}} ymm4 = ymm5[2,1],ymm4[3,3],ymm5[6,5],ymm4[7,7] -; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload +; AVX-NEXT: vshufps {{.*#+}} ymm8 = ymm13[0,0],ymm12[1,0],ymm13[4,4],ymm12[5,4] +; AVX-NEXT: vextractf128 $1, %ymm8, %xmm8 +; AVX-NEXT: vshufps {{.*#+}} xmm7 = xmm8[2,0],xmm7[3,2] +; AVX-NEXT: vblendps {{.*#+}} ymm15 = ymm7[0,1,2,3],ymm3[4,5,6,7] +; AVX-NEXT: vshufps {{.*#+}} ymm4 = ymm6[2,1],ymm4[3,3],ymm6[6,5],ymm4[7,7] ; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload -; AVX-NEXT: vblendps {{.*#+}} xmm6 = xmm7[0],xmm3[1],xmm7[2,3] -; AVX-NEXT: vinsertf128 $1, %xmm6, %ymm0, %ymm6 -; AVX-NEXT: vshufps {{.*#+}} ymm4 = ymm6[1,0],ymm4[2,0],ymm6[5,4],ymm4[6,4] -; AVX-NEXT: vblendps {{.*#+}} xmm6 = xmm8[0,1,2],xmm12[3] +; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload +; AVX-NEXT: vblendps {{.*#+}} xmm7 = xmm3[0],xmm1[1],xmm3[2,3] +; AVX-NEXT: vinsertf128 $1, %xmm7, %ymm0, %ymm7 +; AVX-NEXT: vshufps {{.*#+}} ymm4 = ymm7[1,0],ymm4[2,0],ymm7[5,4],ymm4[6,4] +; AVX-NEXT: vblendps {{.*#+}} xmm7 = xmm9[0,1,2],xmm11[3] +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Reload ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm11 # 32-byte Reload -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm12 # 32-byte Reload -; AVX-NEXT: vshufps {{.*#+}} ymm9 = ymm11[0,0],ymm12[1,0],ymm11[4,4],ymm12[5,4] -; AVX-NEXT: vshufps {{.*#+}} ymm9 = ymm9[2,0,2,3,6,4,6,7] -; AVX-NEXT: vextractf128 $1, %ymm9, %xmm9 -; AVX-NEXT: vshufps {{.*#+}} xmm6 = xmm9[0,1],xmm6[3,2] -; AVX-NEXT: vblendps {{.*#+}} ymm4 = ymm6[0,1,2,3],ymm4[4,5,6,7] -; AVX-NEXT: vperm2f128 {{.*#+}} ymm6 = ymm2[2,3,0,1] -; AVX-NEXT: vshufps {{.*#+}} ymm2 = ymm2[3,0],ymm6[0,0],ymm2[7,4],ymm6[4,4] -; AVX-NEXT: vshufps {{.*#+}} xmm6 = xmm0[2,3,2,3] -; AVX-NEXT: vblendps {{.*#+}} xmm6 = xmm6[0],xmm1[1],xmm6[2,3] -; AVX-NEXT: vinsertf128 $1, %xmm6, %ymm0, %ymm6 -; AVX-NEXT: vshufps {{.*#+}} ymm2 = ymm6[0,1],ymm2[2,0],ymm6[4,5],ymm2[6,4] -; AVX-NEXT: vpermilps $68, {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Folded Reload -; AVX-NEXT: # xmm6 = mem[0,1,0,1] -; AVX-NEXT: vblendps {{.*#+}} xmm6 = xmm6[0,1,2],xmm10[3] -; AVX-NEXT: vshufps {{.*#+}} ymm9 = ymm14[1,0],ymm13[2,0],ymm14[5,4],ymm13[6,4] -; AVX-NEXT: vshufps {{.*#+}} ymm9 = ymm9[2,0,2,3,6,4,6,7] -; AVX-NEXT: vextractf128 $1, %ymm9, %xmm9 -; AVX-NEXT: vblendps {{.*#+}} xmm6 = xmm9[0,1],xmm6[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm2 = ymm6[0,1,2,3],ymm2[4,5,6,7] -; AVX-NEXT: vperm2f128 {{.*#+}} ymm6 = ymm5[2,3,0,1] -; AVX-NEXT: vshufps {{.*#+}} ymm5 = ymm5[3,0],ymm6[0,0],ymm5[7,4],ymm6[4,4] -; AVX-NEXT: vshufps {{.*#+}} xmm6 = xmm3[2,3,2,3] -; AVX-NEXT: vblendps {{.*#+}} xmm6 = xmm6[0],xmm7[1],xmm6[2,3] -; AVX-NEXT: vinsertf128 $1, %xmm6, %ymm0, %ymm6 -; AVX-NEXT: vshufps {{.*#+}} ymm5 = ymm6[0,1],ymm5[2,0],ymm6[4,5],ymm5[6,4] -; AVX-NEXT: vpermilps $68, {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Folded Reload -; AVX-NEXT: # xmm6 = mem[0,1,0,1] -; AVX-NEXT: vblendps {{.*#+}} xmm6 = xmm6[0,1,2],xmm8[3] -; AVX-NEXT: vshufps {{.*#+}} ymm8 = ymm11[1,0],ymm12[2,0],ymm11[5,4],ymm12[6,4] -; AVX-NEXT: vshufps {{.*#+}} ymm8 = ymm8[2,0,2,3,6,4,6,7] +; AVX-NEXT: vshufps {{.*#+}} ymm8 = ymm10[0,0],ymm11[1,0],ymm10[4,4],ymm11[5,4] ; AVX-NEXT: vextractf128 $1, %ymm8, %xmm8 -; AVX-NEXT: vblendps {{.*#+}} xmm6 = xmm8[0,1],xmm6[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm5 = ymm6[0,1,2,3],ymm5[4,5,6,7] +; AVX-NEXT: vshufps {{.*#+}} xmm7 = xmm8[2,0],xmm7[3,2] +; AVX-NEXT: vblendps {{.*#+}} ymm4 = ymm7[0,1,2,3],ymm4[4,5,6,7] +; AVX-NEXT: vpermilps $68, {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Folded Reload +; AVX-NEXT: # xmm7 = mem[0,1,0,1] +; AVX-NEXT: vblendps {{.*#+}} xmm5 = xmm7[0,1,2],xmm5[3] +; AVX-NEXT: vshufps {{.*#+}} ymm7 = ymm13[1,0],ymm12[2,0],ymm13[5,4],ymm12[6,4] +; AVX-NEXT: vextractf128 $1, %ymm7, %xmm7 +; AVX-NEXT: vshufps {{.*#+}} xmm5 = xmm7[2,0],xmm5[2,3] +; AVX-NEXT: vperm2f128 {{.*#+}} ymm7 = ymm2[2,3,0,1] +; AVX-NEXT: vshufps {{.*#+}} ymm2 = ymm2[3,0],ymm7[0,0],ymm2[7,4],ymm7[4,4] +; AVX-NEXT: vshufps {{.*#+}} xmm7 = xmm0[2,3,2,3] +; AVX-NEXT: vblendps {{.*#+}} xmm7 = xmm7[0],xmm14[1],xmm7[2,3] +; AVX-NEXT: vinsertf128 $1, %xmm7, %ymm0, %ymm7 +; AVX-NEXT: vshufps {{.*#+}} ymm2 = ymm7[0,1],ymm2[2,0],ymm7[4,5],ymm2[6,4] +; AVX-NEXT: vblendps {{.*#+}} ymm2 = ymm5[0,1,2,3],ymm2[4,5,6,7] +; AVX-NEXT: vpermilps $68, {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Folded Reload +; AVX-NEXT: # xmm5 = mem[0,1,0,1] +; AVX-NEXT: vblendps {{.*#+}} xmm5 = xmm5[0,1,2],xmm9[3] +; AVX-NEXT: vshufps {{.*#+}} ymm7 = ymm10[1,0],ymm11[2,0],ymm10[5,4],ymm11[6,4] +; AVX-NEXT: vextractf128 $1, %ymm7, %xmm7 +; AVX-NEXT: vshufps {{.*#+}} xmm5 = xmm7[2,0],xmm5[2,3] +; AVX-NEXT: vperm2f128 {{.*#+}} ymm7 = ymm6[2,3,0,1] +; AVX-NEXT: vshufps {{.*#+}} ymm6 = ymm6[3,0],ymm7[0,0],ymm6[7,4],ymm7[4,4] +; AVX-NEXT: vshufps {{.*#+}} xmm7 = xmm1[2,3,2,3] +; AVX-NEXT: vblendps {{.*#+}} xmm7 = xmm7[0],xmm3[1],xmm7[2,3] +; AVX-NEXT: vinsertf128 $1, %xmm7, %ymm0, %ymm7 +; AVX-NEXT: vshufps {{.*#+}} ymm6 = ymm7[0,1],ymm6[2,0],ymm7[4,5],ymm6[6,4] +; AVX-NEXT: vblendps {{.*#+}} ymm5 = ymm5[0,1,2,3],ymm6[4,5,6,7] +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm0, (%rsi) ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload ; AVX-NEXT: vmovaps %ymm6, 32(%rsi) -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm6, (%rsi) -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm0, 32(%rdx) ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload ; AVX-NEXT: vmovaps %ymm0, (%rdx) ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm0, 32(%rcx) +; AVX-NEXT: vmovaps %ymm0, 32(%rdx) ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload ; AVX-NEXT: vmovaps %ymm0, (%rcx) ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm0, 32(%r8) +; AVX-NEXT: vmovaps %ymm0, 32(%rcx) ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload ; AVX-NEXT: vmovaps %ymm0, (%r8) -; AVX-NEXT: vmovups (%rsp), %ymm0 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm0, 32(%r9) ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm0, 32(%r8) +; AVX-NEXT: vmovups (%rsp), %ymm0 # 32-byte Reload ; AVX-NEXT: vmovaps %ymm0, (%r9) +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm0, 32(%r9) ; AVX-NEXT: movq {{[0-9]+}}(%rsp), %rax -; AVX-NEXT: vmovaps %ymm4, 32(%rax) -; AVX-NEXT: vmovaps %ymm15, (%rax) +; AVX-NEXT: vmovaps %ymm4, (%rax) +; AVX-NEXT: vmovaps %ymm15, 32(%rax) ; AVX-NEXT: movq {{[0-9]+}}(%rsp), %rax -; AVX-NEXT: vmovaps %ymm5, 32(%rax) -; AVX-NEXT: vmovaps %ymm2, (%rax) +; AVX-NEXT: vmovaps %ymm5, (%rax) +; AVX-NEXT: vmovaps %ymm2, 32(%rax) ; AVX-NEXT: addq $456, %rsp # imm = 0x1C8 ; AVX-NEXT: vzeroupper ; AVX-NEXT: retq @@ -4893,26 +4885,26 @@ define void @load_i32_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt ; ; AVX-LABEL: load_i32_stride7_vf32: ; AVX: # %bb.0: -; AVX-NEXT: subq $1464, %rsp # imm = 0x5B8 +; AVX-NEXT: subq $1432, %rsp # imm = 0x598 ; AVX-NEXT: vmovaps 480(%rdi), %ymm4 ; AVX-NEXT: vmovups %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX-NEXT: vmovaps 448(%rdi), %ymm3 ; AVX-NEXT: vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vmovaps 544(%rdi), %ymm7 +; AVX-NEXT: vmovaps 544(%rdi), %ymm5 ; AVX-NEXT: vmovaps 32(%rdi), %ymm2 ; AVX-NEXT: vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX-NEXT: vmovaps (%rdi), %ymm1 ; AVX-NEXT: vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vmovaps 96(%rdi), %ymm6 +; AVX-NEXT: vmovaps 96(%rdi), %ymm12 ; AVX-NEXT: vmovaps 80(%rdi), %xmm0 ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vunpcklpd {{.*#+}} ymm0 = ymm6[0],ymm0[0],ymm6[2],ymm0[2] -; AVX-NEXT: vmovups %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vunpcklpd {{.*#+}} ymm0 = ymm12[0],ymm0[0],ymm12[2],ymm0[2] +; AVX-NEXT: vmovups %ymm12, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX-NEXT: vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5],ymm2[6],ymm1[7] ; AVX-NEXT: vextractf128 $1, %ymm1, %xmm1 -; AVX-NEXT: vmovaps (%rdi), %xmm9 -; AVX-NEXT: vblendps {{.*#+}} xmm1 = xmm9[0,1],xmm1[2,3] -; AVX-NEXT: vmovaps %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; AVX-NEXT: vmovaps (%rdi), %xmm8 +; AVX-NEXT: vblendps {{.*#+}} xmm1 = xmm8[0,1],xmm1[2,3] +; AVX-NEXT: vmovaps %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; AVX-NEXT: vshufps {{.*#+}} xmm1 = xmm1[0,3,2,3] ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2],ymm0[3,4,5,6,7] ; AVX-NEXT: vmovaps 160(%rdi), %xmm2 @@ -4920,20 +4912,19 @@ define void @load_i32_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt ; AVX-NEXT: vmovaps 128(%rdi), %xmm1 ; AVX-NEXT: vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; AVX-NEXT: vunpckhpd {{.*#+}} xmm1 = xmm1[1],xmm2[1] -; AVX-NEXT: vmovaps 192(%rdi), %xmm10 -; AVX-NEXT: vinsertps {{.*#+}} xmm1 = zero,xmm1[1,2],xmm10[1] -; AVX-NEXT: vmovaps %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; AVX-NEXT: vmovaps 192(%rdi), %xmm7 +; AVX-NEXT: vinsertps {{.*#+}} xmm1 = zero,xmm1[1,2],xmm7[1] +; AVX-NEXT: vmovaps %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; AVX-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm1 ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm1[5,6,7] ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX-NEXT: vmovaps 528(%rdi), %xmm0 ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vunpcklpd {{.*#+}} ymm0 = ymm7[0],ymm0[0],ymm7[2],ymm0[2] -; AVX-NEXT: vmovups %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vunpcklpd {{.*#+}} ymm0 = ymm5[0],ymm0[0],ymm5[2],ymm0[2] ; AVX-NEXT: vblendps {{.*#+}} ymm1 = ymm3[0,1,2,3,4,5],ymm4[6],ymm3[7] ; AVX-NEXT: vextractf128 $1, %ymm1, %xmm1 -; AVX-NEXT: vmovaps 448(%rdi), %xmm8 -; AVX-NEXT: vblendps {{.*#+}} xmm1 = xmm8[0,1],xmm1[2,3] +; AVX-NEXT: vmovaps 448(%rdi), %xmm10 +; AVX-NEXT: vblendps {{.*#+}} xmm1 = xmm10[0,1],xmm1[2,3] ; AVX-NEXT: vshufps {{.*#+}} xmm1 = xmm1[0,3,2,3] ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2],ymm0[3,4,5,6,7] ; AVX-NEXT: vmovaps 608(%rdi), %xmm2 @@ -4941,9 +4932,9 @@ define void @load_i32_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt ; AVX-NEXT: vmovaps 576(%rdi), %xmm1 ; AVX-NEXT: vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; AVX-NEXT: vunpckhpd {{.*#+}} xmm1 = xmm1[1],xmm2[1] -; AVX-NEXT: vmovaps 640(%rdi), %xmm4 -; AVX-NEXT: vinsertps {{.*#+}} xmm1 = zero,xmm1[1,2],xmm4[1] -; AVX-NEXT: vmovaps %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; AVX-NEXT: vmovaps 640(%rdi), %xmm9 +; AVX-NEXT: vinsertps {{.*#+}} xmm1 = zero,xmm1[1,2],xmm9[1] +; AVX-NEXT: vmovaps %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; AVX-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm1 ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm1[5,6,7] ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill @@ -4953,24 +4944,24 @@ define void @load_i32_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6],ymm0[7] ; AVX-NEXT: vextractf128 $1, %ymm0, %xmm0 -; AVX-NEXT: vmovaps 224(%rdi), %xmm5 -; AVX-NEXT: vblendps {{.*#+}} xmm0 = xmm5[0,1],xmm0[2,3] -; AVX-NEXT: vmovaps %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; AVX-NEXT: vmovaps 224(%rdi), %xmm11 +; AVX-NEXT: vblendps {{.*#+}} xmm0 = xmm11[0,1],xmm0[2,3] +; AVX-NEXT: vmovaps %xmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; AVX-NEXT: vshufps {{.*#+}} xmm0 = xmm0[0,3,2,3] -; AVX-NEXT: vmovaps 320(%rdi), %ymm13 +; AVX-NEXT: vmovaps 320(%rdi), %ymm4 ; AVX-NEXT: vmovaps 304(%rdi), %xmm1 ; AVX-NEXT: vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vunpcklpd {{.*#+}} ymm1 = ymm13[0],ymm1[0],ymm13[2],ymm1[2] -; AVX-NEXT: vmovups %ymm13, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vunpcklpd {{.*#+}} ymm1 = ymm4[0],ymm1[0],ymm4[2],ymm1[2] +; AVX-NEXT: vmovups %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm1[3,4,5,6,7] -; AVX-NEXT: vmovaps 384(%rdi), %xmm2 -; AVX-NEXT: vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; AVX-NEXT: vmovaps 352(%rdi), %xmm1 +; AVX-NEXT: vmovaps 384(%rdi), %xmm1 ; AVX-NEXT: vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; AVX-NEXT: vunpckhpd {{.*#+}} xmm1 = xmm1[1],xmm2[1] -; AVX-NEXT: vmovaps 416(%rdi), %xmm11 -; AVX-NEXT: vinsertps {{.*#+}} xmm1 = zero,xmm1[1,2],xmm11[1] -; AVX-NEXT: vmovaps %xmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; AVX-NEXT: vmovaps 352(%rdi), %xmm2 +; AVX-NEXT: vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; AVX-NEXT: vunpckhpd {{.*#+}} xmm1 = xmm2[1],xmm1[1] +; AVX-NEXT: vmovaps 416(%rdi), %xmm3 +; AVX-NEXT: vinsertps {{.*#+}} xmm1 = zero,xmm1[1,2],xmm3[1] +; AVX-NEXT: vmovaps %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; AVX-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm1 ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm1[5,6,7] ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill @@ -4984,474 +4975,470 @@ define void @load_i32_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt ; AVX-NEXT: vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; AVX-NEXT: vblendps {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3] ; AVX-NEXT: vshufps {{.*#+}} xmm0 = xmm0[0,3,2,3] -; AVX-NEXT: vmovaps 768(%rdi), %ymm3 +; AVX-NEXT: vmovaps 768(%rdi), %ymm14 ; AVX-NEXT: vmovaps 752(%rdi), %xmm1 ; AVX-NEXT: vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vunpcklpd {{.*#+}} ymm1 = ymm3[0],ymm1[0],ymm3[2],ymm1[2] +; AVX-NEXT: vunpcklpd {{.*#+}} ymm1 = ymm14[0],ymm1[0],ymm14[2],ymm1[2] ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm1[3,4,5,6,7] ; AVX-NEXT: vmovaps 832(%rdi), %xmm2 ; AVX-NEXT: vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; AVX-NEXT: vmovaps 800(%rdi), %xmm1 ; AVX-NEXT: vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; AVX-NEXT: vunpckhpd {{.*#+}} xmm1 = xmm1[1],xmm2[1] -; AVX-NEXT: vmovaps 864(%rdi), %xmm2 -; AVX-NEXT: vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; AVX-NEXT: vinsertps {{.*#+}} xmm1 = zero,xmm1[1,2],xmm2[1] +; AVX-NEXT: vmovaps 864(%rdi), %xmm6 +; AVX-NEXT: vinsertps {{.*#+}} xmm1 = zero,xmm1[1,2],xmm6[1] ; AVX-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm1 ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm1[5,6,7] ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX-NEXT: vmovaps 64(%rdi), %ymm0 ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm6[1,1],ymm0[2,2],ymm6[5,5],ymm0[6,6] +; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm12[1,1],ymm0[2,2],ymm12[5,5],ymm0[6,6] ; AVX-NEXT: vperm2f128 {{.*#+}} ymm0 = ymm0[2,3,2,3] ; AVX-NEXT: vmovaps 32(%rdi), %xmm1 ; AVX-NEXT: vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; AVX-NEXT: vblendps {{.*#+}} xmm1 = xmm1[0],xmm9[1],xmm1[2,3] +; AVX-NEXT: vblendps {{.*#+}} xmm1 = xmm1[0],xmm8[1],xmm1[2,3] ; AVX-NEXT: vshufps {{.*#+}} xmm1 = xmm1[1,0],mem[3,3] ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2],ymm0[3,4,5,6,7] ; AVX-NEXT: vmovaps 160(%rdi), %ymm1 ; AVX-NEXT: vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vmovaps 128(%rdi), %ymm2 -; AVX-NEXT: vmovups %ymm2, (%rsp) # 32-byte Spill -; AVX-NEXT: vperm2f128 {{.*#+}} ymm1 = ymm2[2,3],ymm1[0,1] -; AVX-NEXT: vshufps {{.*#+}} ymm1 = ymm2[0,0],ymm1[3,3],ymm2[4,4],ymm1[7,7] +; AVX-NEXT: vmovaps 128(%rdi), %ymm15 +; AVX-NEXT: vperm2f128 {{.*#+}} ymm1 = ymm15[2,3],ymm1[0,1] +; AVX-NEXT: vshufps {{.*#+}} ymm1 = ymm15[0,0],ymm1[3,3],ymm15[4,4],ymm1[7,7] +; AVX-NEXT: vmovups %ymm15, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX-NEXT: vextractf128 $1, %ymm1, %xmm1 -; AVX-NEXT: vinsertps {{.*#+}} xmm1 = zero,xmm1[1,2],xmm10[2] +; AVX-NEXT: vinsertps {{.*#+}} xmm1 = zero,xmm1[1,2],xmm7[2] ; AVX-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm1 ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm1[5,6,7] ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX-NEXT: vmovaps 512(%rdi), %ymm0 ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm7[1,1],ymm0[2,2],ymm7[5,5],ymm0[6,6] +; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm5[1,1],ymm0[2,2],ymm5[5,5],ymm0[6,6] +; AVX-NEXT: vmovaps %ymm5, %ymm7 ; AVX-NEXT: vperm2f128 {{.*#+}} ymm0 = ymm0[2,3,2,3] -; AVX-NEXT: vmovaps 480(%rdi), %xmm10 -; AVX-NEXT: vblendps {{.*#+}} xmm1 = xmm10[0],xmm8[1],xmm10[2,3] +; AVX-NEXT: vmovaps 480(%rdi), %xmm1 +; AVX-NEXT: vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; AVX-NEXT: vblendps {{.*#+}} xmm2 = xmm1[0],xmm10[1],xmm1[2,3] ; AVX-NEXT: vmovaps %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; AVX-NEXT: vmovaps %xmm8, %xmm7 -; AVX-NEXT: vmovaps %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; AVX-NEXT: vshufps {{.*#+}} xmm1 = xmm1[1,0],mem[3,3] -; AVX-NEXT: vblendps {{.*#+}} ymm1 = ymm1[0,1,2],ymm0[3,4,5,6,7] -; AVX-NEXT: vmovaps 608(%rdi), %ymm14 -; AVX-NEXT: vmovaps 576(%rdi), %ymm12 -; AVX-NEXT: vperm2f128 {{.*#+}} ymm2 = ymm12[2,3],ymm14[0,1] -; AVX-NEXT: vmovups %ymm14, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vshufps {{.*#+}} ymm2 = ymm12[0,0],ymm2[3,3],ymm12[4,4],ymm2[7,7] -; AVX-NEXT: vextractf128 $1, %ymm2, %xmm2 -; AVX-NEXT: vinsertps {{.*#+}} xmm2 = zero,xmm2[1,2],xmm4[2] -; AVX-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm2 -; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3,4],ymm2[5,6,7] +; AVX-NEXT: vshufps {{.*#+}} xmm2 = xmm2[1,0],mem[3,3] +; AVX-NEXT: vblendps {{.*#+}} ymm2 = ymm2[0,1,2],ymm0[3,4,5,6,7] +; AVX-NEXT: vmovaps 608(%rdi), %ymm0 ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vmovaps 576(%rdi), %ymm12 +; AVX-NEXT: vperm2f128 {{.*#+}} ymm5 = ymm12[2,3],ymm0[0,1] +; AVX-NEXT: vshufps {{.*#+}} ymm5 = ymm12[0,0],ymm5[3,3],ymm12[4,4],ymm5[7,7] +; AVX-NEXT: vextractf128 $1, %ymm5, %xmm5 +; AVX-NEXT: vinsertps {{.*#+}} xmm5 = zero,xmm5[1,2],xmm9[2] +; AVX-NEXT: vinsertf128 $1, %xmm5, %ymm0, %ymm5 +; AVX-NEXT: vblendps {{.*#+}} ymm2 = ymm2[0,1,2,3,4],ymm5[5,6,7] +; AVX-NEXT: vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX-NEXT: vmovaps 288(%rdi), %ymm0 ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vshufps {{.*#+}} ymm1 = ymm13[1,1],ymm0[2,2],ymm13[5,5],ymm0[6,6] -; AVX-NEXT: vperm2f128 {{.*#+}} ymm1 = ymm1[2,3,2,3] +; AVX-NEXT: vshufps {{.*#+}} ymm2 = ymm4[1,1],ymm0[2,2],ymm4[5,5],ymm0[6,6] +; AVX-NEXT: vperm2f128 {{.*#+}} ymm2 = ymm2[2,3,2,3] ; AVX-NEXT: vmovaps 256(%rdi), %xmm0 ; AVX-NEXT: vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; AVX-NEXT: vblendps {{.*#+}} xmm2 = xmm0[0],xmm5[1],xmm0[2,3] -; AVX-NEXT: vshufps {{.*#+}} xmm2 = xmm2[1,0],mem[3,3] -; AVX-NEXT: vblendps {{.*#+}} ymm1 = ymm2[0,1,2],ymm1[3,4,5,6,7] -; AVX-NEXT: vmovaps 384(%rdi), %ymm0 +; AVX-NEXT: vblendps {{.*#+}} xmm5 = xmm0[0],xmm11[1],xmm0[2,3] +; AVX-NEXT: vshufps {{.*#+}} xmm5 = xmm5[1,0],mem[3,3] +; AVX-NEXT: vblendps {{.*#+}} ymm2 = ymm5[0,1,2],ymm2[3,4,5,6,7] +; AVX-NEXT: vmovaps 384(%rdi), %ymm1 +; AVX-NEXT: vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vmovaps 352(%rdi), %ymm0 ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vmovaps 352(%rdi), %ymm2 -; AVX-NEXT: vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vperm2f128 {{.*#+}} ymm5 = ymm2[2,3],ymm0[0,1] -; AVX-NEXT: vshufps {{.*#+}} ymm5 = ymm2[0,0],ymm5[3,3],ymm2[4,4],ymm5[7,7] +; AVX-NEXT: vperm2f128 {{.*#+}} ymm5 = ymm0[2,3],ymm1[0,1] +; AVX-NEXT: vshufps {{.*#+}} ymm5 = ymm0[0,0],ymm5[3,3],ymm0[4,4],ymm5[7,7] ; AVX-NEXT: vextractf128 $1, %ymm5, %xmm5 -; AVX-NEXT: vinsertps {{.*#+}} xmm5 = zero,xmm5[1,2],xmm11[2] +; AVX-NEXT: vinsertps {{.*#+}} xmm5 = zero,xmm5[1,2],xmm3[2] ; AVX-NEXT: vinsertf128 $1, %xmm5, %ymm0, %ymm5 -; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3,4],ymm5[5,6,7] -; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vmovaps 736(%rdi), %ymm8 -; AVX-NEXT: vshufps {{.*#+}} ymm1 = ymm3[1,1],ymm8[2,2],ymm3[5,5],ymm8[6,6] -; AVX-NEXT: vmovups %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vmovaps %ymm3, %ymm9 -; AVX-NEXT: vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vperm2f128 {{.*#+}} ymm1 = ymm1[2,3,2,3] -; AVX-NEXT: vmovaps 704(%rdi), %xmm6 -; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload -; AVX-NEXT: vblendps {{.*#+}} xmm11 = xmm6[0],xmm2[1],xmm6[2,3] -; AVX-NEXT: vmovaps %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; AVX-NEXT: vshufps {{.*#+}} xmm11 = xmm11[1,0],mem[3,3] -; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm11[0,1,2],ymm1[3,4,5,6,7] -; AVX-NEXT: vmovaps 832(%rdi), %ymm5 -; AVX-NEXT: vmovaps 800(%rdi), %ymm15 -; AVX-NEXT: vperm2f128 {{.*#+}} ymm13 = ymm15[2,3],ymm5[0,1] +; AVX-NEXT: vblendps {{.*#+}} ymm2 = ymm2[0,1,2,3,4],ymm5[5,6,7] +; AVX-NEXT: vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vmovaps 736(%rdi), %ymm5 +; AVX-NEXT: vmovaps %ymm14, %ymm3 +; AVX-NEXT: vmovups %ymm14, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vshufps {{.*#+}} ymm2 = ymm14[1,1],ymm5[2,2],ymm14[5,5],ymm5[6,6] ; AVX-NEXT: vmovups %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vshufps {{.*#+}} ymm13 = ymm15[0,0],ymm13[3,3],ymm15[4,4],ymm13[7,7] -; AVX-NEXT: vextractf128 $1, %ymm13, %xmm13 -; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload -; AVX-NEXT: vinsertps {{.*#+}} xmm13 = zero,xmm13[1,2],xmm4[2] -; AVX-NEXT: vinsertf128 $1, %xmm13, %ymm0, %ymm13 -; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm13[5,6,7] -; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vpermilps $238, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload -; AVX-NEXT: # xmm0 = mem[2,3,2,3] +; AVX-NEXT: vperm2f128 {{.*#+}} ymm2 = ymm2[2,3,2,3] +; AVX-NEXT: vmovaps 704(%rdi), %xmm4 ; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload -; AVX-NEXT: vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3] -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm13 # 32-byte Reload -; AVX-NEXT: vshufps $199, {{[-0-9]+}}(%r{{[sb]}}p), %ymm13, %ymm13 # 32-byte Folded Reload -; AVX-NEXT: # ymm13 = ymm13[3,1],mem[0,3],ymm13[7,5],mem[4,7] -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm11 # 32-byte Reload -; AVX-NEXT: vshufps {{.*#+}} ymm13 = ymm11[2,1],ymm13[2,0],ymm11[6,5],ymm13[6,4] -; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm13[2,3,4,5,6,7] -; AVX-NEXT: vmovups (%rsp), %ymm3 # 32-byte Reload -; AVX-NEXT: vunpcklpd {{[-0-9]+}}(%r{{[sb]}}p), %ymm3, %ymm13 # 32-byte Folded Reload -; AVX-NEXT: # ymm13 = ymm3[0],mem[0],ymm3[2],mem[2] -; AVX-NEXT: vextractf128 $1, %ymm13, %xmm13 -; AVX-NEXT: vblendps $8, {{[-0-9]+}}(%r{{[sb]}}p), %xmm13, %xmm13 # 16-byte Folded Reload -; AVX-NEXT: # xmm13 = xmm13[0,1,2],mem[3] -; AVX-NEXT: vinsertf128 $1, %xmm13, %ymm0, %ymm13 -; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm13[5,6,7] -; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vshufps {{.*#+}} xmm0 = xmm7[2,3,2,3] -; AVX-NEXT: vblendps {{.*#+}} xmm0 = xmm0[0],xmm10[1],xmm0[2,3] -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm13 # 32-byte Reload -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Reload -; AVX-NEXT: vshufps {{.*#+}} ymm13 = ymm13[3,1],ymm10[0,3],ymm13[7,5],ymm10[4,7] -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload -; AVX-NEXT: vshufps {{.*#+}} ymm13 = ymm7[2,1],ymm13[2,0],ymm7[6,5],ymm13[6,4] -; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm13[2,3,4,5,6,7] -; AVX-NEXT: vunpcklpd {{.*#+}} ymm13 = ymm12[0],ymm14[0],ymm12[2],ymm14[2] -; AVX-NEXT: vextractf128 $1, %ymm13, %xmm13 -; AVX-NEXT: vblendps $8, {{[-0-9]+}}(%r{{[sb]}}p), %xmm13, %xmm13 # 16-byte Folded Reload -; AVX-NEXT: # xmm13 = xmm13[0,1,2],mem[3] -; AVX-NEXT: vinsertf128 $1, %xmm13, %ymm0, %ymm13 -; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm13[5,6,7] -; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vshufps {{.*#+}} xmm0 = xmm2[2,3,2,3] -; AVX-NEXT: vblendps {{.*#+}} xmm0 = xmm0[0],xmm6[1],xmm0[2,3] -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm13 # 32-byte Reload -; AVX-NEXT: vshufps {{.*#+}} ymm13 = ymm13[3,1],ymm8[0,3],ymm13[7,5],ymm8[4,7] -; AVX-NEXT: vshufps {{.*#+}} ymm13 = ymm9[2,1],ymm13[2,0],ymm9[6,5],ymm13[6,4] -; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm13[2,3,4,5,6,7] -; AVX-NEXT: vunpcklpd {{.*#+}} ymm13 = ymm15[0],ymm5[0],ymm15[2],ymm5[2] -; AVX-NEXT: vextractf128 $1, %ymm13, %xmm13 -; AVX-NEXT: vblendps {{.*#+}} xmm8 = xmm13[0,1,2],xmm4[3] -; AVX-NEXT: vinsertf128 $1, %xmm8, %ymm0, %ymm8 -; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm8[5,6,7] -; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload -; AVX-NEXT: vshufps {{.*#+}} xmm0 = xmm5[2,3,2,3] -; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload -; AVX-NEXT: vblendps {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2,3] +; AVX-NEXT: vblendps {{.*#+}} xmm11 = xmm4[0],xmm1[1],xmm4[2,3] +; AVX-NEXT: vmovaps %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; AVX-NEXT: vshufps {{.*#+}} xmm11 = xmm11[1,0],mem[3,3] +; AVX-NEXT: vblendps {{.*#+}} ymm11 = ymm11[0,1,2],ymm2[3,4,5,6,7] +; AVX-NEXT: vmovaps 832(%rdi), %ymm13 +; AVX-NEXT: vmovaps 800(%rdi), %ymm2 +; AVX-NEXT: vperm2f128 {{.*#+}} ymm14 = ymm2[2,3],ymm13[0,1] +; AVX-NEXT: vmovups %ymm13, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vshufps {{.*#+}} ymm14 = ymm2[0,0],ymm14[3,3],ymm2[4,4],ymm14[7,7] +; AVX-NEXT: vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vextractf128 $1, %ymm14, %xmm14 +; AVX-NEXT: vinsertps {{.*#+}} xmm14 = zero,xmm14[1,2],xmm6[2] +; AVX-NEXT: vinsertf128 $1, %xmm14, %ymm0, %ymm14 +; AVX-NEXT: vblendps {{.*#+}} ymm11 = ymm11[0,1,2,3,4],ymm14[5,6,7] +; AVX-NEXT: vmovups %ymm11, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vpermilps $238, {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Folded Reload +; AVX-NEXT: # xmm11 = mem[2,3,2,3] +; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload +; AVX-NEXT: vblendps {{.*#+}} xmm11 = xmm11[0],xmm0[1],xmm11[2,3] +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm14 # 32-byte Reload +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Reload +; AVX-NEXT: vshufps {{.*#+}} ymm14 = ymm14[3,1],ymm9[0,3],ymm14[7,5],ymm9[4,7] ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload +; AVX-NEXT: vshufps {{.*#+}} ymm14 = ymm8[2,1],ymm14[2,0],ymm8[6,5],ymm14[6,4] +; AVX-NEXT: vblendps {{.*#+}} ymm11 = ymm11[0,1],ymm14[2,3,4,5,6,7] +; AVX-NEXT: vunpcklpd {{[-0-9]+}}(%r{{[sb]}}p), %ymm15, %ymm14 # 32-byte Folded Reload +; AVX-NEXT: # ymm14 = ymm15[0],mem[0],ymm15[2],mem[2] +; AVX-NEXT: vextractf128 $1, %ymm14, %xmm14 +; AVX-NEXT: vblendps $8, {{[-0-9]+}}(%r{{[sb]}}p), %xmm14, %xmm14 # 16-byte Folded Reload +; AVX-NEXT: # xmm14 = xmm14[0,1,2],mem[3] +; AVX-NEXT: vinsertf128 $1, %xmm14, %ymm0, %ymm14 +; AVX-NEXT: vblendps {{.*#+}} ymm11 = ymm11[0,1,2,3,4],ymm14[5,6,7] +; AVX-NEXT: vmovups %ymm11, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vshufps {{.*#+}} xmm11 = xmm10[2,3,2,3] +; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload +; AVX-NEXT: vblendps {{.*#+}} xmm11 = xmm11[0],xmm15[1],xmm11[2,3] ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm14 # 32-byte Reload -; AVX-NEXT: vshufps {{.*#+}} ymm8 = ymm8[3,1],ymm14[0,3],ymm8[7,5],ymm14[4,7] -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload -; AVX-NEXT: vshufps {{.*#+}} ymm8 = ymm4[2,1],ymm8[2,0],ymm4[6,5],ymm8[6,4] -; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm8[2,3,4,5,6,7] +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Reload +; AVX-NEXT: vshufps {{.*#+}} ymm14 = ymm14[3,1],ymm10[0,3],ymm14[7,5],ymm10[4,7] +; AVX-NEXT: vshufps {{.*#+}} ymm14 = ymm7[2,1],ymm14[2,0],ymm7[6,5],ymm14[6,4] +; AVX-NEXT: vblendps {{.*#+}} ymm11 = ymm11[0,1],ymm14[2,3,4,5,6,7] +; AVX-NEXT: vunpcklpd {{[-0-9]+}}(%r{{[sb]}}p), %ymm12, %ymm14 # 32-byte Folded Reload +; AVX-NEXT: # ymm14 = ymm12[0],mem[0],ymm12[2],mem[2] +; AVX-NEXT: vextractf128 $1, %ymm14, %xmm14 +; AVX-NEXT: vblendps $8, {{[-0-9]+}}(%r{{[sb]}}p), %xmm14, %xmm14 # 16-byte Folded Reload +; AVX-NEXT: # xmm14 = xmm14[0,1,2],mem[3] +; AVX-NEXT: vinsertf128 $1, %xmm14, %ymm0, %ymm14 +; AVX-NEXT: vblendps {{.*#+}} ymm11 = ymm11[0,1,2,3,4],ymm14[5,6,7] +; AVX-NEXT: vmovups %ymm11, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vshufps {{.*#+}} xmm11 = xmm1[2,3,2,3] +; AVX-NEXT: vblendps {{.*#+}} xmm11 = xmm11[0],xmm4[1],xmm11[2,3] +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm14 # 32-byte Reload +; AVX-NEXT: vshufps {{.*#+}} ymm14 = ymm14[3,1],ymm5[0,3],ymm14[7,5],ymm5[4,7] +; AVX-NEXT: vshufps {{.*#+}} ymm14 = ymm3[2,1],ymm14[2,0],ymm3[6,5],ymm14[6,4] +; AVX-NEXT: vblendps {{.*#+}} ymm11 = ymm11[0,1],ymm14[2,3,4,5,6,7] +; AVX-NEXT: vunpcklpd {{.*#+}} ymm14 = ymm2[0],ymm13[0],ymm2[2],ymm13[2] +; AVX-NEXT: vextractf128 $1, %ymm14, %xmm14 +; AVX-NEXT: vblendps {{.*#+}} xmm4 = xmm14[0,1,2],xmm6[3] +; AVX-NEXT: vinsertf128 $1, %xmm4, %ymm0, %ymm4 +; AVX-NEXT: vblendps {{.*#+}} ymm4 = ymm11[0,1,2,3,4],ymm4[5,6,7] +; AVX-NEXT: vmovups %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload +; AVX-NEXT: vshufps {{.*#+}} xmm4 = xmm6[2,3,2,3] +; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload +; AVX-NEXT: vblendps {{.*#+}} xmm4 = xmm4[0],xmm1[1],xmm4[2,3] +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm11 # 32-byte Reload +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload +; AVX-NEXT: vshufps {{.*#+}} ymm11 = ymm11[3,1],ymm3[0,3],ymm11[7,5],ymm3[4,7] +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload +; AVX-NEXT: vshufps {{.*#+}} ymm11 = ymm2[2,1],ymm11[2,0],ymm2[6,5],ymm11[6,4] +; AVX-NEXT: vblendps {{.*#+}} ymm4 = ymm4[0,1],ymm11[2,3,4,5,6,7] ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm13 # 32-byte Reload -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload -; AVX-NEXT: vunpcklpd {{.*#+}} ymm8 = ymm6[0],ymm13[0],ymm6[2],ymm13[2] -; AVX-NEXT: vextractf128 $1, %ymm8, %xmm8 -; AVX-NEXT: vblendps $8, {{[-0-9]+}}(%r{{[sb]}}p), %xmm8, %xmm8 # 16-byte Folded Reload -; AVX-NEXT: # xmm8 = xmm8[0,1,2],mem[3] -; AVX-NEXT: vinsertf128 $1, %xmm8, %ymm0, %ymm8 -; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm8[5,6,7] +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload +; AVX-NEXT: vunpcklpd {{.*#+}} ymm11 = ymm13[0],ymm5[0],ymm13[2],ymm5[2] +; AVX-NEXT: vextractf128 $1, %ymm11, %xmm11 +; AVX-NEXT: vblendps $8, {{[-0-9]+}}(%r{{[sb]}}p), %xmm11, %xmm11 # 16-byte Folded Reload +; AVX-NEXT: # xmm11 = xmm11[0,1,2],mem[3] +; AVX-NEXT: vinsertf128 $1, %xmm11, %ymm0, %ymm11 +; AVX-NEXT: vblendps {{.*#+}} ymm4 = ymm4[0,1,2,3,4],ymm11[5,6,7] +; AVX-NEXT: vmovups %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vshufps {{.*#+}} ymm4 = ymm9[1,0],ymm8[0,0],ymm9[5,4],ymm8[4,4] +; AVX-NEXT: vshufps {{.*#+}} ymm4 = ymm8[3,1],ymm4[0,2],ymm8[7,5],ymm4[4,6] +; AVX-NEXT: vblendps $8, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm9 # 16-byte Folded Reload +; AVX-NEXT: # xmm9 = xmm0[0,1,2],mem[3] +; AVX-NEXT: vshufps {{.*#+}} xmm9 = xmm9[3,2,2,3] +; AVX-NEXT: vblendps {{.*#+}} ymm4 = ymm9[0,1],ymm4[2,3,4,5,6,7] +; AVX-NEXT: vmovaps 192(%rdi), %ymm0 ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm14 # 32-byte Reload +; AVX-NEXT: vshufps {{.*#+}} ymm11 = ymm0[0,1],ymm14[1,3],ymm0[4,5],ymm14[5,7] ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload -; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm0[1,0],ymm11[0,0],ymm0[5,4],ymm11[4,4] -; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm11[3,1],ymm0[0,2],ymm11[7,5],ymm0[4,6] -; AVX-NEXT: vblendps $8, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm8 # 16-byte Folded Reload -; AVX-NEXT: # xmm8 = xmm1[0,1,2],mem[3] +; AVX-NEXT: vshufps {{.*#+}} ymm11 = ymm0[0,2],ymm11[2,0],ymm0[4,6],ymm11[6,4] +; AVX-NEXT: vblendps {{.*#+}} ymm4 = ymm4[0,1,2,3,4],ymm11[5,6,7] +; AVX-NEXT: vmovups %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vshufps {{.*#+}} ymm4 = ymm10[1,0],ymm7[0,0],ymm10[5,4],ymm7[4,4] +; AVX-NEXT: vshufps {{.*#+}} ymm4 = ymm7[3,1],ymm4[0,2],ymm7[7,5],ymm4[4,6] +; AVX-NEXT: vmovaps %xmm15, %xmm10 +; AVX-NEXT: vblendps $8, {{[-0-9]+}}(%r{{[sb]}}p), %xmm15, %xmm8 # 16-byte Folded Reload +; AVX-NEXT: # xmm8 = xmm15[0,1,2],mem[3] ; AVX-NEXT: vshufps {{.*#+}} xmm8 = xmm8[3,2,2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm8[0,1],ymm0[2,3,4,5,6,7] -; AVX-NEXT: vmovaps 192(%rdi), %ymm1 -; AVX-NEXT: vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Reload -; AVX-NEXT: vshufps {{.*#+}} ymm8 = ymm1[0,1],ymm9[1,3],ymm1[4,5],ymm9[5,7] -; AVX-NEXT: vshufps {{.*#+}} ymm8 = ymm3[0,2],ymm8[2,0],ymm3[4,6],ymm8[6,4] -; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm8[5,6,7] +; AVX-NEXT: vblendps {{.*#+}} ymm8 = ymm8[0,1],ymm4[2,3,4,5,6,7] +; AVX-NEXT: vmovaps 640(%rdi), %ymm0 ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm10[1,0],ymm7[0,0],ymm10[5,4],ymm7[4,4] -; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm7[3,1],ymm0[0,2],ymm7[7,5],ymm0[4,6] -; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload -; AVX-NEXT: vblendps $8, {{[-0-9]+}}(%r{{[sb]}}p), %xmm11, %xmm8 # 16-byte Folded Reload -; AVX-NEXT: # xmm8 = xmm11[0,1,2],mem[3] +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload +; AVX-NEXT: vshufps {{.*#+}} ymm11 = ymm0[0,1],ymm4[1,3],ymm0[4,5],ymm4[5,7] +; AVX-NEXT: vshufps {{.*#+}} ymm11 = ymm12[0,2],ymm11[2,0],ymm12[4,6],ymm11[6,4] +; AVX-NEXT: vblendps {{.*#+}} ymm8 = ymm8[0,1,2,3,4],ymm11[5,6,7] +; AVX-NEXT: vmovups %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vshufps {{.*#+}} ymm7 = ymm3[1,0],ymm2[0,0],ymm3[5,4],ymm2[4,4] +; AVX-NEXT: vshufps {{.*#+}} ymm7 = ymm2[3,1],ymm7[0,2],ymm2[7,5],ymm7[4,6] +; AVX-NEXT: vblendps {{.*#+}} xmm8 = xmm1[0,1,2],xmm6[3] ; AVX-NEXT: vshufps {{.*#+}} xmm8 = xmm8[3,2,2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm8[0,1],ymm0[2,3,4,5,6,7] -; AVX-NEXT: vmovaps 640(%rdi), %ymm1 -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload -; AVX-NEXT: vshufps {{.*#+}} ymm10 = ymm1[0,1],ymm8[1,3],ymm1[4,5],ymm8[5,7] -; AVX-NEXT: vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vshufps {{.*#+}} ymm10 = ymm12[0,2],ymm10[2,0],ymm12[4,6],ymm10[6,4] -; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm10[5,6,7] -; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm14[1,0],ymm4[0,0],ymm14[5,4],ymm4[4,4] -; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm4[3,1],ymm0[0,2],ymm4[7,5],ymm0[4,6] -; AVX-NEXT: vblendps {{.*#+}} xmm7 = xmm2[0,1,2],xmm5[3] -; AVX-NEXT: vshufps {{.*#+}} xmm7 = xmm7[3,2,2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm7[0,1],ymm0[2,3,4,5,6,7] -; AVX-NEXT: vmovaps 416(%rdi), %ymm14 -; AVX-NEXT: vshufps {{.*#+}} ymm10 = ymm14[0,1],ymm13[1,3],ymm14[4,5],ymm13[5,7] -; AVX-NEXT: vmovups %ymm14, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vshufps {{.*#+}} ymm10 = ymm6[0,2],ymm10[2,0],ymm6[4,6],ymm10[6,4] -; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm10[5,6,7] +; AVX-NEXT: vblendps {{.*#+}} ymm7 = ymm8[0,1],ymm7[2,3,4,5,6,7] +; AVX-NEXT: vmovaps 416(%rdi), %ymm15 +; AVX-NEXT: vmovaps %ymm5, %ymm8 +; AVX-NEXT: vshufps {{.*#+}} ymm11 = ymm15[0,1],ymm5[1,3],ymm15[4,5],ymm5[5,7] +; AVX-NEXT: vshufps {{.*#+}} ymm11 = ymm13[0,2],ymm11[2,0],ymm13[4,6],ymm11[6,4] +; AVX-NEXT: vblendps {{.*#+}} ymm7 = ymm7[0,1,2,3,4],ymm11[5,6,7] +; AVX-NEXT: vmovups %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload +; AVX-NEXT: vshufps {{.*#+}} ymm5 = ymm1[1,0],ymm0[0,0],ymm1[5,4],ymm0[4,4] +; AVX-NEXT: vshufps {{.*#+}} ymm5 = ymm0[3,1],ymm5[0,2],ymm0[7,5],ymm5[4,6] +; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload +; AVX-NEXT: vblendps $7, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm6 # 16-byte Folded Reload +; AVX-NEXT: # xmm6 = mem[0,1,2],xmm0[3] +; AVX-NEXT: vshufps {{.*#+}} xmm6 = xmm6[3,2,2,3] +; AVX-NEXT: vblendps {{.*#+}} ymm6 = ymm6[0,1],ymm5[2,3,4,5,6,7] +; AVX-NEXT: vmovaps 864(%rdi), %ymm0 ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Reload +; AVX-NEXT: vshufps {{.*#+}} ymm7 = ymm0[0,1],ymm9[1,3],ymm0[4,5],ymm9[5,7] ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload -; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm0[1,0],ymm2[0,0],ymm0[5,4],ymm2[4,4] -; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm2[3,1],ymm0[0,2],ymm2[7,5],ymm0[4,6] -; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload -; AVX-NEXT: vblendps $7, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2, %xmm5 # 16-byte Folded Reload -; AVX-NEXT: # xmm5 = mem[0,1,2],xmm2[3] -; AVX-NEXT: vshufps {{.*#+}} xmm5 = xmm5[3,2,2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm5[0,1],ymm0[2,3,4,5,6,7] -; AVX-NEXT: vmovaps 864(%rdi), %ymm5 -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload -; AVX-NEXT: vshufps {{.*#+}} ymm6 = ymm5[0,1],ymm7[1,3],ymm5[4,5],ymm7[5,7] -; AVX-NEXT: vshufps {{.*#+}} ymm6 = ymm15[0,2],ymm6[2,0],ymm15[4,6],ymm6[6,4] -; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm6[5,6,7] +; AVX-NEXT: vshufps {{.*#+}} ymm7 = ymm0[0,2],ymm7[2,0],ymm0[4,6],ymm7[6,4] +; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm6[0,1,2,3,4],ymm7[5,6,7] ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vperm2f128 {{.*#+}} ymm0 = ymm12[2,3,0,1] -; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm12[3,0],ymm0[0,0],ymm12[7,4],ymm0[4,4] -; AVX-NEXT: vshufps {{.*#+}} ymm3 = ymm1[1,0],ymm8[2,0],ymm1[5,4],ymm8[6,4] -; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm0[2,0],ymm3[2,0],ymm0[6,4],ymm3[6,4] +; AVX-NEXT: vperm2f128 {{.*#+}} ymm6 = ymm12[2,3,0,1] +; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm12[3,0],ymm6[0,0],ymm12[7,4],ymm6[4,4] +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload +; AVX-NEXT: vshufps {{.*#+}} ymm6 = ymm5[1,0],ymm4[2,0],ymm5[5,4],ymm4[6,4] +; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm0[2,0],ymm6[2,0],ymm0[6,4],ymm6[6,4] ; AVX-NEXT: vmovaps 544(%rdi), %xmm1 ; AVX-NEXT: vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; AVX-NEXT: vshufps {{.*#+}} xmm10 = xmm1[0,1,0,1] -; AVX-NEXT: vmovaps 512(%rdi), %xmm6 -; AVX-NEXT: vblendps {{.*#+}} xmm10 = xmm6[0,1,2],xmm10[3] -; AVX-NEXT: vshufps {{.*#+}} xmm12 = xmm11[2,3,2,3] -; AVX-NEXT: vblendps {{.*#+}} xmm12 = mem[0],xmm12[1],mem[2,3] -; AVX-NEXT: vblendps {{.*#+}} xmm10 = xmm12[0,1],xmm10[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm10[0,1,2,3],ymm0[4,5,6,7] +; AVX-NEXT: vshufps {{.*#+}} xmm11 = xmm1[0,1,0,1] +; AVX-NEXT: vmovaps 512(%rdi), %xmm7 +; AVX-NEXT: vblendps {{.*#+}} xmm11 = xmm7[0,1,2],xmm11[3] +; AVX-NEXT: vshufps {{.*#+}} xmm1 = xmm10[2,3,2,3] +; AVX-NEXT: vblendps {{.*#+}} xmm1 = mem[0],xmm1[1],mem[2,3] +; AVX-NEXT: vblendps {{.*#+}} xmm1 = xmm1[0,1],xmm11[2,3] +; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7] ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vmovups (%rsp), %ymm2 # 32-byte Reload -; AVX-NEXT: vperm2f128 {{.*#+}} ymm0 = ymm2[2,3,0,1] -; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm2[3,0],ymm0[0,0],ymm2[7,4],ymm0[4,4] -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload -; AVX-NEXT: vshufps {{.*#+}} ymm4 = ymm3[1,0],ymm9[2,0],ymm3[5,4],ymm9[6,4] -; AVX-NEXT: vshufps {{.*#+}} ymm10 = ymm0[2,0],ymm4[2,0],ymm0[6,4],ymm4[6,4] -; AVX-NEXT: vmovaps 64(%rdi), %xmm0 -; AVX-NEXT: vmovaps 96(%rdi), %xmm4 -; AVX-NEXT: vshufps {{.*#+}} xmm12 = xmm4[0,1,0,1] -; AVX-NEXT: vmovaps %xmm4, (%rsp) # 16-byte Spill -; AVX-NEXT: vblendps {{.*#+}} xmm12 = xmm0[0,1,2],xmm12[3] -; AVX-NEXT: vpermilps $238, {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Folded Reload -; AVX-NEXT: # xmm13 = mem[2,3,2,3] -; AVX-NEXT: vblendps {{.*#+}} xmm13 = mem[0],xmm13[1],mem[2,3] -; AVX-NEXT: vblendps {{.*#+}} xmm12 = xmm13[0,1],xmm12[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm1 = ymm12[0,1,2,3],ymm10[4,5,6,7] -; AVX-NEXT: vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload -; AVX-NEXT: vperm2f128 {{.*#+}} ymm10 = ymm1[2,3,0,1] -; AVX-NEXT: vshufps {{.*#+}} ymm2 = ymm1[3,0],ymm10[0,0],ymm1[7,4],ymm10[4,4] -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm11 # 32-byte Reload -; AVX-NEXT: vshufps {{.*#+}} ymm10 = ymm14[1,0],ymm11[2,0],ymm14[5,4],ymm11[6,4] -; AVX-NEXT: vshufps {{.*#+}} ymm12 = ymm2[2,0],ymm10[2,0],ymm2[6,4],ymm10[6,4] -; AVX-NEXT: vmovaps 320(%rdi), %xmm1 -; AVX-NEXT: vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; AVX-NEXT: vshufps {{.*#+}} xmm13 = xmm1[0,1,0,1] -; AVX-NEXT: vmovaps 288(%rdi), %xmm2 -; AVX-NEXT: vblendps {{.*#+}} xmm13 = xmm2[0,1,2],xmm13[3] +; AVX-NEXT: vperm2f128 {{.*#+}} ymm0 = ymm1[2,3,0,1] +; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm1[3,0],ymm0[0,0],ymm1[7,4],ymm0[4,4] +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Reload +; AVX-NEXT: vshufps {{.*#+}} ymm1 = ymm10[1,0],ymm14[2,0],ymm10[5,4],ymm14[6,4] +; AVX-NEXT: vmovaps %ymm14, %ymm6 +; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm0[2,0],ymm1[2,0],ymm0[6,4],ymm1[6,4] +; AVX-NEXT: vmovaps 64(%rdi), %xmm1 +; AVX-NEXT: vmovaps 96(%rdi), %xmm4 +; AVX-NEXT: vshufps {{.*#+}} xmm3 = xmm4[0,1,0,1] +; AVX-NEXT: vmovaps %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; AVX-NEXT: vblendps {{.*#+}} xmm3 = xmm1[0,1,2],xmm3[3] ; AVX-NEXT: vpermilps $238, {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Folded Reload ; AVX-NEXT: # xmm14 = mem[2,3,2,3] ; AVX-NEXT: vblendps {{.*#+}} xmm14 = mem[0],xmm14[1],mem[2,3] -; AVX-NEXT: vblendps {{.*#+}} xmm13 = xmm14[0,1],xmm13[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm1 = ymm13[0,1,2,3],ymm12[4,5,6,7] -; AVX-NEXT: vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vperm2f128 {{.*#+}} ymm12 = ymm15[2,3,0,1] -; AVX-NEXT: vshufps {{.*#+}} ymm1 = ymm15[3,0],ymm12[0,0],ymm15[7,4],ymm12[4,4] -; AVX-NEXT: vshufps {{.*#+}} ymm12 = ymm5[1,0],ymm7[2,0],ymm5[5,4],ymm7[6,4] -; AVX-NEXT: vshufps {{.*#+}} ymm12 = ymm1[2,0],ymm12[2,0],ymm1[6,4],ymm12[6,4] -; AVX-NEXT: vmovaps 768(%rdi), %xmm1 -; AVX-NEXT: vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; AVX-NEXT: vshufps {{.*#+}} xmm14 = xmm1[0,1,0,1] -; AVX-NEXT: vmovaps 736(%rdi), %xmm13 -; AVX-NEXT: vblendps {{.*#+}} xmm14 = xmm13[0,1,2],xmm14[3] +; AVX-NEXT: vblendps {{.*#+}} xmm3 = xmm14[0,1],xmm3[2,3] +; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm3[0,1,2,3],ymm0[4,5,6,7] +; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vperm2f128 {{.*#+}} ymm0 = ymm13[2,3,0,1] +; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm13[3,0],ymm0[0,0],ymm13[7,4],ymm0[4,4] +; AVX-NEXT: vshufps {{.*#+}} ymm3 = ymm15[1,0],ymm8[2,0],ymm15[5,4],ymm8[6,4] +; AVX-NEXT: vmovaps %ymm15, %ymm11 +; AVX-NEXT: vmovups %ymm15, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm0[2,0],ymm3[2,0],ymm0[6,4],ymm3[6,4] +; AVX-NEXT: vmovaps 320(%rdi), %xmm2 +; AVX-NEXT: vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; AVX-NEXT: vshufps {{.*#+}} xmm14 = xmm2[0,1,0,1] +; AVX-NEXT: vmovaps 288(%rdi), %xmm3 +; AVX-NEXT: vblendps {{.*#+}} xmm14 = xmm3[0,1,2],xmm14[3] ; AVX-NEXT: vpermilps $238, {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Folded Reload ; AVX-NEXT: # xmm15 = mem[2,3,2,3] ; AVX-NEXT: vblendps {{.*#+}} xmm15 = mem[0],xmm15[1],mem[2,3] ; AVX-NEXT: vblendps {{.*#+}} xmm14 = xmm15[0,1],xmm14[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm1 = ymm14[0,1,2,3],ymm12[4,5,6,7] -; AVX-NEXT: vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vshufps {{.*#+}} ymm12 = ymm3[2,1],ymm9[3,3],ymm3[6,5],ymm9[7,7] -; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload -; AVX-NEXT: vblendps $2, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm15 # 16-byte Folded Reload -; AVX-NEXT: # xmm15 = xmm1[0],mem[1],xmm1[2,3] -; AVX-NEXT: vinsertf128 $1, %xmm15, %ymm0, %ymm15 -; AVX-NEXT: vshufps {{.*#+}} ymm12 = ymm15[1,0],ymm12[2,0],ymm15[5,4],ymm12[6,4] -; AVX-NEXT: vblendps {{.*#+}} xmm0 = xmm4[0,1,2],xmm0[3] -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload -; AVX-NEXT: vshufps $16, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm15 # 32-byte Folded Reload -; AVX-NEXT: # ymm15 = ymm1[0,0],mem[1,0],ymm1[4,4],mem[5,4] -; AVX-NEXT: vshufps {{.*#+}} ymm15 = ymm15[2,0,2,3,6,4,6,7] -; AVX-NEXT: vextractf128 $1, %ymm15, %xmm15 -; AVX-NEXT: vshufps {{.*#+}} xmm0 = xmm15[0,1],xmm0[3,2] -; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm12[4,5,6,7] +; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm14[0,1,2,3],ymm0[4,5,6,7] ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload -; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm3[2,1],ymm8[3,3],ymm3[6,5],ymm8[7,7] -; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload -; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload -; AVX-NEXT: vblendps {{.*#+}} xmm15 = xmm8[0],xmm7[1],xmm8[2,3] -; AVX-NEXT: vinsertf128 $1, %xmm15, %ymm0, %ymm15 -; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm15[1,0],ymm0[2,0],ymm15[5,4],ymm0[6,4] -; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload -; AVX-NEXT: vblendps {{.*#+}} xmm6 = xmm10[0,1,2],xmm6[3] -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm12 # 32-byte Reload -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Reload -; AVX-NEXT: vshufps {{.*#+}} ymm15 = ymm9[0,0],ymm12[1,0],ymm9[4,4],ymm12[5,4] -; AVX-NEXT: vshufps {{.*#+}} ymm15 = ymm15[2,0,2,3,6,4,6,7] -; AVX-NEXT: vextractf128 $1, %ymm15, %xmm15 -; AVX-NEXT: vshufps {{.*#+}} xmm6 = xmm15[0,1],xmm6[3,2] -; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm6[0,1,2,3],ymm0[4,5,6,7] +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload +; AVX-NEXT: vperm2f128 {{.*#+}} ymm0 = ymm2[2,3,0,1] +; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm2[3,0],ymm0[0,0],ymm2[7,4],ymm0[4,4] +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload +; AVX-NEXT: vshufps {{.*#+}} ymm2 = ymm8[1,0],ymm9[2,0],ymm8[5,4],ymm9[6,4] +; AVX-NEXT: vshufps {{.*#+}} ymm14 = ymm0[2,0],ymm2[2,0],ymm0[6,4],ymm2[6,4] +; AVX-NEXT: vmovaps 768(%rdi), %xmm0 +; AVX-NEXT: vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; AVX-NEXT: vshufps {{.*#+}} xmm15 = xmm0[0,1,0,1] +; AVX-NEXT: vmovaps 736(%rdi), %xmm2 +; AVX-NEXT: vblendps {{.*#+}} xmm15 = xmm2[0,1,2],xmm15[3] +; AVX-NEXT: vpermilps $238, {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Folded Reload +; AVX-NEXT: # xmm13 = mem[2,3,2,3] +; AVX-NEXT: vblendps {{.*#+}} xmm13 = mem[0],xmm13[1],mem[2,3] +; AVX-NEXT: vblendps {{.*#+}} xmm13 = xmm13[0,1],xmm15[2,3] +; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm13[0,1,2,3],ymm14[4,5,6,7] ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vshufps {{.*#+}} ymm13 = ymm10[2,1],ymm6[3,3],ymm10[6,5],ymm6[7,7] +; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload +; AVX-NEXT: vblendps $2, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm14 # 16-byte Folded Reload +; AVX-NEXT: # xmm14 = xmm0[0],mem[1],xmm0[2,3] +; AVX-NEXT: vinsertf128 $1, %xmm14, %ymm0, %ymm14 +; AVX-NEXT: vshufps {{.*#+}} ymm13 = ymm14[1,0],ymm13[2,0],ymm14[5,4],ymm13[6,4] +; AVX-NEXT: vblendps {{.*#+}} xmm1 = xmm4[0,1,2],xmm1[3] ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload -; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm0[2,1],ymm11[3,3],ymm0[6,5],ymm11[7,7] -; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload -; AVX-NEXT: vblendps $13, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm15 # 16-byte Folded Reload -; AVX-NEXT: # xmm15 = mem[0],xmm1[1],mem[2,3] -; AVX-NEXT: vinsertf128 $1, %xmm15, %ymm0, %ymm15 -; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm15[1,0],ymm0[2,0],ymm15[5,4],ymm0[6,4] -; AVX-NEXT: vblendps $7, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2, %xmm2 # 16-byte Folded Reload -; AVX-NEXT: # xmm2 = mem[0,1,2],xmm2[3] -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload -; AVX-NEXT: vshufps $16, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm15 # 32-byte Folded Reload -; AVX-NEXT: # ymm15 = ymm1[0,0],mem[1,0],ymm1[4,4],mem[5,4] -; AVX-NEXT: vshufps {{.*#+}} ymm15 = ymm15[2,0,2,3,6,4,6,7] -; AVX-NEXT: vextractf128 $1, %ymm15, %xmm15 -; AVX-NEXT: vshufps {{.*#+}} xmm2 = xmm15[0,1],xmm2[3,2] -; AVX-NEXT: vblendps {{.*#+}} ymm15 = ymm2[0,1,2,3],ymm0[4,5,6,7] -; AVX-NEXT: vshufps $246, {{[-0-9]+}}(%r{{[sb]}}p), %ymm5, %ymm0 # 32-byte Folded Reload -; AVX-NEXT: # ymm0 = ymm5[2,1],mem[3,3],ymm5[6,5],mem[7,7] -; AVX-NEXT: vmovaps %ymm5, %ymm14 -; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload -; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload -; AVX-NEXT: vblendps {{.*#+}} xmm2 = xmm4[0],xmm1[1],xmm4[2,3] -; AVX-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm2 -; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm2[1,0],ymm0[2,0],ymm2[5,4],ymm0[6,4] +; AVX-NEXT: vshufps $16, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm14 # 32-byte Folded Reload +; AVX-NEXT: # ymm14 = ymm0[0,0],mem[1,0],ymm0[4,4],mem[5,4] +; AVX-NEXT: vextractf128 $1, %ymm14, %xmm14 +; AVX-NEXT: vshufps {{.*#+}} xmm1 = xmm14[2,0],xmm1[3,2] +; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm13[4,5,6,7] +; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vshufps $246, {{[-0-9]+}}(%r{{[sb]}}p), %ymm5, %ymm13 # 32-byte Folded Reload +; AVX-NEXT: # ymm13 = ymm5[2,1],mem[3,3],ymm5[6,5],mem[7,7] +; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload +; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload +; AVX-NEXT: vblendps {{.*#+}} xmm14 = xmm15[0],xmm10[1],xmm15[2,3] +; AVX-NEXT: vinsertf128 $1, %xmm14, %ymm0, %ymm14 +; AVX-NEXT: vshufps {{.*#+}} ymm13 = ymm14[1,0],ymm13[2,0],ymm14[5,4],ymm13[6,4] ; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload -; AVX-NEXT: vblendps {{.*#+}} xmm2 = xmm6[0,1,2],xmm13[3] -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm13 # 32-byte Reload -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload -; AVX-NEXT: vshufps {{.*#+}} ymm11 = ymm5[0,0],ymm13[1,0],ymm5[4,4],ymm13[5,4] -; AVX-NEXT: vshufps {{.*#+}} ymm11 = ymm11[2,0,2,3,6,4,6,7] -; AVX-NEXT: vextractf128 $1, %ymm11, %xmm11 -; AVX-NEXT: vshufps {{.*#+}} xmm2 = xmm11[0,1],xmm2[3,2] -; AVX-NEXT: vblendps {{.*#+}} ymm11 = ymm2[0,1,2,3],ymm0[4,5,6,7] -; AVX-NEXT: vperm2f128 {{.*#+}} ymm0 = ymm3[2,3,0,1] -; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm3[3,0],ymm0[0,0],ymm3[7,4],ymm0[4,4] -; AVX-NEXT: vshufps {{.*#+}} xmm2 = xmm7[2,3,2,3] -; AVX-NEXT: vblendps {{.*#+}} xmm2 = xmm2[0],xmm8[1],xmm2[2,3] -; AVX-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm2 -; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm2[0,1],ymm0[2,0],ymm2[4,5],ymm0[6,4] +; AVX-NEXT: vblendps {{.*#+}} xmm7 = xmm6[0,1,2],xmm7[3] +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload +; AVX-NEXT: vshufps {{.*#+}} ymm14 = ymm0[0,0],ymm4[1,0],ymm0[4,4],ymm4[5,4] +; AVX-NEXT: vextractf128 $1, %ymm14, %xmm14 +; AVX-NEXT: vshufps {{.*#+}} xmm7 = xmm14[2,0],xmm7[3,2] +; AVX-NEXT: vblendps {{.*#+}} ymm1 = ymm7[0,1,2,3],ymm13[4,5,6,7] +; AVX-NEXT: vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vshufps $246, {{[-0-9]+}}(%r{{[sb]}}p), %ymm11, %ymm13 # 32-byte Folded Reload +; AVX-NEXT: # ymm13 = ymm11[2,1],mem[3,3],ymm11[6,5],mem[7,7] +; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload +; AVX-NEXT: vblendps $13, {{[-0-9]+}}(%r{{[sb]}}p), %xmm7, %xmm14 # 16-byte Folded Reload +; AVX-NEXT: # xmm14 = mem[0],xmm7[1],mem[2,3] +; AVX-NEXT: vinsertf128 $1, %xmm14, %ymm0, %ymm14 +; AVX-NEXT: vshufps {{.*#+}} ymm13 = ymm14[1,0],ymm13[2,0],ymm14[5,4],ymm13[6,4] +; AVX-NEXT: vblendps $7, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3, %xmm3 # 16-byte Folded Reload +; AVX-NEXT: # xmm3 = mem[0,1,2],xmm3[3] +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm11 # 32-byte Reload +; AVX-NEXT: vshufps $16, {{[-0-9]+}}(%r{{[sb]}}p), %ymm11, %ymm14 # 32-byte Folded Reload +; AVX-NEXT: # ymm14 = ymm11[0,0],mem[1,0],ymm11[4,4],mem[5,4] +; AVX-NEXT: vextractf128 $1, %ymm14, %xmm14 +; AVX-NEXT: vshufps {{.*#+}} xmm3 = xmm14[2,0],xmm3[3,2] +; AVX-NEXT: vblendps {{.*#+}} ymm13 = ymm3[0,1,2,3],ymm13[4,5,6,7] +; AVX-NEXT: vmovaps %ymm8, %ymm7 +; AVX-NEXT: vshufps {{.*#+}} ymm3 = ymm8[2,1],ymm9[3,3],ymm8[6,5],ymm9[7,7] +; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload +; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload +; AVX-NEXT: vblendps {{.*#+}} xmm12 = xmm9[0],xmm8[1],xmm9[2,3] +; AVX-NEXT: vinsertf128 $1, %xmm12, %ymm0, %ymm12 +; AVX-NEXT: vshufps {{.*#+}} ymm3 = ymm12[1,0],ymm3[2,0],ymm12[5,4],ymm3[6,4] +; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload +; AVX-NEXT: vblendps {{.*#+}} xmm2 = xmm5[0,1,2],xmm2[3] +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm14 # 32-byte Reload +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload +; AVX-NEXT: vshufps {{.*#+}} ymm12 = ymm1[0,0],ymm14[1,0],ymm1[4,4],ymm14[5,4] +; AVX-NEXT: vextractf128 $1, %ymm12, %xmm12 +; AVX-NEXT: vshufps {{.*#+}} xmm2 = xmm12[2,0],xmm2[3,2] +; AVX-NEXT: vblendps {{.*#+}} ymm12 = ymm2[0,1,2,3],ymm3[4,5,6,7] ; AVX-NEXT: vpermilps $68, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload ; AVX-NEXT: # xmm2 = mem[0,1,0,1] -; AVX-NEXT: vblendps {{.*#+}} xmm2 = xmm2[0,1,2],xmm10[3] -; AVX-NEXT: vshufps {{.*#+}} ymm3 = ymm9[1,0],ymm12[2,0],ymm9[5,4],ymm12[6,4] -; AVX-NEXT: vshufps {{.*#+}} ymm3 = ymm3[2,0,2,3,6,4,6,7] +; AVX-NEXT: vblendps {{.*#+}} xmm2 = xmm2[0,1,2],xmm6[3] +; AVX-NEXT: vshufps {{.*#+}} ymm3 = ymm0[1,0],ymm4[2,0],ymm0[5,4],ymm4[6,4] ; AVX-NEXT: vextractf128 $1, %ymm3, %xmm3 -; AVX-NEXT: vblendps {{.*#+}} xmm2 = xmm3[0,1],xmm2[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm3 = ymm2[0,1,2,3],ymm0[4,5,6,7] -; AVX-NEXT: vperm2f128 {{.*#+}} ymm0 = ymm14[2,3,0,1] -; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm14[3,0],ymm0[0,0],ymm14[7,4],ymm0[4,4] -; AVX-NEXT: vshufps {{.*#+}} xmm2 = xmm1[2,3,2,3] -; AVX-NEXT: vblendps {{.*#+}} xmm2 = xmm2[0],xmm4[1],xmm2[2,3] -; AVX-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm2 -; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm2[0,1],ymm0[2,0],ymm2[4,5],ymm0[6,4] +; AVX-NEXT: vshufps {{.*#+}} xmm2 = xmm3[2,0],xmm2[2,3] +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload +; AVX-NEXT: vperm2f128 {{.*#+}} ymm3 = ymm0[2,3,0,1] +; AVX-NEXT: vshufps {{.*#+}} ymm3 = ymm0[3,0],ymm3[0,0],ymm0[7,4],ymm3[4,4] +; AVX-NEXT: vshufps {{.*#+}} xmm4 = xmm10[2,3,2,3] +; AVX-NEXT: vblendps {{.*#+}} xmm4 = xmm4[0],xmm15[1],xmm4[2,3] +; AVX-NEXT: vinsertf128 $1, %xmm4, %ymm0, %ymm4 +; AVX-NEXT: vshufps {{.*#+}} ymm3 = ymm4[0,1],ymm3[2,0],ymm4[4,5],ymm3[6,4] +; AVX-NEXT: vblendps {{.*#+}} ymm3 = ymm2[0,1,2,3],ymm3[4,5,6,7] ; AVX-NEXT: vpermilps $68, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload ; AVX-NEXT: # xmm2 = mem[0,1,0,1] -; AVX-NEXT: vblendps {{.*#+}} xmm1 = xmm2[0,1,2],xmm6[3] -; AVX-NEXT: vshufps {{.*#+}} ymm2 = ymm5[1,0],ymm13[2,0],ymm5[5,4],ymm13[6,4] -; AVX-NEXT: vshufps {{.*#+}} ymm2 = ymm2[2,0,2,3,6,4,6,7] +; AVX-NEXT: vblendps {{.*#+}} xmm0 = xmm2[0,1,2],xmm5[3] +; AVX-NEXT: vshufps {{.*#+}} ymm2 = ymm1[1,0],ymm14[2,0],ymm1[5,4],ymm14[6,4] ; AVX-NEXT: vextractf128 $1, %ymm2, %xmm2 -; AVX-NEXT: vblendps {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7] -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload -; AVX-NEXT: vperm2f128 {{.*#+}} ymm1 = ymm2[2,3,0,1] -; AVX-NEXT: vshufps {{.*#+}} ymm1 = ymm2[3,0],ymm1[0,0],ymm2[7,4],ymm1[4,4] -; AVX-NEXT: vpermilps $238, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload -; AVX-NEXT: # xmm2 = mem[2,3,2,3] -; AVX-NEXT: vblendps $2, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2, %xmm2 # 16-byte Folded Reload -; AVX-NEXT: # xmm2 = xmm2[0],mem[1],xmm2[2,3] -; AVX-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm2 -; AVX-NEXT: vshufps {{.*#+}} ymm1 = ymm2[0,1],ymm1[2,0],ymm2[4,5],ymm1[6,4] +; AVX-NEXT: vshufps {{.*#+}} xmm0 = xmm2[2,0],xmm0[2,3] +; AVX-NEXT: vperm2f128 {{.*#+}} ymm2 = ymm7[2,3,0,1] +; AVX-NEXT: vshufps {{.*#+}} ymm2 = ymm7[3,0],ymm2[0,0],ymm7[7,4],ymm2[4,4] +; AVX-NEXT: vshufps {{.*#+}} xmm4 = xmm8[2,3,2,3] +; AVX-NEXT: vblendps {{.*#+}} xmm4 = xmm4[0],xmm9[1],xmm4[2,3] +; AVX-NEXT: vinsertf128 $1, %xmm4, %ymm0, %ymm4 +; AVX-NEXT: vshufps {{.*#+}} ymm2 = ymm4[0,1],ymm2[2,0],ymm4[4,5],ymm2[6,4] +; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm2[4,5,6,7] ; AVX-NEXT: vpermilps $68, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload ; AVX-NEXT: # xmm2 = mem[0,1,0,1] -; AVX-NEXT: vblendps $8, (%rsp), %xmm2, %xmm2 # 16-byte Folded Reload +; AVX-NEXT: vblendps $8, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2, %xmm2 # 16-byte Folded Reload ; AVX-NEXT: # xmm2 = xmm2[0,1,2],mem[3] -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload -; AVX-NEXT: vshufps $33, {{[-0-9]+}}(%r{{[sb]}}p), %ymm4, %ymm4 # 32-byte Folded Reload -; AVX-NEXT: # ymm4 = ymm4[1,0],mem[2,0],ymm4[5,4],mem[6,4] -; AVX-NEXT: vshufps {{.*#+}} ymm4 = ymm4[2,0,2,3,6,4,6,7] +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload +; AVX-NEXT: vshufps $33, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm4 # 32-byte Folded Reload +; AVX-NEXT: # ymm4 = ymm1[1,0],mem[2,0],ymm1[5,4],mem[6,4] ; AVX-NEXT: vextractf128 $1, %ymm4, %xmm4 -; AVX-NEXT: vblendps {{.*#+}} xmm2 = xmm4[0,1],xmm2[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7] -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload -; AVX-NEXT: vperm2f128 {{.*#+}} ymm2 = ymm4[2,3,0,1] -; AVX-NEXT: vshufps {{.*#+}} ymm2 = ymm4[3,0],ymm2[0,0],ymm4[7,4],ymm2[4,4] -; AVX-NEXT: vpermilps $238, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload -; AVX-NEXT: # xmm4 = mem[2,3,2,3] -; AVX-NEXT: vblendps $2, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4, %xmm4 # 16-byte Folded Reload -; AVX-NEXT: # xmm4 = xmm4[0],mem[1],xmm4[2,3] -; AVX-NEXT: vinsertf128 $1, %xmm4, %ymm0, %ymm4 -; AVX-NEXT: vshufps {{.*#+}} ymm2 = ymm4[0,1],ymm2[2,0],ymm4[4,5],ymm2[6,4] +; AVX-NEXT: vshufps {{.*#+}} xmm2 = xmm4[2,0],xmm2[2,3] +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload +; AVX-NEXT: vperm2f128 {{.*#+}} ymm4 = ymm1[2,3,0,1] +; AVX-NEXT: vshufps {{.*#+}} ymm4 = ymm1[3,0],ymm4[0,0],ymm1[7,4],ymm4[4,4] +; AVX-NEXT: vpermilps $238, {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Folded Reload +; AVX-NEXT: # xmm5 = mem[2,3,2,3] +; AVX-NEXT: vblendps $2, {{[-0-9]+}}(%r{{[sb]}}p), %xmm5, %xmm5 # 16-byte Folded Reload +; AVX-NEXT: # xmm5 = xmm5[0],mem[1],xmm5[2,3] +; AVX-NEXT: vinsertf128 $1, %xmm5, %ymm0, %ymm5 +; AVX-NEXT: vshufps {{.*#+}} ymm4 = ymm5[0,1],ymm4[2,0],ymm5[4,5],ymm4[6,4] +; AVX-NEXT: vblendps {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm4[4,5,6,7] ; AVX-NEXT: vpermilps $68, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload ; AVX-NEXT: # xmm4 = mem[0,1,0,1] ; AVX-NEXT: vblendps $8, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4, %xmm4 # 16-byte Folded Reload ; AVX-NEXT: # xmm4 = xmm4[0,1,2],mem[3] -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload -; AVX-NEXT: vshufps $33, {{[-0-9]+}}(%r{{[sb]}}p), %ymm5, %ymm5 # 32-byte Folded Reload -; AVX-NEXT: # ymm5 = ymm5[1,0],mem[2,0],ymm5[5,4],mem[6,4] -; AVX-NEXT: vshufps {{.*#+}} ymm5 = ymm5[2,0,2,3,6,4,6,7] +; AVX-NEXT: vshufps $33, {{[-0-9]+}}(%r{{[sb]}}p), %ymm11, %ymm5 # 32-byte Folded Reload +; AVX-NEXT: # ymm5 = ymm11[1,0],mem[2,0],ymm11[5,4],mem[6,4] ; AVX-NEXT: vextractf128 $1, %ymm5, %xmm5 -; AVX-NEXT: vblendps {{.*#+}} xmm4 = xmm5[0,1],xmm4[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm2 = ymm4[0,1,2,3],ymm2[4,5,6,7] -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm4, 96(%rsi) -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm4, 32(%rsi) -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm4, 64(%rsi) -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm4, (%rsi) -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm4, 96(%rdx) -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm4, 32(%rdx) -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm4, 64(%rdx) -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm4, (%rdx) -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm4, 32(%rcx) -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm4, 96(%rcx) -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm4, 64(%rcx) -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm4, (%rcx) -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm4, 96(%r8) -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm4, 32(%r8) -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm4, 64(%r8) -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm4, (%r8) -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm4, 96(%r9) -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm4, 32(%r9) -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm4, (%r9) -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm4, 64(%r9) -; AVX-NEXT: movq {{[0-9]+}}(%rsp), %rax -; AVX-NEXT: vmovaps %ymm11, 96(%rax) -; AVX-NEXT: vmovaps %ymm15, 32(%rax) -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm4, 64(%rax) -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm4, (%rax) +; AVX-NEXT: vshufps {{.*#+}} xmm4 = xmm5[2,0],xmm4[2,3] +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload +; AVX-NEXT: vperm2f128 {{.*#+}} ymm5 = ymm1[2,3,0,1] +; AVX-NEXT: vshufps {{.*#+}} ymm5 = ymm1[3,0],ymm5[0,0],ymm1[7,4],ymm5[4,4] +; AVX-NEXT: vpermilps $238, {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Folded Reload +; AVX-NEXT: # xmm6 = mem[2,3,2,3] +; AVX-NEXT: vblendps $2, {{[-0-9]+}}(%r{{[sb]}}p), %xmm6, %xmm6 # 16-byte Folded Reload +; AVX-NEXT: # xmm6 = xmm6[0],mem[1],xmm6[2,3] +; AVX-NEXT: vinsertf128 $1, %xmm6, %ymm0, %ymm6 +; AVX-NEXT: vshufps {{.*#+}} ymm5 = ymm6[0,1],ymm5[2,0],ymm6[4,5],ymm5[6,4] +; AVX-NEXT: vblendps {{.*#+}} ymm4 = ymm4[0,1,2,3],ymm5[4,5,6,7] +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm1, 96(%rsi) +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm1, 32(%rsi) +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm5, 64(%rsi) +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm5, (%rsi) +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm1, 96(%rdx) +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm1, 32(%rdx) +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm1, 64(%rdx) +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm1, (%rdx) +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm1, 32(%rcx) +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm1, 96(%rcx) +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm1, 64(%rcx) +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm1, (%rcx) +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm1, 96(%r8) +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm1, 32(%r8) +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm1, 64(%r8) +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm1, (%r8) +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm1, 96(%r9) +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm1, 32(%r9) +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm1, (%r9) +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm1, 64(%r9) ; AVX-NEXT: movq {{[0-9]+}}(%rsp), %rax -; AVX-NEXT: vmovaps %ymm2, 32(%rax) +; AVX-NEXT: vmovaps %ymm12, 96(%rax) +; AVX-NEXT: vmovaps %ymm13, 32(%rax) +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm1, 64(%rax) +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload ; AVX-NEXT: vmovaps %ymm1, (%rax) +; AVX-NEXT: movq {{[0-9]+}}(%rsp), %rax +; AVX-NEXT: vmovaps %ymm4, 32(%rax) +; AVX-NEXT: vmovaps %ymm2, (%rax) ; AVX-NEXT: vmovaps %ymm0, 96(%rax) ; AVX-NEXT: vmovaps %ymm3, 64(%rax) -; AVX-NEXT: addq $1464, %rsp # imm = 0x5B8 +; AVX-NEXT: addq $1432, %rsp # imm = 0x598 ; AVX-NEXT: vzeroupper ; AVX-NEXT: retq ; @@ -9949,27 +9936,27 @@ define void @load_i32_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt ; AVX-NEXT: vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX-NEXT: vmovaps 672(%rdi), %ymm3 ; AVX-NEXT: vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vmovaps 768(%rdi), %ymm5 +; AVX-NEXT: vmovaps 768(%rdi), %ymm11 ; AVX-NEXT: vmovaps 256(%rdi), %ymm4 ; AVX-NEXT: vmovups %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX-NEXT: vmovaps 224(%rdi), %ymm1 ; AVX-NEXT: vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vmovaps 320(%rdi), %ymm7 +; AVX-NEXT: vmovaps 320(%rdi), %ymm6 ; AVX-NEXT: vmovaps 304(%rdi), %xmm0 ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vunpcklpd {{.*#+}} ymm0 = ymm7[0],ymm0[0],ymm7[2],ymm0[2] -; AVX-NEXT: vmovups %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vunpcklpd {{.*#+}} ymm0 = ymm6[0],ymm0[0],ymm6[2],ymm0[2] +; AVX-NEXT: vmovups %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX-NEXT: vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5],ymm4[6],ymm1[7] ; AVX-NEXT: vextractf128 $1, %ymm1, %xmm1 -; AVX-NEXT: vmovaps 224(%rdi), %xmm13 -; AVX-NEXT: vblendps {{.*#+}} xmm1 = xmm13[0,1],xmm1[2,3] +; AVX-NEXT: vmovaps 224(%rdi), %xmm10 +; AVX-NEXT: vblendps {{.*#+}} xmm1 = xmm10[0,1],xmm1[2,3] ; AVX-NEXT: vshufps {{.*#+}} xmm1 = xmm1[0,3,2,3] ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2],ymm0[3,4,5,6,7] -; AVX-NEXT: vmovaps 384(%rdi), %xmm4 -; AVX-NEXT: vmovaps %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; AVX-NEXT: vmovaps 352(%rdi), %xmm1 +; AVX-NEXT: vmovaps 384(%rdi), %xmm1 ; AVX-NEXT: vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; AVX-NEXT: vunpckhpd {{.*#+}} xmm1 = xmm1[1],xmm4[1] +; AVX-NEXT: vmovaps 352(%rdi), %xmm4 +; AVX-NEXT: vmovaps %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; AVX-NEXT: vunpckhpd {{.*#+}} xmm1 = xmm4[1],xmm1[1] ; AVX-NEXT: vmovaps 416(%rdi), %xmm4 ; AVX-NEXT: vmovaps %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; AVX-NEXT: vinsertps {{.*#+}} xmm1 = zero,xmm1[1,2],xmm4[1] @@ -9978,22 +9965,21 @@ define void @load_i32_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX-NEXT: vmovaps 752(%rdi), %xmm0 ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vunpcklpd {{.*#+}} ymm0 = ymm5[0],ymm0[0],ymm5[2],ymm0[2] -; AVX-NEXT: vmovups %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vunpcklpd {{.*#+}} ymm0 = ymm11[0],ymm0[0],ymm11[2],ymm0[2] +; AVX-NEXT: vmovups %ymm11, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX-NEXT: vblendps {{.*#+}} ymm1 = ymm3[0,1,2,3,4,5],ymm2[6],ymm3[7] ; AVX-NEXT: vextractf128 $1, %ymm1, %xmm1 -; AVX-NEXT: vmovaps 672(%rdi), %xmm3 -; AVX-NEXT: vblendps {{.*#+}} xmm1 = xmm3[0,1],xmm1[2,3] +; AVX-NEXT: vmovaps 672(%rdi), %xmm15 +; AVX-NEXT: vblendps {{.*#+}} xmm1 = xmm15[0,1],xmm1[2,3] ; AVX-NEXT: vshufps {{.*#+}} xmm1 = xmm1[0,3,2,3] ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2],ymm0[3,4,5,6,7] -; AVX-NEXT: vmovaps 832(%rdi), %xmm2 -; AVX-NEXT: vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; AVX-NEXT: vmovaps 800(%rdi), %xmm1 +; AVX-NEXT: vmovaps 832(%rdi), %xmm1 ; AVX-NEXT: vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; AVX-NEXT: vunpckhpd {{.*#+}} xmm1 = xmm1[1],xmm2[1] -; AVX-NEXT: vmovaps 864(%rdi), %xmm2 +; AVX-NEXT: vmovaps 800(%rdi), %xmm2 ; AVX-NEXT: vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; AVX-NEXT: vinsertps {{.*#+}} xmm1 = zero,xmm1[1,2],xmm2[1] +; AVX-NEXT: vunpckhpd {{.*#+}} xmm1 = xmm2[1],xmm1[1] +; AVX-NEXT: vmovaps 864(%rdi), %xmm13 +; AVX-NEXT: vinsertps {{.*#+}} xmm1 = zero,xmm1[1,2],xmm13[1] ; AVX-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm1 ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm1[5,6,7] ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill @@ -10051,27 +10037,27 @@ define void @load_i32_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt ; AVX-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm1 ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm1[5,6,7] ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vmovaps 32(%rdi), %ymm0 -; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vmovaps (%rdi), %ymm1 +; AVX-NEXT: vmovaps 32(%rdi), %ymm1 ; AVX-NEXT: vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6],ymm1[7] +; AVX-NEXT: vmovaps (%rdi), %ymm0 +; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6],ymm0[7] ; AVX-NEXT: vextractf128 $1, %ymm0, %xmm0 ; AVX-NEXT: vmovaps (%rdi), %xmm1 ; AVX-NEXT: vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; AVX-NEXT: vblendps {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3] ; AVX-NEXT: vshufps {{.*#+}} xmm0 = xmm0[0,3,2,3] -; AVX-NEXT: vmovaps 96(%rdi), %ymm15 +; AVX-NEXT: vmovaps 96(%rdi), %ymm14 ; AVX-NEXT: vmovaps 80(%rdi), %xmm1 ; AVX-NEXT: vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vunpcklpd {{.*#+}} ymm1 = ymm15[0],ymm1[0],ymm15[2],ymm1[2] -; AVX-NEXT: vmovups %ymm15, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vunpcklpd {{.*#+}} ymm1 = ymm14[0],ymm1[0],ymm14[2],ymm1[2] +; AVX-NEXT: vmovups %ymm14, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm1[3,4,5,6,7] -; AVX-NEXT: vmovaps 160(%rdi), %xmm2 -; AVX-NEXT: vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; AVX-NEXT: vmovaps 128(%rdi), %xmm1 +; AVX-NEXT: vmovaps 160(%rdi), %xmm1 ; AVX-NEXT: vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; AVX-NEXT: vunpckhpd {{.*#+}} xmm1 = xmm1[1],xmm2[1] +; AVX-NEXT: vmovaps 128(%rdi), %xmm2 +; AVX-NEXT: vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; AVX-NEXT: vunpckhpd {{.*#+}} xmm1 = xmm2[1],xmm1[1] ; AVX-NEXT: vmovaps 192(%rdi), %xmm2 ; AVX-NEXT: vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; AVX-NEXT: vinsertps {{.*#+}} xmm1 = zero,xmm1[1,2],xmm2[1] @@ -10113,22 +10099,21 @@ define void @load_i32_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt ; AVX-NEXT: vextractf128 $1, %ymm0, %xmm0 ; AVX-NEXT: vmovaps 896(%rdi), %xmm12 ; AVX-NEXT: vblendps {{.*#+}} xmm0 = xmm12[0,1],xmm0[2,3] -; AVX-NEXT: vmovaps %xmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; AVX-NEXT: vshufps {{.*#+}} xmm0 = xmm0[0,3,2,3] -; AVX-NEXT: vmovaps 992(%rdi), %ymm4 +; AVX-NEXT: vmovaps 992(%rdi), %ymm5 ; AVX-NEXT: vmovaps 976(%rdi), %xmm1 ; AVX-NEXT: vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vunpcklpd {{.*#+}} ymm1 = ymm4[0],ymm1[0],ymm4[2],ymm1[2] -; AVX-NEXT: vmovups %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vunpcklpd {{.*#+}} ymm1 = ymm5[0],ymm1[0],ymm5[2],ymm1[2] +; AVX-NEXT: vmovups %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm1[3,4,5,6,7] ; AVX-NEXT: vmovaps 1056(%rdi), %xmm1 ; AVX-NEXT: vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; AVX-NEXT: vmovaps 1024(%rdi), %xmm2 ; AVX-NEXT: vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; AVX-NEXT: vunpckhpd {{.*#+}} xmm1 = xmm2[1],xmm1[1] -; AVX-NEXT: vmovaps 1088(%rdi), %xmm10 -; AVX-NEXT: vinsertps {{.*#+}} xmm1 = zero,xmm1[1,2],xmm10[1] -; AVX-NEXT: vmovaps %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; AVX-NEXT: vmovaps 1088(%rdi), %xmm8 +; AVX-NEXT: vinsertps {{.*#+}} xmm1 = zero,xmm1[1,2],xmm8[1] +; AVX-NEXT: vmovaps %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; AVX-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm1 ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm1[5,6,7] ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill @@ -10142,31 +10127,31 @@ define void @load_i32_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt ; AVX-NEXT: vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; AVX-NEXT: vblendps {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3] ; AVX-NEXT: vshufps {{.*#+}} xmm0 = xmm0[0,3,2,3] -; AVX-NEXT: vmovaps 1440(%rdi), %ymm8 +; AVX-NEXT: vmovaps 1440(%rdi), %ymm4 ; AVX-NEXT: vmovaps 1424(%rdi), %xmm1 ; AVX-NEXT: vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vunpcklpd {{.*#+}} ymm1 = ymm8[0],ymm1[0],ymm8[2],ymm1[2] -; AVX-NEXT: vmovups %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vunpcklpd {{.*#+}} ymm1 = ymm4[0],ymm1[0],ymm4[2],ymm1[2] +; AVX-NEXT: vmovups %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm1[3,4,5,6,7] -; AVX-NEXT: vmovaps 1504(%rdi), %xmm2 -; AVX-NEXT: vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; AVX-NEXT: vmovaps 1472(%rdi), %xmm1 +; AVX-NEXT: vmovaps 1504(%rdi), %xmm1 ; AVX-NEXT: vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; AVX-NEXT: vunpckhpd {{.*#+}} xmm1 = xmm1[1],xmm2[1] -; AVX-NEXT: vmovaps 1536(%rdi), %xmm14 -; AVX-NEXT: vinsertps {{.*#+}} xmm1 = zero,xmm1[1,2],xmm14[1] -; AVX-NEXT: vmovaps %xmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; AVX-NEXT: vmovaps 1472(%rdi), %xmm2 +; AVX-NEXT: vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; AVX-NEXT: vunpckhpd {{.*#+}} xmm1 = xmm2[1],xmm1[1] +; AVX-NEXT: vmovaps 1536(%rdi), %xmm2 +; AVX-NEXT: vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; AVX-NEXT: vinsertps {{.*#+}} xmm1 = zero,xmm1[1,2],xmm2[1] ; AVX-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm1 ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm1[5,6,7] ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX-NEXT: vmovaps 288(%rdi), %ymm0 ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm7[1,1],ymm0[2,2],ymm7[5,5],ymm0[6,6] +; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm6[1,1],ymm0[2,2],ymm6[5,5],ymm0[6,6] ; AVX-NEXT: vperm2f128 {{.*#+}} ymm0 = ymm0[2,3,2,3] ; AVX-NEXT: vmovaps 256(%rdi), %xmm1 ; AVX-NEXT: vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; AVX-NEXT: vblendps {{.*#+}} xmm1 = xmm1[0],xmm13[1],xmm1[2,3] -; AVX-NEXT: vmovaps %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; AVX-NEXT: vblendps {{.*#+}} xmm1 = xmm1[0],xmm10[1],xmm1[2,3] +; AVX-NEXT: vmovaps %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; AVX-NEXT: vshufps {{.*#+}} xmm1 = xmm1[1,0],mem[3,3] ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2],ymm0[3,4,5,6,7] ; AVX-NEXT: vmovaps 384(%rdi), %ymm1 @@ -10183,24 +10168,22 @@ define void @load_i32_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX-NEXT: vmovaps 736(%rdi), %ymm0 ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm5[1,1],ymm0[2,2],ymm5[5,5],ymm0[6,6] +; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm11[1,1],ymm0[2,2],ymm11[5,5],ymm0[6,6] ; AVX-NEXT: vperm2f128 {{.*#+}} ymm0 = ymm0[2,3,2,3] ; AVX-NEXT: vmovaps 704(%rdi), %xmm1 ; AVX-NEXT: vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; AVX-NEXT: vblendps {{.*#+}} xmm1 = xmm1[0],xmm3[1],xmm1[2,3] -; AVX-NEXT: vmovaps %xmm3, %xmm5 -; AVX-NEXT: vmovaps %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; AVX-NEXT: vblendps {{.*#+}} xmm1 = xmm1[0],xmm15[1],xmm1[2,3] +; AVX-NEXT: vmovaps %xmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; AVX-NEXT: vshufps {{.*#+}} xmm1 = xmm1[1,0],mem[3,3] ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2],ymm0[3,4,5,6,7] ; AVX-NEXT: vmovaps 832(%rdi), %ymm1 ; AVX-NEXT: vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vmovaps 800(%rdi), %ymm3 -; AVX-NEXT: vmovups %ymm3, (%rsp) # 32-byte Spill -; AVX-NEXT: vperm2f128 {{.*#+}} ymm1 = ymm3[2,3],ymm1[0,1] -; AVX-NEXT: vshufps {{.*#+}} ymm1 = ymm3[0,0],ymm1[3,3],ymm3[4,4],ymm1[7,7] +; AVX-NEXT: vmovaps 800(%rdi), %ymm2 +; AVX-NEXT: vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vperm2f128 {{.*#+}} ymm1 = ymm2[2,3],ymm1[0,1] +; AVX-NEXT: vshufps {{.*#+}} ymm1 = ymm2[0,0],ymm1[3,3],ymm2[4,4],ymm1[7,7] ; AVX-NEXT: vextractf128 $1, %ymm1, %xmm1 -; AVX-NEXT: vinsertps $49, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm1 # 16-byte Folded Reload -; AVX-NEXT: # xmm1 = zero,xmm1[1,2],mem[0] +; AVX-NEXT: vinsertps {{.*#+}} xmm1 = zero,xmm1[1,2],xmm13[2] ; AVX-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm1 ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm1[5,6,7] ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill @@ -10208,10 +10191,10 @@ define void @load_i32_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm9[1,1],ymm0[2,2],ymm9[5,5],ymm0[6,6] ; AVX-NEXT: vperm2f128 {{.*#+}} ymm0 = ymm0[2,3,2,3] -; AVX-NEXT: vmovaps 1152(%rdi), %xmm11 -; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload -; AVX-NEXT: vblendps {{.*#+}} xmm1 = xmm11[0],xmm9[1],xmm11[2,3] -; AVX-NEXT: vmovaps %xmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; AVX-NEXT: vmovaps 1152(%rdi), %xmm1 +; AVX-NEXT: vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload +; AVX-NEXT: vblendps {{.*#+}} xmm1 = xmm1[0],xmm11[1],xmm1[2,3] ; AVX-NEXT: vshufps {{.*#+}} xmm1 = xmm1[1,0],mem[3,3] ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2],ymm0[3,4,5,6,7] ; AVX-NEXT: vmovaps 1280(%rdi), %ymm1 @@ -10221,8 +10204,8 @@ define void @load_i32_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt ; AVX-NEXT: vperm2f128 {{.*#+}} ymm1 = ymm2[2,3],ymm1[0,1] ; AVX-NEXT: vshufps {{.*#+}} ymm1 = ymm2[0,0],ymm1[3,3],ymm2[4,4],ymm1[7,7] ; AVX-NEXT: vextractf128 $1, %ymm1, %xmm1 -; AVX-NEXT: vinsertps $49, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm1 # 16-byte Folded Reload -; AVX-NEXT: # xmm1 = zero,xmm1[1,2],mem[0] +; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload +; AVX-NEXT: vinsertps {{.*#+}} xmm1 = zero,xmm1[1,2],xmm9[2] ; AVX-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm1 ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm1[5,6,7] ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill @@ -10251,7 +10234,7 @@ define void @load_i32_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX-NEXT: vmovaps 64(%rdi), %ymm0 ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm15[1,1],ymm0[2,2],ymm15[5,5],ymm0[6,6] +; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm14[1,1],ymm0[2,2],ymm14[5,5],ymm0[6,6] ; AVX-NEXT: vperm2f128 {{.*#+}} ymm0 = ymm0[2,3,2,3] ; AVX-NEXT: vmovaps 32(%rdi), %xmm1 ; AVX-NEXT: vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill @@ -10273,13 +10256,13 @@ define void @load_i32_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt ; AVX-NEXT: vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX-NEXT: vmovaps 512(%rdi), %ymm0 ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload -; AVX-NEXT: vshufps {{.*#+}} ymm1 = ymm6[1,1],ymm0[2,2],ymm6[5,5],ymm0[6,6] +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload +; AVX-NEXT: vshufps {{.*#+}} ymm1 = ymm1[1,1],ymm0[2,2],ymm1[5,5],ymm0[6,6] ; AVX-NEXT: vperm2f128 {{.*#+}} ymm1 = ymm1[2,3,2,3] ; AVX-NEXT: vmovaps 480(%rdi), %xmm0 ; AVX-NEXT: vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload -; AVX-NEXT: vblendps {{.*#+}} xmm2 = xmm0[0],xmm15[1],xmm0[2,3] +; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload +; AVX-NEXT: vblendps {{.*#+}} xmm2 = xmm0[0],xmm14[1],xmm0[2,3] ; AVX-NEXT: vshufps {{.*#+}} xmm2 = xmm2[1,0],mem[3,3] ; AVX-NEXT: vblendps {{.*#+}} ymm2 = ymm2[0,1,2],ymm1[3,4,5,6,7] ; AVX-NEXT: vmovaps 608(%rdi), %ymm0 @@ -10296,180 +10279,183 @@ define void @load_i32_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt ; AVX-NEXT: vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX-NEXT: vmovaps 960(%rdi), %ymm0 ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vshufps {{.*#+}} ymm2 = ymm4[1,1],ymm0[2,2],ymm4[5,5],ymm0[6,6] +; AVX-NEXT: vshufps {{.*#+}} ymm2 = ymm5[1,1],ymm0[2,2],ymm5[5,5],ymm0[6,6] ; AVX-NEXT: vperm2f128 {{.*#+}} ymm2 = ymm2[2,3,2,3] ; AVX-NEXT: vmovaps 928(%rdi), %xmm0 ; AVX-NEXT: vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; AVX-NEXT: vblendps {{.*#+}} xmm3 = xmm0[0],xmm12[1],xmm0[2,3] +; AVX-NEXT: vmovaps %xmm12, %xmm6 +; AVX-NEXT: vmovaps %xmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; AVX-NEXT: vshufps {{.*#+}} xmm3 = xmm3[1,0],mem[3,3] ; AVX-NEXT: vblendps {{.*#+}} ymm2 = ymm3[0,1,2],ymm2[3,4,5,6,7] ; AVX-NEXT: vmovaps 1056(%rdi), %ymm0 ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX-NEXT: vmovaps 1024(%rdi), %ymm1 ; AVX-NEXT: vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vperm2f128 {{.*#+}} ymm4 = ymm1[2,3],ymm0[0,1] -; AVX-NEXT: vshufps {{.*#+}} ymm4 = ymm1[0,0],ymm4[3,3],ymm1[4,4],ymm4[7,7] -; AVX-NEXT: vextractf128 $1, %ymm4, %xmm4 -; AVX-NEXT: vinsertps {{.*#+}} xmm4 = zero,xmm4[1,2],xmm10[2] -; AVX-NEXT: vinsertf128 $1, %xmm4, %ymm0, %ymm4 -; AVX-NEXT: vblendps {{.*#+}} ymm2 = ymm2[0,1,2,3,4],ymm4[5,6,7] +; AVX-NEXT: vperm2f128 {{.*#+}} ymm5 = ymm1[2,3],ymm0[0,1] +; AVX-NEXT: vshufps {{.*#+}} ymm5 = ymm1[0,0],ymm5[3,3],ymm1[4,4],ymm5[7,7] +; AVX-NEXT: vextractf128 $1, %ymm5, %xmm5 +; AVX-NEXT: vinsertps {{.*#+}} xmm5 = zero,xmm5[1,2],xmm8[2] +; AVX-NEXT: vinsertf128 $1, %xmm5, %ymm0, %ymm5 +; AVX-NEXT: vblendps {{.*#+}} ymm2 = ymm2[0,1,2,3,4],ymm5[5,6,7] ; AVX-NEXT: vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX-NEXT: vmovaps 1408(%rdi), %ymm0 ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vshufps {{.*#+}} ymm4 = ymm8[1,1],ymm0[2,2],ymm8[5,5],ymm0[6,6] -; AVX-NEXT: vperm2f128 {{.*#+}} ymm4 = ymm4[2,3,2,3] -; AVX-NEXT: vmovaps 1376(%rdi), %xmm0 -; AVX-NEXT: vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; AVX-NEXT: vshufps {{.*#+}} ymm5 = ymm4[1,1],ymm0[2,2],ymm4[5,5],ymm0[6,6] +; AVX-NEXT: vperm2f128 {{.*#+}} ymm5 = ymm5[2,3,2,3] +; AVX-NEXT: vmovaps 1376(%rdi), %xmm4 ; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload -; AVX-NEXT: vblendps {{.*#+}} xmm12 = xmm0[0],xmm3[1],xmm0[2,3] +; AVX-NEXT: vblendps {{.*#+}} xmm12 = xmm4[0],xmm3[1],xmm4[2,3] ; AVX-NEXT: vshufps {{.*#+}} xmm12 = xmm12[1,0],mem[3,3] -; AVX-NEXT: vblendps {{.*#+}} ymm12 = ymm12[0,1,2],ymm4[3,4,5,6,7] +; AVX-NEXT: vblendps {{.*#+}} ymm12 = ymm12[0,1,2],ymm5[3,4,5,6,7] ; AVX-NEXT: vmovaps 1504(%rdi), %ymm0 ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX-NEXT: vmovaps 1472(%rdi), %ymm1 ; AVX-NEXT: vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vperm2f128 {{.*#+}} ymm10 = ymm1[2,3],ymm0[0,1] -; AVX-NEXT: vshufps {{.*#+}} ymm10 = ymm1[0,0],ymm10[3,3],ymm1[4,4],ymm10[7,7] -; AVX-NEXT: vextractf128 $1, %ymm10, %xmm10 -; AVX-NEXT: vinsertps {{.*#+}} xmm10 = zero,xmm10[1,2],xmm14[2] -; AVX-NEXT: vinsertf128 $1, %xmm10, %ymm0, %ymm10 -; AVX-NEXT: vblendps {{.*#+}} ymm10 = ymm12[0,1,2,3,4],ymm10[5,6,7] -; AVX-NEXT: vmovups %ymm10, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vshufps {{.*#+}} xmm10 = xmm13[2,3,2,3] -; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload -; AVX-NEXT: vblendps {{.*#+}} xmm10 = xmm10[0],xmm14[1],xmm10[2,3] +; AVX-NEXT: vperm2f128 {{.*#+}} ymm8 = ymm1[2,3],ymm0[0,1] +; AVX-NEXT: vshufps {{.*#+}} ymm8 = ymm1[0,0],ymm8[3,3],ymm1[4,4],ymm8[7,7] +; AVX-NEXT: vextractf128 $1, %ymm8, %xmm8 +; AVX-NEXT: vinsertps $49, {{[-0-9]+}}(%r{{[sb]}}p), %xmm8, %xmm8 # 16-byte Folded Reload +; AVX-NEXT: # xmm8 = zero,xmm8[1,2],mem[0] +; AVX-NEXT: vinsertf128 $1, %xmm8, %ymm0, %ymm8 +; AVX-NEXT: vblendps {{.*#+}} ymm8 = ymm12[0,1,2,3,4],ymm8[5,6,7] +; AVX-NEXT: vmovups %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vshufps {{.*#+}} xmm8 = xmm10[2,3,2,3] +; AVX-NEXT: vblendps $2, {{[-0-9]+}}(%r{{[sb]}}p), %xmm8, %xmm8 # 16-byte Folded Reload +; AVX-NEXT: # xmm8 = xmm8[0],mem[1],xmm8[2,3] ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm12 # 32-byte Reload ; AVX-NEXT: vshufps $199, {{[-0-9]+}}(%r{{[sb]}}p), %ymm12, %ymm12 # 32-byte Folded Reload ; AVX-NEXT: # ymm12 = ymm12[3,1],mem[0,3],ymm12[7,5],mem[4,7] ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload ; AVX-NEXT: vshufps {{.*#+}} ymm12 = ymm0[2,1],ymm12[2,0],ymm0[6,5],ymm12[6,4] -; AVX-NEXT: vblendps {{.*#+}} ymm10 = ymm10[0,1],ymm12[2,3,4,5,6,7] -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload +; AVX-NEXT: vblendps {{.*#+}} ymm8 = ymm8[0,1],ymm12[2,3,4,5,6,7] +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Reload ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload -; AVX-NEXT: vunpcklpd {{.*#+}} ymm12 = ymm0[0],ymm8[0],ymm0[2],ymm8[2] +; AVX-NEXT: vunpcklpd {{.*#+}} ymm12 = ymm0[0],ymm10[0],ymm0[2],ymm10[2] ; AVX-NEXT: vextractf128 $1, %ymm12, %xmm12 ; AVX-NEXT: vblendps $8, {{[-0-9]+}}(%r{{[sb]}}p), %xmm12, %xmm12 # 16-byte Folded Reload ; AVX-NEXT: # xmm12 = xmm12[0,1,2],mem[3] ; AVX-NEXT: vinsertf128 $1, %xmm12, %ymm0, %ymm12 -; AVX-NEXT: vblendps {{.*#+}} ymm10 = ymm10[0,1,2,3,4],ymm12[5,6,7] -; AVX-NEXT: vmovups %ymm10, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vshufps {{.*#+}} xmm10 = xmm5[2,3,2,3] -; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload -; AVX-NEXT: vblendps {{.*#+}} xmm10 = xmm10[0],xmm5[1],xmm10[2,3] +; AVX-NEXT: vblendps {{.*#+}} ymm8 = ymm8[0,1,2,3,4],ymm12[5,6,7] +; AVX-NEXT: vmovups %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vshufps {{.*#+}} xmm8 = xmm15[2,3,2,3] +; AVX-NEXT: vblendps $2, {{[-0-9]+}}(%r{{[sb]}}p), %xmm8, %xmm8 # 16-byte Folded Reload +; AVX-NEXT: # xmm8 = xmm8[0],mem[1],xmm8[2,3] ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm12 # 32-byte Reload ; AVX-NEXT: vshufps $199, {{[-0-9]+}}(%r{{[sb]}}p), %ymm12, %ymm12 # 32-byte Folded Reload ; AVX-NEXT: # ymm12 = ymm12[3,1],mem[0,3],ymm12[7,5],mem[4,7] ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload ; AVX-NEXT: vshufps {{.*#+}} ymm12 = ymm0[2,1],ymm12[2,0],ymm0[6,5],ymm12[6,4] -; AVX-NEXT: vblendps {{.*#+}} ymm10 = ymm10[0,1],ymm12[2,3,4,5,6,7] -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload -; AVX-NEXT: vmovups (%rsp), %ymm0 # 32-byte Reload -; AVX-NEXT: vunpcklpd {{.*#+}} ymm12 = ymm0[0],ymm1[0],ymm0[2],ymm1[2] +; AVX-NEXT: vblendps {{.*#+}} ymm8 = ymm8[0,1],ymm12[2,3,4,5,6,7] +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload +; AVX-NEXT: vunpcklpd {{.*#+}} ymm12 = ymm0[0],ymm2[0],ymm0[2],ymm2[2] ; AVX-NEXT: vextractf128 $1, %ymm12, %xmm12 -; AVX-NEXT: vblendps $8, {{[-0-9]+}}(%r{{[sb]}}p), %xmm12, %xmm12 # 16-byte Folded Reload -; AVX-NEXT: # xmm12 = xmm12[0,1,2],mem[3] +; AVX-NEXT: vblendps {{.*#+}} xmm12 = xmm12[0,1,2],xmm13[3] ; AVX-NEXT: vinsertf128 $1, %xmm12, %ymm0, %ymm12 -; AVX-NEXT: vblendps {{.*#+}} ymm10 = ymm10[0,1,2,3,4],ymm12[5,6,7] -; AVX-NEXT: vmovups %ymm10, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vshufps {{.*#+}} xmm10 = xmm9[2,3,2,3] -; AVX-NEXT: vblendps {{.*#+}} xmm10 = xmm10[0],xmm11[1],xmm10[2,3] +; AVX-NEXT: vblendps {{.*#+}} ymm8 = ymm8[0,1,2,3,4],ymm12[5,6,7] +; AVX-NEXT: vmovups %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vshufps {{.*#+}} xmm8 = xmm11[2,3,2,3] +; AVX-NEXT: vblendps $2, {{[-0-9]+}}(%r{{[sb]}}p), %xmm8, %xmm8 # 16-byte Folded Reload +; AVX-NEXT: # xmm8 = xmm8[0],mem[1],xmm8[2,3] ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm12 # 32-byte Reload ; AVX-NEXT: vshufps $199, {{[-0-9]+}}(%r{{[sb]}}p), %ymm12, %ymm12 # 32-byte Folded Reload ; AVX-NEXT: # ymm12 = ymm12[3,1],mem[0,3],ymm12[7,5],mem[4,7] ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload ; AVX-NEXT: vshufps {{.*#+}} ymm12 = ymm0[2,1],ymm12[2,0],ymm0[6,5],ymm12[6,4] -; AVX-NEXT: vblendps {{.*#+}} ymm10 = ymm10[0,1],ymm12[2,3,4,5,6,7] +; AVX-NEXT: vblendps {{.*#+}} ymm8 = ymm8[0,1],ymm12[2,3,4,5,6,7] +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm13 # 32-byte Reload -; AVX-NEXT: vunpcklpd {{.*#+}} ymm12 = ymm13[0],ymm0[0],ymm13[2],ymm0[2] +; AVX-NEXT: vunpcklpd {{.*#+}} ymm12 = ymm0[0],ymm1[0],ymm0[2],ymm1[2] ; AVX-NEXT: vextractf128 $1, %ymm12, %xmm12 -; AVX-NEXT: vblendps $8, {{[-0-9]+}}(%r{{[sb]}}p), %xmm12, %xmm12 # 16-byte Folded Reload -; AVX-NEXT: # xmm12 = xmm12[0,1,2],mem[3] +; AVX-NEXT: vblendps {{.*#+}} xmm12 = xmm12[0,1,2],xmm9[3] ; AVX-NEXT: vinsertf128 $1, %xmm12, %ymm0, %ymm12 -; AVX-NEXT: vblendps {{.*#+}} ymm10 = ymm10[0,1,2,3,4],ymm12[5,6,7] -; AVX-NEXT: vmovups %ymm10, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vshufps {{.*#+}} xmm10 = xmm7[2,3,2,3] -; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload -; AVX-NEXT: vblendps {{.*#+}} xmm10 = xmm10[0],xmm9[1],xmm10[2,3] +; AVX-NEXT: vblendps {{.*#+}} ymm8 = ymm8[0,1,2,3,4],ymm12[5,6,7] +; AVX-NEXT: vmovups %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vshufps {{.*#+}} xmm8 = xmm7[2,3,2,3] +; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload +; AVX-NEXT: vblendps {{.*#+}} xmm8 = xmm8[0],xmm13[1],xmm8[2,3] ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm12 # 32-byte Reload ; AVX-NEXT: vshufps $199, {{[-0-9]+}}(%r{{[sb]}}p), %ymm12, %ymm12 # 32-byte Folded Reload ; AVX-NEXT: # ymm12 = ymm12[3,1],mem[0,3],ymm12[7,5],mem[4,7] -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload -; AVX-NEXT: vshufps {{.*#+}} ymm12 = ymm2[2,1],ymm12[2,0],ymm2[6,5],ymm12[6,4] -; AVX-NEXT: vblendps {{.*#+}} ymm10 = ymm10[0,1],ymm12[2,3,4,5,6,7] +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload +; AVX-NEXT: vshufps {{.*#+}} ymm12 = ymm0[2,1],ymm12[2,0],ymm0[6,5],ymm12[6,4] +; AVX-NEXT: vblendps {{.*#+}} ymm8 = ymm8[0,1],ymm12[2,3,4,5,6,7] ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm11 # 32-byte Reload -; AVX-NEXT: vunpcklpd {{[-0-9]+}}(%r{{[sb]}}p), %ymm11, %ymm12 # 32-byte Folded Reload -; AVX-NEXT: # ymm12 = ymm11[0],mem[0],ymm11[2],mem[2] +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm15 # 32-byte Reload +; AVX-NEXT: vunpcklpd {{.*#+}} ymm12 = ymm15[0],ymm11[0],ymm15[2],ymm11[2] ; AVX-NEXT: vextractf128 $1, %ymm12, %xmm12 ; AVX-NEXT: vblendps $8, {{[-0-9]+}}(%r{{[sb]}}p), %xmm12, %xmm12 # 16-byte Folded Reload ; AVX-NEXT: # xmm12 = xmm12[0,1,2],mem[3] ; AVX-NEXT: vinsertf128 $1, %xmm12, %ymm0, %ymm12 -; AVX-NEXT: vblendps {{.*#+}} ymm10 = ymm10[0,1,2,3,4],ymm12[5,6,7] -; AVX-NEXT: vmovups %ymm10, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vpermilps $238, {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Folded Reload -; AVX-NEXT: # xmm10 = mem[2,3,2,3] -; AVX-NEXT: vblendps $2, {{[-0-9]+}}(%r{{[sb]}}p), %xmm10, %xmm10 # 16-byte Folded Reload -; AVX-NEXT: # xmm10 = xmm10[0],mem[1],xmm10[2,3] +; AVX-NEXT: vblendps {{.*#+}} ymm8 = ymm8[0,1,2,3,4],ymm12[5,6,7] +; AVX-NEXT: vmovups %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vpermilps $238, {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Folded Reload +; AVX-NEXT: # xmm8 = mem[2,3,2,3] +; AVX-NEXT: vblendps $2, {{[-0-9]+}}(%r{{[sb]}}p), %xmm8, %xmm8 # 16-byte Folded Reload +; AVX-NEXT: # xmm8 = xmm8[0],mem[1],xmm8[2,3] ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm12 # 32-byte Reload ; AVX-NEXT: vshufps $199, {{[-0-9]+}}(%r{{[sb]}}p), %ymm12, %ymm12 # 32-byte Folded Reload ; AVX-NEXT: # ymm12 = ymm12[3,1],mem[0,3],ymm12[7,5],mem[4,7] -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload -; AVX-NEXT: vshufps {{.*#+}} ymm12 = ymm2[2,1],ymm12[2,0],ymm2[6,5],ymm12[6,4] -; AVX-NEXT: vblendps {{.*#+}} ymm10 = ymm10[0,1],ymm12[2,3,4,5,6,7] -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload -; AVX-NEXT: vunpcklpd {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm12 # 32-byte Folded Reload -; AVX-NEXT: # ymm12 = ymm2[0],mem[0],ymm2[2],mem[2] +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload +; AVX-NEXT: vshufps {{.*#+}} ymm12 = ymm0[2,1],ymm12[2,0],ymm0[6,5],ymm12[6,4] +; AVX-NEXT: vblendps {{.*#+}} ymm8 = ymm8[0,1],ymm12[2,3,4,5,6,7] +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload +; AVX-NEXT: vunpcklpd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm12 # 32-byte Folded Reload +; AVX-NEXT: # ymm12 = ymm0[0],mem[0],ymm0[2],mem[2] ; AVX-NEXT: vextractf128 $1, %ymm12, %xmm12 ; AVX-NEXT: vblendps $8, {{[-0-9]+}}(%r{{[sb]}}p), %xmm12, %xmm12 # 16-byte Folded Reload ; AVX-NEXT: # xmm12 = xmm12[0,1,2],mem[3] ; AVX-NEXT: vinsertf128 $1, %xmm12, %ymm0, %ymm12 -; AVX-NEXT: vblendps {{.*#+}} ymm10 = ymm10[0,1,2,3,4],ymm12[5,6,7] -; AVX-NEXT: vmovups %ymm10, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vshufps {{.*#+}} xmm10 = xmm15[2,3,2,3] -; AVX-NEXT: vblendps $2, {{[-0-9]+}}(%r{{[sb]}}p), %xmm10, %xmm10 # 16-byte Folded Reload -; AVX-NEXT: # xmm10 = xmm10[0],mem[1],xmm10[2,3] +; AVX-NEXT: vblendps {{.*#+}} ymm8 = ymm8[0,1,2,3,4],ymm12[5,6,7] +; AVX-NEXT: vmovups %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vshufps {{.*#+}} xmm8 = xmm14[2,3,2,3] +; AVX-NEXT: vblendps $2, {{[-0-9]+}}(%r{{[sb]}}p), %xmm8, %xmm8 # 16-byte Folded Reload +; AVX-NEXT: # xmm8 = xmm8[0],mem[1],xmm8[2,3] ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm12 # 32-byte Reload ; AVX-NEXT: vshufps $199, {{[-0-9]+}}(%r{{[sb]}}p), %ymm12, %ymm12 # 32-byte Folded Reload ; AVX-NEXT: # ymm12 = ymm12[3,1],mem[0,3],ymm12[7,5],mem[4,7] -; AVX-NEXT: vshufps {{.*#+}} ymm12 = ymm6[2,1],ymm12[2,0],ymm6[6,5],ymm12[6,4] -; AVX-NEXT: vblendps {{.*#+}} ymm10 = ymm10[0,1],ymm12[2,3,4,5,6,7] -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload -; AVX-NEXT: vunpcklpd {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm12 # 32-byte Folded Reload -; AVX-NEXT: # ymm12 = ymm2[0],mem[0],ymm2[2],mem[2] +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload +; AVX-NEXT: vshufps {{.*#+}} ymm12 = ymm0[2,1],ymm12[2,0],ymm0[6,5],ymm12[6,4] +; AVX-NEXT: vblendps {{.*#+}} ymm8 = ymm8[0,1],ymm12[2,3,4,5,6,7] +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload +; AVX-NEXT: vunpcklpd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm12 # 32-byte Folded Reload +; AVX-NEXT: # ymm12 = ymm0[0],mem[0],ymm0[2],mem[2] ; AVX-NEXT: vextractf128 $1, %ymm12, %xmm12 ; AVX-NEXT: vblendps $8, {{[-0-9]+}}(%r{{[sb]}}p), %xmm12, %xmm12 # 16-byte Folded Reload ; AVX-NEXT: # xmm12 = xmm12[0,1,2],mem[3] ; AVX-NEXT: vinsertf128 $1, %xmm12, %ymm0, %ymm12 -; AVX-NEXT: vblendps {{.*#+}} ymm10 = ymm10[0,1,2,3,4],ymm12[5,6,7] -; AVX-NEXT: vmovups %ymm10, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vpermilps $238, {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Folded Reload -; AVX-NEXT: # xmm10 = mem[2,3,2,3] -; AVX-NEXT: vblendps $2, {{[-0-9]+}}(%r{{[sb]}}p), %xmm10, %xmm10 # 16-byte Folded Reload -; AVX-NEXT: # xmm10 = xmm10[0],mem[1],xmm10[2,3] +; AVX-NEXT: vblendps {{.*#+}} ymm8 = ymm8[0,1,2,3,4],ymm12[5,6,7] +; AVX-NEXT: vmovups %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vshufps {{.*#+}} xmm8 = xmm6[2,3,2,3] +; AVX-NEXT: vblendps $2, {{[-0-9]+}}(%r{{[sb]}}p), %xmm8, %xmm8 # 16-byte Folded Reload +; AVX-NEXT: # xmm8 = xmm8[0],mem[1],xmm8[2,3] ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm12 # 32-byte Reload ; AVX-NEXT: vshufps $199, {{[-0-9]+}}(%r{{[sb]}}p), %ymm12, %ymm12 # 32-byte Folded Reload ; AVX-NEXT: # ymm12 = ymm12[3,1],mem[0,3],ymm12[7,5],mem[4,7] -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload -; AVX-NEXT: vshufps {{.*#+}} ymm12 = ymm2[2,1],ymm12[2,0],ymm2[6,5],ymm12[6,4] -; AVX-NEXT: vblendps {{.*#+}} ymm10 = ymm10[0,1],ymm12[2,3,4,5,6,7] -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload -; AVX-NEXT: vunpcklpd {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm12 # 32-byte Folded Reload -; AVX-NEXT: # ymm12 = ymm2[0],mem[0],ymm2[2],mem[2] +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm14 # 32-byte Reload +; AVX-NEXT: vshufps {{.*#+}} ymm12 = ymm14[2,1],ymm12[2,0],ymm14[6,5],ymm12[6,4] +; AVX-NEXT: vblendps {{.*#+}} ymm8 = ymm8[0,1],ymm12[2,3,4,5,6,7] +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload +; AVX-NEXT: vunpcklpd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm12 # 32-byte Folded Reload +; AVX-NEXT: # ymm12 = ymm0[0],mem[0],ymm0[2],mem[2] ; AVX-NEXT: vextractf128 $1, %ymm12, %xmm12 ; AVX-NEXT: vblendps $8, {{[-0-9]+}}(%r{{[sb]}}p), %xmm12, %xmm12 # 16-byte Folded Reload ; AVX-NEXT: # xmm12 = xmm12[0,1,2],mem[3] ; AVX-NEXT: vinsertf128 $1, %xmm12, %ymm0, %ymm12 -; AVX-NEXT: vblendps {{.*#+}} ymm10 = ymm10[0,1,2,3,4],ymm12[5,6,7] -; AVX-NEXT: vmovups %ymm10, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vshufps {{.*#+}} xmm10 = xmm3[2,3,2,3] -; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload -; AVX-NEXT: vblendps {{.*#+}} xmm10 = xmm10[0],xmm4[1],xmm10[2,3] +; AVX-NEXT: vblendps {{.*#+}} ymm8 = ymm8[0,1,2,3,4],ymm12[5,6,7] +; AVX-NEXT: vmovups %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vmovaps %xmm3, %xmm9 +; AVX-NEXT: vshufps {{.*#+}} xmm8 = xmm3[2,3,2,3] +; AVX-NEXT: vblendps {{.*#+}} xmm8 = xmm8[0],xmm4[1],xmm8[2,3] +; AVX-NEXT: vmovaps %xmm4, %xmm6 +; AVX-NEXT: vmovaps %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm12 # 32-byte Reload -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload -; AVX-NEXT: vshufps {{.*#+}} ymm12 = ymm12[3,1],ymm6[0,3],ymm12[7,5],ymm6[4,7] -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload -; AVX-NEXT: vshufps {{.*#+}} ymm12 = ymm2[2,1],ymm12[2,0],ymm2[6,5],ymm12[6,4] -; AVX-NEXT: vblendps {{.*#+}} ymm10 = ymm10[0,1],ymm12[2,3,4,5,6,7] +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload +; AVX-NEXT: vshufps {{.*#+}} ymm12 = ymm12[3,1],ymm5[0,3],ymm12[7,5],ymm5[4,7] +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload +; AVX-NEXT: vshufps {{.*#+}} ymm12 = ymm4[2,1],ymm12[2,0],ymm4[6,5],ymm12[6,4] +; AVX-NEXT: vblendps {{.*#+}} ymm8 = ymm8[0,1],ymm12[2,3,4,5,6,7] ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload ; AVX-NEXT: vunpcklpd {{.*#+}} ymm12 = ymm7[0],ymm3[0],ymm7[2],ymm3[2] @@ -10477,217 +10463,218 @@ define void @load_i32_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt ; AVX-NEXT: vblendps $8, {{[-0-9]+}}(%r{{[sb]}}p), %xmm12, %xmm12 # 16-byte Folded Reload ; AVX-NEXT: # xmm12 = xmm12[0,1,2],mem[3] ; AVX-NEXT: vinsertf128 $1, %xmm12, %ymm0, %ymm12 -; AVX-NEXT: vblendps {{.*#+}} ymm10 = ymm10[0,1,2,3,4],ymm12[5,6,7] -; AVX-NEXT: vmovups %ymm10, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vblendps {{.*#+}} ymm8 = ymm8[0,1,2,3,4],ymm12[5,6,7] +; AVX-NEXT: vmovups %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm12 # 32-byte Reload -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Reload -; AVX-NEXT: vshufps {{.*#+}} ymm10 = ymm10[1,0],ymm12[0,0],ymm10[5,4],ymm12[4,4] -; AVX-NEXT: vshufps {{.*#+}} ymm10 = ymm12[3,1],ymm10[0,2],ymm12[7,5],ymm10[4,6] -; AVX-NEXT: vblendps $8, {{[-0-9]+}}(%r{{[sb]}}p), %xmm14, %xmm12 # 16-byte Folded Reload -; AVX-NEXT: # xmm12 = xmm14[0,1,2],mem[3] +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload +; AVX-NEXT: vshufps {{.*#+}} ymm8 = ymm0[1,0],ymm12[0,0],ymm0[5,4],ymm12[4,4] +; AVX-NEXT: vshufps {{.*#+}} ymm8 = ymm12[3,1],ymm8[0,2],ymm12[7,5],ymm8[4,6] +; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload +; AVX-NEXT: vblendps $7, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm12 # 16-byte Folded Reload +; AVX-NEXT: # xmm12 = mem[0,1,2],xmm0[3] ; AVX-NEXT: vshufps {{.*#+}} xmm12 = xmm12[3,2,2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm10 = ymm12[0,1],ymm10[2,3,4,5,6,7] -; AVX-NEXT: vmovaps 416(%rdi), %ymm12 -; AVX-NEXT: vmovups %ymm12, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vshufps {{.*#+}} ymm15 = ymm12[0,1],ymm8[1,3],ymm12[4,5],ymm8[5,7] -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload -; AVX-NEXT: vshufps {{.*#+}} ymm15 = ymm8[0,2],ymm15[2,0],ymm8[4,6],ymm15[6,4] -; AVX-NEXT: vblendps {{.*#+}} ymm10 = ymm10[0,1,2,3,4],ymm15[5,6,7] -; AVX-NEXT: vmovups %ymm10, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vblendps {{.*#+}} ymm8 = ymm12[0,1],ymm8[2,3,4,5,6,7] +; AVX-NEXT: vmovaps 416(%rdi), %ymm0 +; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vshufps {{.*#+}} ymm12 = ymm0[0,1],ymm10[1,3],ymm0[4,5],ymm10[5,7] +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload +; AVX-NEXT: vshufps {{.*#+}} ymm12 = ymm0[0,2],ymm12[2,0],ymm0[4,6],ymm12[6,4] +; AVX-NEXT: vblendps {{.*#+}} ymm8 = ymm8[0,1,2,3,4],ymm12[5,6,7] +; AVX-NEXT: vmovups %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm12 # 32-byte Reload -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload -; AVX-NEXT: vshufps {{.*#+}} ymm10 = ymm8[1,0],ymm12[0,0],ymm8[5,4],ymm12[4,4] -; AVX-NEXT: vshufps {{.*#+}} ymm10 = ymm12[3,1],ymm10[0,2],ymm12[7,5],ymm10[4,6] -; AVX-NEXT: vblendps $8, {{[-0-9]+}}(%r{{[sb]}}p), %xmm5, %xmm15 # 16-byte Folded Reload -; AVX-NEXT: # xmm15 = xmm5[0,1,2],mem[3] -; AVX-NEXT: vshufps {{.*#+}} xmm15 = xmm15[3,2,2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm10 = ymm15[0,1],ymm10[2,3,4,5,6,7] -; AVX-NEXT: vmovaps 864(%rdi), %ymm5 -; AVX-NEXT: vmovups %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vshufps {{.*#+}} ymm15 = ymm5[0,1],ymm1[1,3],ymm5[4,5],ymm1[5,7] -; AVX-NEXT: vmovups (%rsp), %ymm1 # 32-byte Reload -; AVX-NEXT: vshufps {{.*#+}} ymm15 = ymm1[0,2],ymm15[2,0],ymm1[4,6],ymm15[6,4] -; AVX-NEXT: vblendps {{.*#+}} ymm1 = ymm10[0,1,2,3,4],ymm15[5,6,7] -; AVX-NEXT: vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload -; AVX-NEXT: vshufps {{.*#+}} ymm10 = ymm5[1,0],ymm1[0,0],ymm5[5,4],ymm1[4,4] -; AVX-NEXT: vshufps {{.*#+}} ymm10 = ymm1[3,1],ymm10[0,2],ymm1[7,5],ymm10[4,6] -; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload -; AVX-NEXT: vblendps $7, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm15 # 16-byte Folded Reload -; AVX-NEXT: # xmm15 = mem[0,1,2],xmm1[3] -; AVX-NEXT: vshufps {{.*#+}} xmm15 = xmm15[3,2,2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm10 = ymm15[0,1],ymm10[2,3,4,5,6,7] -; AVX-NEXT: vmovaps 1312(%rdi), %ymm1 -; AVX-NEXT: vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vshufps {{.*#+}} ymm15 = ymm1[0,1],ymm0[1,3],ymm1[4,5],ymm0[5,7] -; AVX-NEXT: vshufps {{.*#+}} ymm15 = ymm13[0,2],ymm15[2,0],ymm13[4,6],ymm15[6,4] -; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm10[0,1,2,3,4],ymm15[5,6,7] +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload +; AVX-NEXT: vshufps {{.*#+}} ymm8 = ymm0[1,0],ymm12[0,0],ymm0[5,4],ymm12[4,4] +; AVX-NEXT: vshufps {{.*#+}} ymm8 = ymm12[3,1],ymm8[0,2],ymm12[7,5],ymm8[4,6] +; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload +; AVX-NEXT: vblendps $7, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm12 # 16-byte Folded Reload +; AVX-NEXT: # xmm12 = mem[0,1,2],xmm0[3] +; AVX-NEXT: vshufps {{.*#+}} xmm12 = xmm12[3,2,2,3] +; AVX-NEXT: vblendps {{.*#+}} ymm8 = ymm12[0,1],ymm8[2,3,4,5,6,7] +; AVX-NEXT: vmovaps 864(%rdi), %ymm0 ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vshufps {{.*#+}} ymm12 = ymm0[0,1],ymm2[1,3],ymm0[4,5],ymm2[5,7] +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload +; AVX-NEXT: vshufps {{.*#+}} ymm12 = ymm0[0,2],ymm12[2,0],ymm0[4,6],ymm12[6,4] +; AVX-NEXT: vblendps {{.*#+}} ymm2 = ymm8[0,1,2,3,4],ymm12[5,6,7] +; AVX-NEXT: vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload +; AVX-NEXT: vshufps {{.*#+}} ymm8 = ymm0[1,0],ymm2[0,0],ymm0[5,4],ymm2[4,4] +; AVX-NEXT: vshufps {{.*#+}} ymm8 = ymm2[3,1],ymm8[0,2],ymm2[7,5],ymm8[4,6] +; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload +; AVX-NEXT: vblendps $7, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm12 # 16-byte Folded Reload +; AVX-NEXT: # xmm12 = mem[0,1,2],xmm0[3] +; AVX-NEXT: vshufps {{.*#+}} xmm12 = xmm12[3,2,2,3] +; AVX-NEXT: vblendps {{.*#+}} ymm8 = ymm12[0,1],ymm8[2,3,4,5,6,7] +; AVX-NEXT: vmovaps 1312(%rdi), %ymm0 +; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vshufps {{.*#+}} ymm12 = ymm0[0,1],ymm1[1,3],ymm0[4,5],ymm1[5,7] +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload +; AVX-NEXT: vshufps {{.*#+}} ymm12 = ymm0[0,2],ymm12[2,0],ymm0[4,6],ymm12[6,4] +; AVX-NEXT: vblendps {{.*#+}} ymm1 = ymm8[0,1,2,3,4],ymm12[5,6,7] +; AVX-NEXT: vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload -; AVX-NEXT: vshufps {{.*#+}} ymm10 = ymm1[1,0],ymm0[0,0],ymm1[5,4],ymm0[4,4] -; AVX-NEXT: vshufps {{.*#+}} ymm10 = ymm0[3,1],ymm10[0,2],ymm0[7,5],ymm10[4,6] -; AVX-NEXT: vblendps $8, {{[-0-9]+}}(%r{{[sb]}}p), %xmm9, %xmm15 # 16-byte Folded Reload -; AVX-NEXT: # xmm15 = xmm9[0,1,2],mem[3] -; AVX-NEXT: vshufps {{.*#+}} xmm15 = xmm15[3,2,2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm10 = ymm15[0,1],ymm10[2,3,4,5,6,7] +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload +; AVX-NEXT: vshufps {{.*#+}} ymm8 = ymm0[1,0],ymm1[0,0],ymm0[5,4],ymm1[4,4] +; AVX-NEXT: vshufps {{.*#+}} ymm8 = ymm1[3,1],ymm8[0,2],ymm1[7,5],ymm8[4,6] +; AVX-NEXT: vblendps $8, {{[-0-9]+}}(%r{{[sb]}}p), %xmm13, %xmm12 # 16-byte Folded Reload +; AVX-NEXT: # xmm12 = xmm13[0,1,2],mem[3] +; AVX-NEXT: vshufps {{.*#+}} xmm12 = xmm12[3,2,2,3] +; AVX-NEXT: vblendps {{.*#+}} ymm8 = ymm12[0,1],ymm8[2,3,4,5,6,7] ; AVX-NEXT: vmovaps 1760(%rdi), %ymm0 ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vshufps $212, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm15 # 32-byte Folded Reload -; AVX-NEXT: # ymm15 = ymm0[0,1],mem[1,3],ymm0[4,5],mem[5,7] -; AVX-NEXT: vshufps {{.*#+}} ymm15 = ymm11[0,2],ymm15[2,0],ymm11[4,6],ymm15[6,4] -; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm10[0,1,2,3,4],ymm15[5,6,7] +; AVX-NEXT: vshufps {{.*#+}} ymm12 = ymm0[0,1],ymm11[1,3],ymm0[4,5],ymm11[5,7] +; AVX-NEXT: vshufps {{.*#+}} ymm12 = ymm15[0,2],ymm12[2,0],ymm15[4,6],ymm12[6,4] +; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm8[0,1,2,3,4],ymm12[5,6,7] ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vmovaps %ymm2, %ymm0 -; AVX-NEXT: vshufps {{.*#+}} ymm2 = ymm6[1,0],ymm2[0,0],ymm6[5,4],ymm2[4,4] -; AVX-NEXT: vshufps {{.*#+}} ymm2 = ymm0[3,1],ymm2[0,2],ymm0[7,5],ymm2[4,6] -; AVX-NEXT: vblendps $8, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4, %xmm9 # 16-byte Folded Reload -; AVX-NEXT: # xmm9 = xmm4[0,1,2],mem[3] -; AVX-NEXT: vshufps {{.*#+}} xmm9 = xmm9[3,2,2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm2 = ymm9[0,1],ymm2[2,3,4,5,6,7] +; AVX-NEXT: vshufps {{.*#+}} ymm2 = ymm5[1,0],ymm4[0,0],ymm5[5,4],ymm4[4,4] +; AVX-NEXT: vshufps {{.*#+}} ymm2 = ymm4[3,1],ymm2[0,2],ymm4[7,5],ymm2[4,6] +; AVX-NEXT: vblendps {{.*#+}} xmm4 = xmm6[0,1,2],xmm9[3] +; AVX-NEXT: vshufps {{.*#+}} xmm4 = xmm4[3,2,2,3] +; AVX-NEXT: vblendps {{.*#+}} ymm2 = ymm4[0,1],ymm2[2,3,4,5,6,7] ; AVX-NEXT: vmovaps 1536(%rdi), %ymm0 ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vshufps {{.*#+}} ymm9 = ymm0[0,1],ymm3[1,3],ymm0[4,5],ymm3[5,7] -; AVX-NEXT: vshufps {{.*#+}} ymm9 = ymm7[0,2],ymm9[2,0],ymm7[4,6],ymm9[6,4] -; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm2[0,1,2,3,4],ymm9[5,6,7] +; AVX-NEXT: vshufps {{.*#+}} ymm4 = ymm0[0,1],ymm3[1,3],ymm0[4,5],ymm3[5,7] +; AVX-NEXT: vmovaps %ymm3, %ymm15 +; AVX-NEXT: vshufps {{.*#+}} ymm4 = ymm7[0,2],ymm4[2,0],ymm7[4,6],ymm4[6,4] +; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm2[0,1,2,3,4],ymm4[5,6,7] ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload -; AVX-NEXT: vshufps {{.*#+}} ymm2 = ymm1[1,0],ymm0[0,0],ymm1[5,4],ymm0[4,4] -; AVX-NEXT: vshufps {{.*#+}} ymm2 = ymm0[3,1],ymm2[0,2],ymm0[7,5],ymm2[4,6] -; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload -; AVX-NEXT: vblendps $8, {{[-0-9]+}}(%r{{[sb]}}p), %xmm15, %xmm7 # 16-byte Folded Reload -; AVX-NEXT: # xmm7 = xmm15[0,1,2],mem[3] -; AVX-NEXT: vshufps {{.*#+}} xmm7 = xmm7[3,2,2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm2 = ymm7[0,1],ymm2[2,3,4,5,6,7] -; AVX-NEXT: vmovaps 1088(%rdi), %ymm12 -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload -; AVX-NEXT: vshufps {{.*#+}} ymm7 = ymm12[0,1],ymm8[1,3],ymm12[4,5],ymm8[5,7] -; AVX-NEXT: vmovups %ymm12, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm11 # 32-byte Reload -; AVX-NEXT: vshufps {{.*#+}} ymm7 = ymm11[0,2],ymm7[2,0],ymm11[4,6],ymm7[6,4] -; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm2[0,1,2,3,4],ymm7[5,6,7] +; AVX-NEXT: vshufps {{.*#+}} ymm2 = ymm1[1,0],ymm14[0,0],ymm1[5,4],ymm14[4,4] +; AVX-NEXT: vshufps {{.*#+}} ymm2 = ymm14[3,1],ymm2[0,2],ymm14[7,5],ymm2[4,6] +; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Reload +; AVX-NEXT: vblendps $8, {{[-0-9]+}}(%r{{[sb]}}p), %xmm12, %xmm4 # 16-byte Folded Reload +; AVX-NEXT: # xmm4 = xmm12[0,1,2],mem[3] +; AVX-NEXT: vshufps {{.*#+}} xmm4 = xmm4[3,2,2,3] +; AVX-NEXT: vblendps {{.*#+}} ymm2 = ymm4[0,1],ymm2[2,3,4,5,6,7] +; AVX-NEXT: vmovaps 1088(%rdi), %ymm11 +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Reload +; AVX-NEXT: vshufps {{.*#+}} ymm4 = ymm11[0,1],ymm10[1,3],ymm11[4,5],ymm10[5,7] +; AVX-NEXT: vmovups %ymm11, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm14 # 32-byte Reload +; AVX-NEXT: vshufps {{.*#+}} ymm4 = ymm14[0,2],ymm4[2,0],ymm14[4,6],ymm4[6,4] +; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm2[0,1,2,3,4],ymm4[5,6,7] ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload ; AVX-NEXT: vshufps {{.*#+}} ymm2 = ymm1[1,0],ymm0[0,0],ymm1[5,4],ymm0[4,4] ; AVX-NEXT: vshufps {{.*#+}} ymm2 = ymm0[3,1],ymm2[0,2],ymm0[7,5],ymm2[4,6] -; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload -; AVX-NEXT: vblendps $8, {{[-0-9]+}}(%r{{[sb]}}p), %xmm14, %xmm7 # 16-byte Folded Reload -; AVX-NEXT: # xmm7 = xmm14[0,1,2],mem[3] -; AVX-NEXT: vshufps {{.*#+}} xmm7 = xmm7[3,2,2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm2 = ymm7[0,1],ymm2[2,3,4,5,6,7] -; AVX-NEXT: vmovaps 640(%rdi), %ymm4 -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Reload -; AVX-NEXT: vshufps {{.*#+}} ymm7 = ymm4[0,1],ymm10[1,3],ymm4[4,5],ymm10[5,7] -; AVX-NEXT: vmovups %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload +; AVX-NEXT: vblendps $8, {{[-0-9]+}}(%r{{[sb]}}p), %xmm9, %xmm4 # 16-byte Folded Reload +; AVX-NEXT: # xmm4 = xmm9[0,1,2],mem[3] +; AVX-NEXT: vshufps {{.*#+}} xmm4 = xmm4[3,2,2,3] +; AVX-NEXT: vblendps {{.*#+}} ymm2 = ymm4[0,1],ymm2[2,3,4,5,6,7] +; AVX-NEXT: vmovaps 640(%rdi), %ymm8 +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload +; AVX-NEXT: vshufps {{.*#+}} ymm4 = ymm8[0,1],ymm5[1,3],ymm8[4,5],ymm5[5,7] +; AVX-NEXT: vmovups %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload -; AVX-NEXT: vshufps {{.*#+}} ymm7 = ymm3[0,2],ymm7[2,0],ymm3[4,6],ymm7[6,4] -; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm2[0,1,2,3,4],ymm7[5,6,7] +; AVX-NEXT: vshufps {{.*#+}} ymm4 = ymm3[0,2],ymm4[2,0],ymm3[4,6],ymm4[6,4] +; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm2[0,1,2,3,4],ymm4[5,6,7] ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload ; AVX-NEXT: vshufps {{.*#+}} ymm2 = ymm1[1,0],ymm0[0,0],ymm1[5,4],ymm0[4,4] ; AVX-NEXT: vshufps {{.*#+}} ymm2 = ymm0[3,1],ymm2[0,2],ymm0[7,5],ymm2[4,6] -; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload -; AVX-NEXT: vblendps $8, {{[-0-9]+}}(%r{{[sb]}}p), %xmm5, %xmm7 # 16-byte Folded Reload -; AVX-NEXT: # xmm7 = xmm5[0,1,2],mem[3] -; AVX-NEXT: vshufps {{.*#+}} xmm7 = xmm7[3,2,2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm7 = ymm7[0,1],ymm2[2,3,4,5,6,7] +; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload +; AVX-NEXT: vblendps $8, {{[-0-9]+}}(%r{{[sb]}}p), %xmm6, %xmm4 # 16-byte Folded Reload +; AVX-NEXT: # xmm4 = xmm6[0,1,2],mem[3] +; AVX-NEXT: vshufps {{.*#+}} xmm4 = xmm4[3,2,2,3] +; AVX-NEXT: vblendps {{.*#+}} ymm4 = ymm4[0,1],ymm2[2,3,4,5,6,7] ; AVX-NEXT: vmovaps 192(%rdi), %ymm13 +; AVX-NEXT: vmovups %ymm13, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload -; AVX-NEXT: vshufps {{.*#+}} ymm9 = ymm13[0,1],ymm1[1,3],ymm13[4,5],ymm1[5,7] +; AVX-NEXT: vshufps {{.*#+}} ymm7 = ymm13[0,1],ymm1[1,3],ymm13[4,5],ymm1[5,7] ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload -; AVX-NEXT: vshufps {{.*#+}} ymm9 = ymm2[0,2],ymm9[2,0],ymm2[4,6],ymm9[6,4] -; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm7[0,1,2,3,4],ymm9[5,6,7] +; AVX-NEXT: vshufps {{.*#+}} ymm7 = ymm2[0,2],ymm7[2,0],ymm2[4,6],ymm7[6,4] +; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm4[0,1,2,3,4],ymm7[5,6,7] ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vperm2f128 {{.*#+}} ymm7 = ymm2[2,3,0,1] -; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm2[3,0],ymm7[0,0],ymm2[7,4],ymm7[4,4] -; AVX-NEXT: vshufps {{.*#+}} ymm7 = ymm13[1,0],ymm1[2,0],ymm13[5,4],ymm1[6,4] -; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm0[2,0],ymm7[2,0],ymm0[6,4],ymm7[6,4] -; AVX-NEXT: vmovaps 64(%rdi), %xmm9 +; AVX-NEXT: vperm2f128 {{.*#+}} ymm4 = ymm2[2,3,0,1] +; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm2[3,0],ymm4[0,0],ymm2[7,4],ymm4[4,4] +; AVX-NEXT: vshufps {{.*#+}} ymm4 = ymm13[1,0],ymm1[2,0],ymm13[5,4],ymm1[6,4] +; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm0[2,0],ymm4[2,0],ymm0[6,4],ymm4[6,4] +; AVX-NEXT: vmovaps 64(%rdi), %xmm2 +; AVX-NEXT: vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; AVX-NEXT: vmovaps 96(%rdi), %xmm1 ; AVX-NEXT: vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; AVX-NEXT: vshufps {{.*#+}} xmm7 = xmm1[0,1,0,1] -; AVX-NEXT: vblendps {{.*#+}} xmm7 = xmm9[0,1,2],xmm7[3] -; AVX-NEXT: vshufps {{.*#+}} xmm6 = xmm5[2,3,2,3] +; AVX-NEXT: vshufps {{.*#+}} xmm4 = xmm1[0,1,0,1] +; AVX-NEXT: vblendps {{.*#+}} xmm4 = xmm2[0,1,2],xmm4[3] +; AVX-NEXT: vshufps {{.*#+}} xmm6 = xmm6[2,3,2,3] ; AVX-NEXT: vblendps {{.*#+}} xmm6 = mem[0],xmm6[1],mem[2,3] -; AVX-NEXT: vblendps {{.*#+}} xmm6 = xmm6[0,1],xmm7[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm6[0,1,2,3],ymm0[4,5,6,7] +; AVX-NEXT: vblendps {{.*#+}} xmm4 = xmm6[0,1],xmm4[2,3] +; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm4[0,1,2,3],ymm0[4,5,6,7] ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload ; AVX-NEXT: vperm2f128 {{.*#+}} ymm0 = ymm1[2,3,0,1] ; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm1[3,0],ymm0[0,0],ymm1[7,4],ymm0[4,4] ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload -; AVX-NEXT: vshufps $33, {{[-0-9]+}}(%r{{[sb]}}p), %ymm7, %ymm5 # 32-byte Folded Reload -; AVX-NEXT: # ymm5 = ymm7[1,0],mem[2,0],ymm7[5,4],mem[6,4] -; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm0[2,0],ymm5[2,0],ymm0[6,4],ymm5[6,4] +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm13 # 32-byte Reload +; AVX-NEXT: vshufps {{.*#+}} ymm4 = ymm13[1,0],ymm7[2,0],ymm13[5,4],ymm7[6,4] +; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm0[2,0],ymm4[2,0],ymm0[6,4],ymm4[6,4] ; AVX-NEXT: vmovaps 320(%rdi), %xmm1 ; AVX-NEXT: vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; AVX-NEXT: vshufps {{.*#+}} xmm5 = xmm1[0,1,0,1] +; AVX-NEXT: vshufps {{.*#+}} xmm4 = xmm1[0,1,0,1] ; AVX-NEXT: vmovaps 288(%rdi), %xmm1 ; AVX-NEXT: vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; AVX-NEXT: vblendps {{.*#+}} xmm5 = xmm1[0,1,2],xmm5[3] +; AVX-NEXT: vblendps {{.*#+}} xmm4 = xmm1[0,1,2],xmm4[3] ; AVX-NEXT: vpermilps $238, {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Folded Reload ; AVX-NEXT: # xmm6 = mem[2,3,2,3] ; AVX-NEXT: vblendps {{.*#+}} xmm6 = mem[0],xmm6[1],mem[2,3] -; AVX-NEXT: vblendps {{.*#+}} xmm5 = xmm6[0,1],xmm5[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm5[0,1,2,3],ymm0[4,5,6,7] +; AVX-NEXT: vblendps {{.*#+}} xmm4 = xmm6[0,1],xmm4[2,3] +; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm4[0,1,2,3],ymm0[4,5,6,7] ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX-NEXT: vperm2f128 {{.*#+}} ymm0 = ymm3[2,3,0,1] ; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm3[3,0],ymm0[0,0],ymm3[7,4],ymm0[4,4] -; AVX-NEXT: vshufps {{.*#+}} ymm1 = ymm4[1,0],ymm10[2,0],ymm4[5,4],ymm10[6,4] +; AVX-NEXT: vshufps {{.*#+}} ymm1 = ymm8[1,0],ymm5[2,0],ymm8[5,4],ymm5[6,4] ; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm0[2,0],ymm1[2,0],ymm0[6,4],ymm1[6,4] ; AVX-NEXT: vmovaps 544(%rdi), %xmm1 ; AVX-NEXT: vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; AVX-NEXT: vshufps {{.*#+}} xmm1 = xmm1[0,1,0,1] ; AVX-NEXT: vmovaps 512(%rdi), %xmm6 ; AVX-NEXT: vblendps {{.*#+}} xmm1 = xmm6[0,1,2],xmm1[3] -; AVX-NEXT: vshufps {{.*#+}} xmm5 = xmm14[2,3,2,3] -; AVX-NEXT: vblendps {{.*#+}} xmm5 = mem[0],xmm5[1],mem[2,3] -; AVX-NEXT: vblendps {{.*#+}} xmm1 = xmm5[0,1],xmm1[2,3] +; AVX-NEXT: vshufps {{.*#+}} xmm4 = xmm9[2,3,2,3] +; AVX-NEXT: vblendps {{.*#+}} xmm4 = mem[0],xmm4[1],mem[2,3] +; AVX-NEXT: vblendps {{.*#+}} xmm1 = xmm4[0,1],xmm1[2,3] ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7] ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vmovups (%rsp), %ymm1 # 32-byte Reload +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload ; AVX-NEXT: vperm2f128 {{.*#+}} ymm0 = ymm1[2,3,0,1] ; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm1[3,0],ymm0[0,0],ymm1[7,4],ymm0[4,4] -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload -; AVX-NEXT: vshufps $33, {{[-0-9]+}}(%r{{[sb]}}p), %ymm5, %ymm1 # 32-byte Folded Reload -; AVX-NEXT: # ymm1 = ymm5[1,0],mem[2,0],ymm5[5,4],mem[6,4] +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload +; AVX-NEXT: vshufps $33, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm1 # 32-byte Folded Reload +; AVX-NEXT: # ymm1 = ymm1[1,0],mem[2,0],ymm1[5,4],mem[6,4] ; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm0[2,0],ymm1[2,0],ymm0[6,4],ymm1[6,4] ; AVX-NEXT: vmovaps 768(%rdi), %xmm1 -; AVX-NEXT: vmovaps %xmm1, (%rsp) # 16-byte Spill +; AVX-NEXT: vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; AVX-NEXT: vshufps {{.*#+}} xmm1 = xmm1[0,1,0,1] -; AVX-NEXT: vmovaps 736(%rdi), %xmm2 -; AVX-NEXT: vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; AVX-NEXT: vblendps {{.*#+}} xmm1 = xmm2[0,1,2],xmm1[3] -; AVX-NEXT: vpermilps $238, {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Folded Reload -; AVX-NEXT: # xmm14 = mem[2,3,2,3] -; AVX-NEXT: vblendps {{.*#+}} xmm14 = mem[0],xmm14[1],mem[2,3] -; AVX-NEXT: vblendps {{.*#+}} xmm1 = xmm14[0,1],xmm1[2,3] +; AVX-NEXT: vmovaps 736(%rdi), %xmm4 +; AVX-NEXT: vblendps {{.*#+}} xmm1 = xmm4[0,1,2],xmm1[3] +; AVX-NEXT: vpermilps $238, {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Folded Reload +; AVX-NEXT: # xmm9 = mem[2,3,2,3] +; AVX-NEXT: vblendps {{.*#+}} xmm9 = mem[0],xmm9[1],mem[2,3] +; AVX-NEXT: vblendps {{.*#+}} xmm1 = xmm9[0,1],xmm1[2,3] ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7] ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vperm2f128 {{.*#+}} ymm0 = ymm11[2,3,0,1] -; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm11[3,0],ymm0[0,0],ymm11[7,4],ymm0[4,4] -; AVX-NEXT: vshufps {{.*#+}} ymm1 = ymm12[1,0],ymm8[2,0],ymm12[5,4],ymm8[6,4] +; AVX-NEXT: vperm2f128 {{.*#+}} ymm0 = ymm14[2,3,0,1] +; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm14[3,0],ymm0[0,0],ymm14[7,4],ymm0[4,4] +; AVX-NEXT: vshufps {{.*#+}} ymm1 = ymm11[1,0],ymm10[2,0],ymm11[5,4],ymm10[6,4] ; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm0[2,0],ymm1[2,0],ymm0[6,4],ymm1[6,4] ; AVX-NEXT: vmovaps 992(%rdi), %xmm1 ; AVX-NEXT: vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; AVX-NEXT: vshufps {{.*#+}} xmm1 = xmm1[0,1,0,1] ; AVX-NEXT: vmovaps 960(%rdi), %xmm3 ; AVX-NEXT: vblendps {{.*#+}} xmm1 = xmm3[0,1,2],xmm1[3] -; AVX-NEXT: vshufps {{.*#+}} xmm14 = xmm15[2,3,2,3] -; AVX-NEXT: vblendps {{.*#+}} xmm14 = mem[0],xmm14[1],mem[2,3] -; AVX-NEXT: vblendps {{.*#+}} xmm1 = xmm14[0,1],xmm1[2,3] +; AVX-NEXT: vshufps {{.*#+}} xmm9 = xmm12[2,3,2,3] +; AVX-NEXT: vblendps {{.*#+}} xmm9 = mem[0],xmm9[1],mem[2,3] +; AVX-NEXT: vblendps {{.*#+}} xmm1 = xmm9[0,1],xmm1[2,3] ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7] ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload ; AVX-NEXT: vperm2f128 {{.*#+}} ymm0 = ymm1[2,3,0,1] ; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm1[3,0],ymm0[0,0],ymm1[7,4],ymm0[4,4] -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload -; AVX-NEXT: vshufps $33, {{[-0-9]+}}(%r{{[sb]}}p), %ymm8, %ymm1 # 32-byte Folded Reload -; AVX-NEXT: # ymm1 = ymm8[1,0],mem[2,0],ymm8[5,4],mem[6,4] +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm11 # 32-byte Reload +; AVX-NEXT: vshufps $33, {{[-0-9]+}}(%r{{[sb]}}p), %ymm11, %ymm1 # 32-byte Folded Reload +; AVX-NEXT: # ymm1 = ymm11[1,0],mem[2,0],ymm11[5,4],mem[6,4] ; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm0[2,0],ymm1[2,0],ymm0[6,4],ymm1[6,4] ; AVX-NEXT: vmovaps 1216(%rdi), %xmm1 ; AVX-NEXT: vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill @@ -10705,151 +10692,145 @@ define void @load_i32_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt ; AVX-NEXT: vperm2f128 {{.*#+}} ymm0 = ymm1[2,3,0,1] ; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm1[3,0],ymm0[0,0],ymm1[7,4],ymm0[4,4] ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload -; AVX-NEXT: vshufps $33, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm1 # 32-byte Folded Reload -; AVX-NEXT: # ymm1 = ymm1[1,0],mem[2,0],ymm1[5,4],mem[6,4] +; AVX-NEXT: vshufps {{.*#+}} ymm1 = ymm1[1,0],ymm15[2,0],ymm1[5,4],ymm15[6,4] ; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm0[2,0],ymm1[2,0],ymm0[6,4],ymm1[6,4] ; AVX-NEXT: vmovaps 1440(%rdi), %xmm1 ; AVX-NEXT: vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; AVX-NEXT: vshufps {{.*#+}} xmm14 = xmm1[0,1,0,1] ; AVX-NEXT: vmovaps 1408(%rdi), %xmm1 ; AVX-NEXT: vblendps {{.*#+}} xmm14 = xmm1[0,1,2],xmm14[3] -; AVX-NEXT: vpermilps $238, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload -; AVX-NEXT: # xmm4 = mem[2,3,2,3] -; AVX-NEXT: vblendps {{.*#+}} xmm4 = mem[0],xmm4[1],mem[2,3] -; AVX-NEXT: vblendps {{.*#+}} xmm4 = xmm4[0,1],xmm14[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm4[0,1,2,3],ymm0[4,5,6,7] +; AVX-NEXT: vpermilps $238, {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Folded Reload +; AVX-NEXT: # xmm10 = mem[2,3,2,3] +; AVX-NEXT: vblendps {{.*#+}} xmm10 = mem[0],xmm10[1],mem[2,3] +; AVX-NEXT: vblendps {{.*#+}} xmm10 = xmm10[0,1],xmm14[2,3] +; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm10[0,1,2,3],ymm0[4,5,6,7] ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload ; AVX-NEXT: vperm2f128 {{.*#+}} ymm0 = ymm2[2,3,0,1] ; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm2[3,0],ymm0[0,0],ymm2[7,4],ymm0[4,4] -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload -; AVX-NEXT: vshufps $33, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm4 # 32-byte Folded Reload -; AVX-NEXT: # ymm4 = ymm2[1,0],mem[2,0],ymm2[5,4],mem[6,4] -; AVX-NEXT: vshufps {{.*#+}} ymm4 = ymm0[2,0],ymm4[2,0],ymm0[6,4],ymm4[6,4] +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm15 # 32-byte Reload +; AVX-NEXT: vshufps {{.*#+}} ymm10 = ymm5[1,0],ymm15[2,0],ymm5[5,4],ymm15[6,4] +; AVX-NEXT: vshufps {{.*#+}} ymm10 = ymm0[2,0],ymm10[2,0],ymm0[6,4],ymm10[6,4] ; AVX-NEXT: vmovaps 1664(%rdi), %xmm0 ; AVX-NEXT: vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; AVX-NEXT: vshufps {{.*#+}} xmm14 = xmm0[0,1,0,1] ; AVX-NEXT: vmovaps 1632(%rdi), %xmm0 ; AVX-NEXT: vblendps {{.*#+}} xmm14 = xmm0[0,1,2],xmm14[3] -; AVX-NEXT: vpermilps $238, {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Folded Reload -; AVX-NEXT: # xmm11 = mem[2,3,2,3] -; AVX-NEXT: vblendps {{.*#+}} xmm11 = mem[0],xmm11[1],mem[2,3] -; AVX-NEXT: vblendps {{.*#+}} xmm11 = xmm11[0,1],xmm14[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm4 = ymm11[0,1,2,3],ymm4[4,5,6,7] -; AVX-NEXT: vmovups %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vshufps $246, {{[-0-9]+}}(%r{{[sb]}}p), %ymm13, %ymm4 # 32-byte Folded Reload -; AVX-NEXT: # ymm4 = ymm13[2,1],mem[3,3],ymm13[6,5],mem[7,7] -; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Reload -; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload -; AVX-NEXT: vblendps {{.*#+}} xmm11 = xmm12[0],xmm10[1],xmm12[2,3] -; AVX-NEXT: vinsertf128 $1, %xmm11, %ymm0, %ymm11 -; AVX-NEXT: vshufps {{.*#+}} ymm4 = ymm11[1,0],ymm4[2,0],ymm11[5,4],ymm4[6,4] -; AVX-NEXT: vblendps $7, {{[-0-9]+}}(%r{{[sb]}}p), %xmm9, %xmm11 # 16-byte Folded Reload -; AVX-NEXT: # xmm11 = mem[0,1,2],xmm9[3] -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm15 # 32-byte Reload -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Reload -; AVX-NEXT: vshufps {{.*#+}} ymm14 = ymm9[0,0],ymm15[1,0],ymm9[4,4],ymm15[5,4] -; AVX-NEXT: vshufps {{.*#+}} ymm14 = ymm14[2,0,2,3,6,4,6,7] -; AVX-NEXT: vextractf128 $1, %ymm14, %xmm14 -; AVX-NEXT: vshufps {{.*#+}} xmm11 = xmm14[0,1],xmm11[3,2] -; AVX-NEXT: vblendps {{.*#+}} ymm4 = ymm11[0,1,2,3],ymm4[4,5,6,7] -; AVX-NEXT: vmovups %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vshufps $246, {{[-0-9]+}}(%r{{[sb]}}p), %ymm7, %ymm4 # 32-byte Folded Reload -; AVX-NEXT: # ymm4 = ymm7[2,1],mem[3,3],ymm7[6,5],mem[7,7] +; AVX-NEXT: vpermilps $238, {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Folded Reload +; AVX-NEXT: # xmm9 = mem[2,3,2,3] +; AVX-NEXT: vblendps {{.*#+}} xmm9 = mem[0],xmm9[1],mem[2,3] +; AVX-NEXT: vblendps {{.*#+}} xmm9 = xmm9[0,1],xmm14[2,3] +; AVX-NEXT: vblendps {{.*#+}} ymm2 = ymm9[0,1,2,3],ymm10[4,5,6,7] +; AVX-NEXT: vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload +; AVX-NEXT: vshufps $246, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm9 # 32-byte Folded Reload +; AVX-NEXT: # ymm9 = ymm2[2,1],mem[3,3],ymm2[6,5],mem[7,7] +; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload +; AVX-NEXT: vblendps $13, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2, %xmm10 # 16-byte Folded Reload +; AVX-NEXT: # xmm10 = mem[0],xmm2[1],mem[2,3] +; AVX-NEXT: vinsertf128 $1, %xmm10, %ymm0, %ymm10 +; AVX-NEXT: vshufps {{.*#+}} ymm9 = ymm10[1,0],ymm9[2,0],ymm10[5,4],ymm9[6,4] ; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload +; AVX-NEXT: vblendps $8, {{[-0-9]+}}(%r{{[sb]}}p), %xmm14, %xmm8 # 16-byte Folded Reload +; AVX-NEXT: # xmm8 = xmm14[0,1,2],mem[3] +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm12 # 32-byte Reload +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload +; AVX-NEXT: vshufps {{.*#+}} ymm10 = ymm2[0,0],ymm12[1,0],ymm2[4,4],ymm12[5,4] +; AVX-NEXT: vextractf128 $1, %ymm10, %xmm10 +; AVX-NEXT: vshufps {{.*#+}} xmm8 = xmm10[2,0],xmm8[3,2] +; AVX-NEXT: vblendps {{.*#+}} ymm8 = ymm8[0,1,2,3],ymm9[4,5,6,7] +; AVX-NEXT: vmovups %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vshufps {{.*#+}} ymm8 = ymm13[2,1],ymm7[3,3],ymm13[6,5],ymm7[7,7] ; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload -; AVX-NEXT: vblendps {{.*#+}} xmm11 = xmm14[0],xmm9[1],xmm14[2,3] -; AVX-NEXT: vinsertf128 $1, %xmm11, %ymm0, %ymm11 -; AVX-NEXT: vshufps {{.*#+}} ymm4 = ymm11[1,0],ymm4[2,0],ymm11[5,4],ymm4[6,4] +; AVX-NEXT: vblendps $13, {{[-0-9]+}}(%r{{[sb]}}p), %xmm9, %xmm9 # 16-byte Folded Reload +; AVX-NEXT: # xmm9 = mem[0],xmm9[1],mem[2,3] +; AVX-NEXT: vinsertf128 $1, %xmm9, %ymm0, %ymm9 +; AVX-NEXT: vshufps {{.*#+}} ymm8 = ymm9[1,0],ymm8[2,0],ymm9[5,4],ymm8[6,4] ; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload ; AVX-NEXT: vblendps $7, {{[-0-9]+}}(%r{{[sb]}}p), %xmm7, %xmm7 # 16-byte Folded Reload ; AVX-NEXT: # xmm7 = mem[0,1,2],xmm7[3] -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm11 # 32-byte Reload -; AVX-NEXT: vshufps $16, {{[-0-9]+}}(%r{{[sb]}}p), %ymm11, %ymm11 # 32-byte Folded Reload -; AVX-NEXT: # ymm11 = ymm11[0,0],mem[1,0],ymm11[4,4],mem[5,4] -; AVX-NEXT: vshufps {{.*#+}} ymm11 = ymm11[2,0,2,3,6,4,6,7] -; AVX-NEXT: vextractf128 $1, %ymm11, %xmm11 -; AVX-NEXT: vshufps {{.*#+}} xmm7 = xmm11[0,1],xmm7[3,2] -; AVX-NEXT: vblendps {{.*#+}} ymm4 = ymm7[0,1,2,3],ymm4[4,5,6,7] -; AVX-NEXT: vmovups %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload -; AVX-NEXT: vshufps $246, {{[-0-9]+}}(%r{{[sb]}}p), %ymm4, %ymm4 # 32-byte Folded Reload -; AVX-NEXT: # ymm4 = ymm4[2,1],mem[3,3],ymm4[6,5],mem[7,7] -; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload -; AVX-NEXT: vblendps $13, {{[-0-9]+}}(%r{{[sb]}}p), %xmm11, %xmm7 # 16-byte Folded Reload -; AVX-NEXT: # xmm7 = mem[0],xmm11[1],mem[2,3] -; AVX-NEXT: vinsertf128 $1, %xmm7, %ymm0, %ymm7 -; AVX-NEXT: vshufps {{.*#+}} ymm4 = ymm7[1,0],ymm4[2,0],ymm7[5,4],ymm4[6,4] +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm13 # 32-byte Reload +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Reload +; AVX-NEXT: vshufps {{.*#+}} ymm9 = ymm10[0,0],ymm13[1,0],ymm10[4,4],ymm13[5,4] +; AVX-NEXT: vextractf128 $1, %ymm9, %xmm9 +; AVX-NEXT: vshufps {{.*#+}} xmm7 = xmm9[2,0],xmm7[3,2] +; AVX-NEXT: vblendps {{.*#+}} ymm7 = ymm7[0,1,2,3],ymm8[4,5,6,7] +; AVX-NEXT: vmovups %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload +; AVX-NEXT: vshufps $246, {{[-0-9]+}}(%r{{[sb]}}p), %ymm7, %ymm7 # 32-byte Folded Reload +; AVX-NEXT: # ymm7 = ymm7[2,1],mem[3,3],ymm7[6,5],mem[7,7] +; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload +; AVX-NEXT: vblendps $13, {{[-0-9]+}}(%r{{[sb]}}p), %xmm8, %xmm8 # 16-byte Folded Reload +; AVX-NEXT: # xmm8 = mem[0],xmm8[1],mem[2,3] +; AVX-NEXT: vinsertf128 $1, %xmm8, %ymm0, %ymm8 +; AVX-NEXT: vshufps {{.*#+}} ymm7 = ymm8[1,0],ymm7[2,0],ymm8[5,4],ymm7[6,4] ; AVX-NEXT: vblendps $7, {{[-0-9]+}}(%r{{[sb]}}p), %xmm6, %xmm6 # 16-byte Folded Reload ; AVX-NEXT: # xmm6 = mem[0,1,2],xmm6[3] -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload -; AVX-NEXT: vshufps $16, {{[-0-9]+}}(%r{{[sb]}}p), %ymm7, %ymm7 # 32-byte Folded Reload -; AVX-NEXT: # ymm7 = ymm7[0,0],mem[1,0],ymm7[4,4],mem[5,4] -; AVX-NEXT: vshufps {{.*#+}} ymm7 = ymm7[2,0,2,3,6,4,6,7] -; AVX-NEXT: vextractf128 $1, %ymm7, %xmm7 -; AVX-NEXT: vshufps {{.*#+}} xmm6 = xmm7[0,1],xmm6[3,2] -; AVX-NEXT: vblendps {{.*#+}} ymm4 = ymm6[0,1,2,3],ymm4[4,5,6,7] -; AVX-NEXT: vmovups %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vshufps $246, {{[-0-9]+}}(%r{{[sb]}}p), %ymm5, %ymm4 # 32-byte Folded Reload -; AVX-NEXT: # ymm4 = ymm5[2,1],mem[3,3],ymm5[6,5],mem[7,7] -; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload -; AVX-NEXT: vblendps $13, {{[-0-9]+}}(%r{{[sb]}}p), %xmm7, %xmm6 # 16-byte Folded Reload -; AVX-NEXT: # xmm6 = mem[0],xmm7[1],mem[2,3] -; AVX-NEXT: vinsertf128 $1, %xmm6, %ymm0, %ymm6 -; AVX-NEXT: vshufps {{.*#+}} ymm4 = ymm6[1,0],ymm4[2,0],ymm6[5,4],ymm4[6,4] -; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload -; AVX-NEXT: vblendps $7, (%rsp), %xmm5, %xmm5 # 16-byte Folded Reload -; AVX-NEXT: # xmm5 = mem[0,1,2],xmm5[3] +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Reload +; AVX-NEXT: vshufps $16, {{[-0-9]+}}(%r{{[sb]}}p), %ymm9, %ymm8 # 32-byte Folded Reload +; AVX-NEXT: # ymm8 = ymm9[0,0],mem[1,0],ymm9[4,4],mem[5,4] +; AVX-NEXT: vextractf128 $1, %ymm8, %xmm8 +; AVX-NEXT: vshufps {{.*#+}} xmm6 = xmm8[2,0],xmm6[3,2] +; AVX-NEXT: vblendps {{.*#+}} ymm6 = ymm6[0,1,2,3],ymm7[4,5,6,7] +; AVX-NEXT: vmovups %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload -; AVX-NEXT: vshufps $16, {{[-0-9]+}}(%r{{[sb]}}p), %ymm6, %ymm6 # 32-byte Folded Reload -; AVX-NEXT: # ymm6 = ymm6[0,0],mem[1,0],ymm6[4,4],mem[5,4] -; AVX-NEXT: vshufps {{.*#+}} ymm6 = ymm6[2,0,2,3,6,4,6,7] -; AVX-NEXT: vextractf128 $1, %ymm6, %xmm6 -; AVX-NEXT: vshufps {{.*#+}} xmm5 = xmm6[0,1],xmm5[3,2] -; AVX-NEXT: vblendps {{.*#+}} ymm4 = ymm5[0,1,2,3],ymm4[4,5,6,7] +; AVX-NEXT: vshufps $246, {{[-0-9]+}}(%r{{[sb]}}p), %ymm6, %ymm6 # 32-byte Folded Reload +; AVX-NEXT: # ymm6 = ymm6[2,1],mem[3,3],ymm6[6,5],mem[7,7] +; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload +; AVX-NEXT: vblendps $13, {{[-0-9]+}}(%r{{[sb]}}p), %xmm7, %xmm7 # 16-byte Folded Reload +; AVX-NEXT: # xmm7 = mem[0],xmm7[1],mem[2,3] +; AVX-NEXT: vinsertf128 $1, %xmm7, %ymm0, %ymm7 +; AVX-NEXT: vshufps {{.*#+}} ymm6 = ymm7[1,0],ymm6[2,0],ymm7[5,4],ymm6[6,4] +; AVX-NEXT: vblendps $7, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4, %xmm4 # 16-byte Folded Reload +; AVX-NEXT: # xmm4 = mem[0,1,2],xmm4[3] +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload +; AVX-NEXT: vshufps $16, {{[-0-9]+}}(%r{{[sb]}}p), %ymm8, %ymm7 # 32-byte Folded Reload +; AVX-NEXT: # ymm7 = ymm8[0,0],mem[1,0],ymm8[4,4],mem[5,4] +; AVX-NEXT: vextractf128 $1, %ymm7, %xmm7 +; AVX-NEXT: vshufps {{.*#+}} xmm4 = xmm7[2,0],xmm4[3,2] +; AVX-NEXT: vblendps {{.*#+}} ymm4 = ymm4[0,1,2,3],ymm6[4,5,6,7] ; AVX-NEXT: vmovups %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload ; AVX-NEXT: vshufps $246, {{[-0-9]+}}(%r{{[sb]}}p), %ymm4, %ymm4 # 32-byte Folded Reload ; AVX-NEXT: # ymm4 = ymm4[2,1],mem[3,3],ymm4[6,5],mem[7,7] -; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload -; AVX-NEXT: vblendps $13, {{[-0-9]+}}(%r{{[sb]}}p), %xmm5, %xmm5 # 16-byte Folded Reload -; AVX-NEXT: # xmm5 = mem[0],xmm5[1],mem[2,3] -; AVX-NEXT: vinsertf128 $1, %xmm5, %ymm0, %ymm5 -; AVX-NEXT: vshufps {{.*#+}} ymm4 = ymm5[1,0],ymm4[2,0],ymm5[5,4],ymm4[6,4] +; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload +; AVX-NEXT: vblendps $13, {{[-0-9]+}}(%r{{[sb]}}p), %xmm6, %xmm6 # 16-byte Folded Reload +; AVX-NEXT: # xmm6 = mem[0],xmm6[1],mem[2,3] +; AVX-NEXT: vinsertf128 $1, %xmm6, %ymm0, %ymm6 +; AVX-NEXT: vshufps {{.*#+}} ymm4 = ymm6[1,0],ymm4[2,0],ymm6[5,4],ymm4[6,4] ; AVX-NEXT: vblendps $7, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3, %xmm3 # 16-byte Folded Reload ; AVX-NEXT: # xmm3 = mem[0,1,2],xmm3[3] -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload -; AVX-NEXT: vshufps $16, {{[-0-9]+}}(%r{{[sb]}}p), %ymm5, %ymm5 # 32-byte Folded Reload -; AVX-NEXT: # ymm5 = ymm5[0,0],mem[1,0],ymm5[4,4],mem[5,4] -; AVX-NEXT: vshufps {{.*#+}} ymm5 = ymm5[2,0,2,3,6,4,6,7] -; AVX-NEXT: vextractf128 $1, %ymm5, %xmm5 -; AVX-NEXT: vshufps {{.*#+}} xmm3 = xmm5[0,1],xmm3[3,2] +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload +; AVX-NEXT: vshufps $16, {{[-0-9]+}}(%r{{[sb]}}p), %ymm7, %ymm6 # 32-byte Folded Reload +; AVX-NEXT: # ymm6 = ymm7[0,0],mem[1,0],ymm7[4,4],mem[5,4] +; AVX-NEXT: vextractf128 $1, %ymm6, %xmm6 +; AVX-NEXT: vshufps {{.*#+}} xmm3 = xmm6[2,0],xmm3[3,2] ; AVX-NEXT: vblendps {{.*#+}} ymm3 = ymm3[0,1,2,3],ymm4[4,5,6,7] ; AVX-NEXT: vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vshufps $246, {{[-0-9]+}}(%r{{[sb]}}p), %ymm8, %ymm3 # 32-byte Folded Reload -; AVX-NEXT: # ymm3 = ymm8[2,1],mem[3,3],ymm8[6,5],mem[7,7] -; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload -; AVX-NEXT: vblendps $13, {{[-0-9]+}}(%r{{[sb]}}p), %xmm6, %xmm4 # 16-byte Folded Reload -; AVX-NEXT: # xmm4 = mem[0],xmm6[1],mem[2,3] +; AVX-NEXT: vshufps $246, {{[-0-9]+}}(%r{{[sb]}}p), %ymm11, %ymm3 # 32-byte Folded Reload +; AVX-NEXT: # ymm3 = ymm11[2,1],mem[3,3],ymm11[6,5],mem[7,7] +; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload +; AVX-NEXT: vblendps $13, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4, %xmm4 # 16-byte Folded Reload +; AVX-NEXT: # xmm4 = mem[0],xmm4[1],mem[2,3] ; AVX-NEXT: vinsertf128 $1, %xmm4, %ymm0, %ymm4 ; AVX-NEXT: vshufps {{.*#+}} ymm3 = ymm4[1,0],ymm3[2,0],ymm4[5,4],ymm3[6,4] -; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload -; AVX-NEXT: vblendps $7, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4, %xmm4 # 16-byte Folded Reload -; AVX-NEXT: # xmm4 = mem[0,1,2],xmm4[3] -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload -; AVX-NEXT: vshufps $16, {{[-0-9]+}}(%r{{[sb]}}p), %ymm5, %ymm5 # 32-byte Folded Reload -; AVX-NEXT: # ymm5 = ymm5[0,0],mem[1,0],ymm5[4,4],mem[5,4] -; AVX-NEXT: vshufps {{.*#+}} ymm5 = ymm5[2,0,2,3,6,4,6,7] -; AVX-NEXT: vextractf128 $1, %ymm5, %xmm5 -; AVX-NEXT: vshufps {{.*#+}} xmm4 = xmm5[0,1],xmm4[3,2] +; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload +; AVX-NEXT: vblendps $8, {{[-0-9]+}}(%r{{[sb]}}p), %xmm11, %xmm4 # 16-byte Folded Reload +; AVX-NEXT: # xmm4 = xmm11[0,1,2],mem[3] +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload +; AVX-NEXT: vshufps $16, {{[-0-9]+}}(%r{{[sb]}}p), %ymm6, %ymm6 # 32-byte Folded Reload +; AVX-NEXT: # ymm6 = ymm6[0,0],mem[1,0],ymm6[4,4],mem[5,4] +; AVX-NEXT: vextractf128 $1, %ymm6, %xmm6 +; AVX-NEXT: vshufps {{.*#+}} xmm4 = xmm6[2,0],xmm4[3,2] ; AVX-NEXT: vblendps {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7] ; AVX-NEXT: vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload -; AVX-NEXT: vshufps $246, {{[-0-9]+}}(%r{{[sb]}}p), %ymm8, %ymm3 # 32-byte Folded Reload -; AVX-NEXT: # ymm3 = ymm8[2,1],mem[3,3],ymm8[6,5],mem[7,7] -; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload -; AVX-NEXT: vblendps $2, {{[-0-9]+}}(%r{{[sb]}}p), %xmm5, %xmm4 # 16-byte Folded Reload -; AVX-NEXT: # xmm4 = xmm5[0],mem[1],xmm5[2,3] +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload +; AVX-NEXT: vshufps $246, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3, %ymm3 # 32-byte Folded Reload +; AVX-NEXT: # ymm3 = ymm3[2,1],mem[3,3],ymm3[6,5],mem[7,7] +; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload +; AVX-NEXT: vblendps $13, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4, %xmm4 # 16-byte Folded Reload +; AVX-NEXT: # xmm4 = mem[0],xmm4[1],mem[2,3] ; AVX-NEXT: vinsertf128 $1, %xmm4, %ymm0, %ymm4 ; AVX-NEXT: vshufps {{.*#+}} ymm3 = ymm4[1,0],ymm3[2,0],ymm4[5,4],ymm3[6,4] ; AVX-NEXT: vblendps $7, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm1 # 16-byte Folded Reload @@ -10857,100 +10838,66 @@ define void @load_i32_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload ; AVX-NEXT: vshufps $16, {{[-0-9]+}}(%r{{[sb]}}p), %ymm4, %ymm4 # 32-byte Folded Reload ; AVX-NEXT: # ymm4 = ymm4[0,0],mem[1,0],ymm4[4,4],mem[5,4] -; AVX-NEXT: vshufps {{.*#+}} ymm4 = ymm4[2,0,2,3,6,4,6,7] ; AVX-NEXT: vextractf128 $1, %ymm4, %xmm4 -; AVX-NEXT: vshufps {{.*#+}} xmm1 = xmm4[0,1],xmm1[3,2] +; AVX-NEXT: vshufps {{.*#+}} xmm1 = xmm4[2,0],xmm1[3,2] ; AVX-NEXT: vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm3[4,5,6,7] ; AVX-NEXT: vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vshufps $246, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm1 # 32-byte Folded Reload -; AVX-NEXT: # ymm1 = ymm2[2,1],mem[3,3],ymm2[6,5],mem[7,7] +; AVX-NEXT: vshufps {{.*#+}} ymm1 = ymm5[2,1],ymm15[3,3],ymm5[6,5],ymm15[7,7] ; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload ; AVX-NEXT: vblendps $13, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3, %xmm3 # 16-byte Folded Reload ; AVX-NEXT: # xmm3 = mem[0],xmm3[1],mem[2,3] ; AVX-NEXT: vinsertf128 $1, %xmm3, %ymm0, %ymm3 ; AVX-NEXT: vshufps {{.*#+}} ymm1 = ymm3[1,0],ymm1[2,0],ymm3[5,4],ymm1[6,4] -; AVX-NEXT: vblendps $7, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload -; AVX-NEXT: # xmm0 = mem[0,1,2],xmm0[3] -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload -; AVX-NEXT: vshufps $16, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3, %ymm3 # 32-byte Folded Reload -; AVX-NEXT: # ymm3 = ymm3[0,0],mem[1,0],ymm3[4,4],mem[5,4] -; AVX-NEXT: vshufps {{.*#+}} ymm3 = ymm3[2,0,2,3,6,4,6,7] +; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload +; AVX-NEXT: vblendps {{.*#+}} xmm0 = xmm15[0,1,2],xmm0[3] +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload +; AVX-NEXT: vshufps $16, {{[-0-9]+}}(%r{{[sb]}}p), %ymm6, %ymm3 # 32-byte Folded Reload +; AVX-NEXT: # ymm3 = ymm6[0,0],mem[1,0],ymm6[4,4],mem[5,4] ; AVX-NEXT: vextractf128 $1, %ymm3, %xmm3 -; AVX-NEXT: vshufps {{.*#+}} xmm0 = xmm3[0,1],xmm0[3,2] +; AVX-NEXT: vshufps {{.*#+}} xmm0 = xmm3[2,0],xmm0[3,2] ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7] ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vperm2f128 {{.*#+}} ymm0 = ymm13[2,3,0,1] -; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm13[3,0],ymm0[0,0],ymm13[7,4],ymm0[4,4] -; AVX-NEXT: vshufps {{.*#+}} xmm1 = xmm10[2,3,2,3] -; AVX-NEXT: vblendps {{.*#+}} xmm1 = xmm1[0],xmm12[1],xmm1[2,3] -; AVX-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm1 -; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,0],ymm1[4,5],ymm0[6,4] -; AVX-NEXT: vpermilps $68, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload -; AVX-NEXT: # xmm1 = mem[0,1,0,1] -; AVX-NEXT: vblendps $8, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm1 # 16-byte Folded Reload -; AVX-NEXT: # xmm1 = xmm1[0,1,2],mem[3] -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload -; AVX-NEXT: vshufps {{.*#+}} ymm2 = ymm2[1,0],ymm15[2,0],ymm2[5,4],ymm15[6,4] -; AVX-NEXT: vshufps {{.*#+}} ymm2 = ymm2[2,0,2,3,6,4,6,7] -; AVX-NEXT: vextractf128 $1, %ymm2, %xmm2 -; AVX-NEXT: vblendps {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm13 = ymm1[0,1,2,3],ymm0[4,5,6,7] -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload -; AVX-NEXT: vperm2f128 {{.*#+}} ymm1 = ymm0[2,3,0,1] -; AVX-NEXT: vshufps {{.*#+}} ymm1 = ymm0[3,0],ymm1[0,0],ymm0[7,4],ymm1[4,4] -; AVX-NEXT: vshufps {{.*#+}} xmm2 = xmm9[2,3,2,3] -; AVX-NEXT: vblendps {{.*#+}} xmm2 = xmm2[0],xmm14[1],xmm2[2,3] -; AVX-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm2 -; AVX-NEXT: vshufps {{.*#+}} ymm1 = ymm2[0,1],ymm1[2,0],ymm2[4,5],ymm1[6,4] -; AVX-NEXT: vpermilps $68, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload -; AVX-NEXT: # xmm2 = mem[0,1,0,1] -; AVX-NEXT: vblendps $8, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2, %xmm2 # 16-byte Folded Reload -; AVX-NEXT: # xmm2 = xmm2[0,1,2],mem[3] -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload -; AVX-NEXT: vshufps $33, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm3 # 32-byte Folded Reload -; AVX-NEXT: # ymm3 = ymm0[1,0],mem[2,0],ymm0[5,4],mem[6,4] -; AVX-NEXT: vshufps {{.*#+}} ymm3 = ymm3[2,0,2,3,6,4,6,7] +; AVX-NEXT: vpermilps $68, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload +; AVX-NEXT: # xmm0 = mem[0,1,0,1] +; AVX-NEXT: vblendps {{.*#+}} xmm0 = xmm0[0,1,2],xmm14[3] +; AVX-NEXT: vshufps {{.*#+}} ymm3 = ymm2[1,0],ymm12[2,0],ymm2[5,4],ymm12[6,4] ; AVX-NEXT: vextractf128 $1, %ymm3, %xmm3 -; AVX-NEXT: vblendps {{.*#+}} xmm2 = xmm3[0,1],xmm2[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm1[4,5,6,7] -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload -; AVX-NEXT: vperm2f128 {{.*#+}} ymm1 = ymm0[2,3,0,1] -; AVX-NEXT: vshufps {{.*#+}} ymm1 = ymm0[3,0],ymm1[0,0],ymm0[7,4],ymm1[4,4] -; AVX-NEXT: vshufps {{.*#+}} xmm3 = xmm11[2,3,2,3] -; AVX-NEXT: vblendps $2, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3, %xmm3 # 16-byte Folded Reload -; AVX-NEXT: # xmm3 = xmm3[0],mem[1],xmm3[2,3] -; AVX-NEXT: vinsertf128 $1, %xmm3, %ymm0, %ymm3 -; AVX-NEXT: vshufps {{.*#+}} ymm1 = ymm3[0,1],ymm1[2,0],ymm3[4,5],ymm1[6,4] -; AVX-NEXT: vpermilps $68, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload -; AVX-NEXT: # xmm3 = mem[0,1,0,1] -; AVX-NEXT: vblendps $8, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3, %xmm3 # 16-byte Folded Reload -; AVX-NEXT: # xmm3 = xmm3[0,1,2],mem[3] -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload -; AVX-NEXT: vshufps $33, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm4 # 32-byte Folded Reload -; AVX-NEXT: # ymm4 = ymm0[1,0],mem[2,0],ymm0[5,4],mem[6,4] -; AVX-NEXT: vshufps {{.*#+}} ymm4 = ymm4[2,0,2,3,6,4,6,7] -; AVX-NEXT: vextractf128 $1, %ymm4, %xmm4 -; AVX-NEXT: vblendps {{.*#+}} xmm3 = xmm4[0,1],xmm3[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm3[0,1,2,3],ymm1[4,5,6,7] -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload -; AVX-NEXT: vperm2f128 {{.*#+}} ymm1 = ymm3[2,3,0,1] -; AVX-NEXT: vshufps {{.*#+}} ymm1 = ymm3[3,0],ymm1[0,0],ymm3[7,4],ymm1[4,4] -; AVX-NEXT: vshufps {{.*#+}} xmm3 = xmm7[2,3,2,3] +; AVX-NEXT: vshufps {{.*#+}} xmm0 = xmm3[2,0],xmm0[2,3] +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload +; AVX-NEXT: vperm2f128 {{.*#+}} ymm3 = ymm1[2,3,0,1] +; AVX-NEXT: vshufps {{.*#+}} ymm2 = ymm1[3,0],ymm3[0,0],ymm1[7,4],ymm3[4,4] +; AVX-NEXT: vpermilps $238, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload +; AVX-NEXT: # xmm3 = mem[2,3,2,3] ; AVX-NEXT: vblendps $2, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3, %xmm3 # 16-byte Folded Reload ; AVX-NEXT: # xmm3 = xmm3[0],mem[1],xmm3[2,3] ; AVX-NEXT: vinsertf128 $1, %xmm3, %ymm0, %ymm3 -; AVX-NEXT: vshufps {{.*#+}} ymm1 = ymm3[0,1],ymm1[2,0],ymm3[4,5],ymm1[6,4] -; AVX-NEXT: vpermilps $68, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload -; AVX-NEXT: # xmm3 = mem[0,1,0,1] -; AVX-NEXT: vblendps $8, (%rsp), %xmm3, %xmm3 # 16-byte Folded Reload -; AVX-NEXT: # xmm3 = xmm3[0,1,2],mem[3] -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload -; AVX-NEXT: vshufps $33, {{[-0-9]+}}(%r{{[sb]}}p), %ymm4, %ymm4 # 32-byte Folded Reload -; AVX-NEXT: # ymm4 = ymm4[1,0],mem[2,0],ymm4[5,4],mem[6,4] -; AVX-NEXT: vshufps {{.*#+}} ymm4 = ymm4[2,0,2,3,6,4,6,7] -; AVX-NEXT: vextractf128 $1, %ymm4, %xmm4 -; AVX-NEXT: vblendps {{.*#+}} xmm3 = xmm4[0,1],xmm3[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7] +; AVX-NEXT: vshufps {{.*#+}} ymm2 = ymm3[0,1],ymm2[2,0],ymm3[4,5],ymm2[6,4] +; AVX-NEXT: vblendps {{.*#+}} ymm2 = ymm0[0,1,2,3],ymm2[4,5,6,7] +; AVX-NEXT: vpermilps $68, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload +; AVX-NEXT: # xmm0 = mem[0,1,0,1] +; AVX-NEXT: vblendps $8, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload +; AVX-NEXT: # xmm0 = xmm0[0,1,2],mem[3] +; AVX-NEXT: vshufps {{.*#+}} ymm3 = ymm10[1,0],ymm13[2,0],ymm10[5,4],ymm13[6,4] +; AVX-NEXT: vextractf128 $1, %ymm3, %xmm3 +; AVX-NEXT: vshufps {{.*#+}} xmm0 = xmm3[2,0],xmm0[2,3] +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload +; AVX-NEXT: vperm2f128 {{.*#+}} ymm3 = ymm1[2,3,0,1] +; AVX-NEXT: vshufps {{.*#+}} ymm3 = ymm1[3,0],ymm3[0,0],ymm1[7,4],ymm3[4,4] +; AVX-NEXT: vpermilps $238, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload +; AVX-NEXT: # xmm4 = mem[2,3,2,3] +; AVX-NEXT: vblendps $2, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4, %xmm4 # 16-byte Folded Reload +; AVX-NEXT: # xmm4 = xmm4[0],mem[1],xmm4[2,3] +; AVX-NEXT: vinsertf128 $1, %xmm4, %ymm0, %ymm4 +; AVX-NEXT: vshufps {{.*#+}} ymm3 = ymm4[0,1],ymm3[2,0],ymm4[4,5],ymm3[6,4] +; AVX-NEXT: vblendps {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm3[4,5,6,7] +; AVX-NEXT: vpermilps $68, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload +; AVX-NEXT: # xmm0 = mem[0,1,0,1] +; AVX-NEXT: vblendps $8, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload +; AVX-NEXT: # xmm0 = xmm0[0,1,2],mem[3] +; AVX-NEXT: vshufps $33, {{[-0-9]+}}(%r{{[sb]}}p), %ymm9, %ymm3 # 32-byte Folded Reload +; AVX-NEXT: # ymm3 = ymm9[1,0],mem[2,0],ymm9[5,4],mem[6,4] +; AVX-NEXT: vextractf128 $1, %ymm3, %xmm3 +; AVX-NEXT: vshufps {{.*#+}} xmm0 = xmm3[2,0],xmm0[2,3] ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload ; AVX-NEXT: vperm2f128 {{.*#+}} ymm3 = ymm4[2,3,0,1] ; AVX-NEXT: vshufps {{.*#+}} ymm3 = ymm4[3,0],ymm3[0,0],ymm4[7,4],ymm3[4,4] @@ -10960,180 +10907,203 @@ define void @load_i32_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt ; AVX-NEXT: # xmm4 = xmm4[0],mem[1],xmm4[2,3] ; AVX-NEXT: vinsertf128 $1, %xmm4, %ymm0, %ymm4 ; AVX-NEXT: vshufps {{.*#+}} ymm3 = ymm4[0,1],ymm3[2,0],ymm4[4,5],ymm3[6,4] +; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm3[4,5,6,7] +; AVX-NEXT: vpermilps $68, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload +; AVX-NEXT: # xmm3 = mem[0,1,0,1] +; AVX-NEXT: vblendps $8, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3, %xmm3 # 16-byte Folded Reload +; AVX-NEXT: # xmm3 = xmm3[0,1,2],mem[3] +; AVX-NEXT: vshufps $33, {{[-0-9]+}}(%r{{[sb]}}p), %ymm8, %ymm4 # 32-byte Folded Reload +; AVX-NEXT: # ymm4 = ymm8[1,0],mem[2,0],ymm8[5,4],mem[6,4] +; AVX-NEXT: vextractf128 $1, %ymm4, %xmm4 +; AVX-NEXT: vshufps {{.*#+}} xmm3 = xmm4[2,0],xmm3[2,3] +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload +; AVX-NEXT: vperm2f128 {{.*#+}} ymm4 = ymm5[2,3,0,1] +; AVX-NEXT: vshufps {{.*#+}} ymm4 = ymm5[3,0],ymm4[0,0],ymm5[7,4],ymm4[4,4] +; AVX-NEXT: vpermilps $238, {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Folded Reload +; AVX-NEXT: # xmm8 = mem[2,3,2,3] +; AVX-NEXT: vblendps $2, {{[-0-9]+}}(%r{{[sb]}}p), %xmm8, %xmm8 # 16-byte Folded Reload +; AVX-NEXT: # xmm8 = xmm8[0],mem[1],xmm8[2,3] +; AVX-NEXT: vinsertf128 $1, %xmm8, %ymm0, %ymm8 +; AVX-NEXT: vshufps {{.*#+}} ymm4 = ymm8[0,1],ymm4[2,0],ymm8[4,5],ymm4[6,4] +; AVX-NEXT: vblendps {{.*#+}} ymm3 = ymm3[0,1,2,3],ymm4[4,5,6,7] ; AVX-NEXT: vpermilps $68, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload ; AVX-NEXT: # xmm4 = mem[0,1,0,1] ; AVX-NEXT: vblendps $8, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4, %xmm4 # 16-byte Folded Reload ; AVX-NEXT: # xmm4 = xmm4[0,1,2],mem[3] +; AVX-NEXT: vshufps $33, {{[-0-9]+}}(%r{{[sb]}}p), %ymm7, %ymm8 # 32-byte Folded Reload +; AVX-NEXT: # ymm8 = ymm7[1,0],mem[2,0],ymm7[5,4],mem[6,4] +; AVX-NEXT: vextractf128 $1, %ymm8, %xmm8 +; AVX-NEXT: vshufps {{.*#+}} xmm4 = xmm8[2,0],xmm4[2,3] +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload +; AVX-NEXT: vperm2f128 {{.*#+}} ymm8 = ymm5[2,3,0,1] +; AVX-NEXT: vshufps {{.*#+}} ymm8 = ymm5[3,0],ymm8[0,0],ymm5[7,4],ymm8[4,4] +; AVX-NEXT: vpermilps $238, {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Folded Reload +; AVX-NEXT: # xmm9 = mem[2,3,2,3] +; AVX-NEXT: vblendps $2, {{[-0-9]+}}(%r{{[sb]}}p), %xmm9, %xmm9 # 16-byte Folded Reload +; AVX-NEXT: # xmm9 = xmm9[0],mem[1],xmm9[2,3] +; AVX-NEXT: vinsertf128 $1, %xmm9, %ymm0, %ymm9 +; AVX-NEXT: vshufps {{.*#+}} ymm8 = ymm9[0,1],ymm8[2,0],ymm9[4,5],ymm8[6,4] +; AVX-NEXT: vblendps {{.*#+}} ymm8 = ymm4[0,1,2,3],ymm8[4,5,6,7] +; AVX-NEXT: vpermilps $68, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload +; AVX-NEXT: # xmm4 = mem[0,1,0,1] +; AVX-NEXT: vblendps {{.*#+}} xmm4 = xmm4[0,1,2],xmm11[3] +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload +; AVX-NEXT: vshufps $33, {{[-0-9]+}}(%r{{[sb]}}p), %ymm5, %ymm9 # 32-byte Folded Reload +; AVX-NEXT: # ymm9 = ymm5[1,0],mem[2,0],ymm5[5,4],mem[6,4] +; AVX-NEXT: vextractf128 $1, %ymm9, %xmm9 +; AVX-NEXT: vshufps {{.*#+}} xmm4 = xmm9[2,0],xmm4[2,3] +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload +; AVX-NEXT: vperm2f128 {{.*#+}} ymm9 = ymm5[2,3,0,1] +; AVX-NEXT: vshufps {{.*#+}} ymm9 = ymm5[3,0],ymm9[0,0],ymm5[7,4],ymm9[4,4] +; AVX-NEXT: vpermilps $238, {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Folded Reload +; AVX-NEXT: # xmm10 = mem[2,3,2,3] +; AVX-NEXT: vblendps $2, {{[-0-9]+}}(%r{{[sb]}}p), %xmm10, %xmm10 # 16-byte Folded Reload +; AVX-NEXT: # xmm10 = xmm10[0],mem[1],xmm10[2,3] +; AVX-NEXT: vinsertf128 $1, %xmm10, %ymm0, %ymm10 +; AVX-NEXT: vshufps {{.*#+}} ymm9 = ymm10[0,1],ymm9[2,0],ymm10[4,5],ymm9[6,4] +; AVX-NEXT: vblendps {{.*#+}} ymm4 = ymm4[0,1,2,3],ymm9[4,5,6,7] +; AVX-NEXT: vpermilps $68, {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Folded Reload +; AVX-NEXT: # xmm9 = mem[0,1,0,1] +; AVX-NEXT: vblendps $8, {{[-0-9]+}}(%r{{[sb]}}p), %xmm9, %xmm5 # 16-byte Folded Reload +; AVX-NEXT: # xmm5 = xmm9[0,1,2],mem[3] ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload ; AVX-NEXT: vshufps $33, {{[-0-9]+}}(%r{{[sb]}}p), %ymm7, %ymm9 # 32-byte Folded Reload ; AVX-NEXT: # ymm9 = ymm7[1,0],mem[2,0],ymm7[5,4],mem[6,4] -; AVX-NEXT: vshufps {{.*#+}} ymm9 = ymm9[2,0,2,3,6,4,6,7] ; AVX-NEXT: vextractf128 $1, %ymm9, %xmm9 -; AVX-NEXT: vblendps {{.*#+}} xmm4 = xmm9[0,1],xmm4[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7] +; AVX-NEXT: vshufps {{.*#+}} xmm5 = xmm9[2,0],xmm5[2,3] ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload -; AVX-NEXT: vperm2f128 {{.*#+}} ymm4 = ymm7[2,3,0,1] -; AVX-NEXT: vshufps {{.*#+}} ymm4 = ymm7[3,0],ymm4[0,0],ymm7[7,4],ymm4[4,4] -; AVX-NEXT: vshufps {{.*#+}} xmm9 = xmm6[2,3,2,3] -; AVX-NEXT: vblendps $2, {{[-0-9]+}}(%r{{[sb]}}p), %xmm9, %xmm9 # 16-byte Folded Reload -; AVX-NEXT: # xmm9 = xmm9[0],mem[1],xmm9[2,3] -; AVX-NEXT: vinsertf128 $1, %xmm9, %ymm0, %ymm9 -; AVX-NEXT: vshufps {{.*#+}} ymm4 = ymm9[0,1],ymm4[2,0],ymm9[4,5],ymm4[6,4] +; AVX-NEXT: vperm2f128 {{.*#+}} ymm9 = ymm7[2,3,0,1] +; AVX-NEXT: vshufps {{.*#+}} ymm9 = ymm7[3,0],ymm9[0,0],ymm7[7,4],ymm9[4,4] +; AVX-NEXT: vpermilps $238, {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Folded Reload +; AVX-NEXT: # xmm10 = mem[2,3,2,3] +; AVX-NEXT: vblendps $2, {{[-0-9]+}}(%r{{[sb]}}p), %xmm10, %xmm10 # 16-byte Folded Reload +; AVX-NEXT: # xmm10 = xmm10[0],mem[1],xmm10[2,3] +; AVX-NEXT: vinsertf128 $1, %xmm10, %ymm0, %ymm10 +; AVX-NEXT: vshufps {{.*#+}} ymm9 = ymm10[0,1],ymm9[2,0],ymm10[4,5],ymm9[6,4] +; AVX-NEXT: vblendps {{.*#+}} ymm5 = ymm5[0,1,2,3],ymm9[4,5,6,7] ; AVX-NEXT: vpermilps $68, {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Folded Reload ; AVX-NEXT: # xmm9 = mem[0,1,0,1] -; AVX-NEXT: vblendps $8, {{[-0-9]+}}(%r{{[sb]}}p), %xmm9, %xmm9 # 16-byte Folded Reload -; AVX-NEXT: # xmm9 = xmm9[0,1,2],mem[3] -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload +; AVX-NEXT: vblendps {{.*#+}} xmm9 = xmm9[0,1,2],xmm15[3] ; AVX-NEXT: vshufps $33, {{[-0-9]+}}(%r{{[sb]}}p), %ymm6, %ymm10 # 32-byte Folded Reload ; AVX-NEXT: # ymm10 = ymm6[1,0],mem[2,0],ymm6[5,4],mem[6,4] -; AVX-NEXT: vshufps {{.*#+}} ymm10 = ymm10[2,0,2,3,6,4,6,7] ; AVX-NEXT: vextractf128 $1, %ymm10, %xmm10 -; AVX-NEXT: vblendps {{.*#+}} xmm9 = xmm10[0,1],xmm9[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm9 = ymm9[0,1,2,3],ymm4[4,5,6,7] -; AVX-NEXT: vperm2f128 {{.*#+}} ymm4 = ymm8[2,3,0,1] -; AVX-NEXT: vshufps {{.*#+}} ymm4 = ymm8[3,0],ymm4[0,0],ymm8[7,4],ymm4[4,4] -; AVX-NEXT: vpermilps $238, {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Folded Reload -; AVX-NEXT: # xmm10 = mem[2,3,2,3] -; AVX-NEXT: vblendps {{.*#+}} xmm10 = xmm10[0],xmm5[1],xmm10[2,3] -; AVX-NEXT: vinsertf128 $1, %xmm10, %ymm0, %ymm10 -; AVX-NEXT: vshufps {{.*#+}} ymm4 = ymm10[0,1],ymm4[2,0],ymm10[4,5],ymm4[6,4] -; AVX-NEXT: vpermilps $68, {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Folded Reload -; AVX-NEXT: # xmm10 = mem[0,1,0,1] -; AVX-NEXT: vblendps $8, {{[-0-9]+}}(%r{{[sb]}}p), %xmm10, %xmm10 # 16-byte Folded Reload -; AVX-NEXT: # xmm10 = xmm10[0,1,2],mem[3] +; AVX-NEXT: vshufps {{.*#+}} xmm9 = xmm10[2,0],xmm9[2,3] ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload -; AVX-NEXT: vshufps $33, {{[-0-9]+}}(%r{{[sb]}}p), %ymm6, %ymm11 # 32-byte Folded Reload -; AVX-NEXT: # ymm11 = ymm6[1,0],mem[2,0],ymm6[5,4],mem[6,4] -; AVX-NEXT: vshufps {{.*#+}} ymm11 = ymm11[2,0,2,3,6,4,6,7] -; AVX-NEXT: vextractf128 $1, %ymm11, %xmm11 -; AVX-NEXT: vblendps {{.*#+}} xmm10 = xmm11[0,1],xmm10[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm4 = ymm10[0,1,2,3],ymm4[4,5,6,7] -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload -; AVX-NEXT: vperm2f128 {{.*#+}} ymm10 = ymm5[2,3,0,1] -; AVX-NEXT: vshufps {{.*#+}} ymm10 = ymm5[3,0],ymm10[0,0],ymm5[7,4],ymm10[4,4] -; AVX-NEXT: vpermilps $238, {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Folded Reload -; AVX-NEXT: # xmm11 = mem[2,3,2,3] -; AVX-NEXT: vblendps $2, {{[-0-9]+}}(%r{{[sb]}}p), %xmm11, %xmm11 # 16-byte Folded Reload -; AVX-NEXT: # xmm11 = xmm11[0],mem[1],xmm11[2,3] -; AVX-NEXT: vinsertf128 $1, %xmm11, %ymm0, %ymm11 -; AVX-NEXT: vshufps {{.*#+}} ymm10 = ymm11[0,1],ymm10[2,0],ymm11[4,5],ymm10[6,4] -; AVX-NEXT: vpermilps $68, {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Folded Reload -; AVX-NEXT: # xmm11 = mem[0,1,0,1] -; AVX-NEXT: vblendps $8, {{[-0-9]+}}(%r{{[sb]}}p), %xmm11, %xmm11 # 16-byte Folded Reload -; AVX-NEXT: # xmm11 = xmm11[0,1,2],mem[3] -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload -; AVX-NEXT: vshufps $33, {{[-0-9]+}}(%r{{[sb]}}p), %ymm5, %ymm12 # 32-byte Folded Reload -; AVX-NEXT: # ymm12 = ymm5[1,0],mem[2,0],ymm5[5,4],mem[6,4] -; AVX-NEXT: vshufps {{.*#+}} ymm12 = ymm12[2,0,2,3,6,4,6,7] -; AVX-NEXT: vextractf128 $1, %ymm12, %xmm12 -; AVX-NEXT: vblendps {{.*#+}} xmm11 = xmm12[0,1],xmm11[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm10 = ymm11[0,1,2,3],ymm10[4,5,6,7] -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm5, 192(%rsi) -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm5, 128(%rsi) -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm5, 64(%rsi) -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm5, (%rsi) -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm11 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm11, 224(%rsi) -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm11 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm11, 160(%rsi) -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm11 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm11, 96(%rsi) -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm11 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm11, 32(%rsi) -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm5, 192(%rdx) -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm5, 128(%rdx) -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm5, 64(%rdx) -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm5, (%rdx) -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm5, 224(%rdx) -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm5, 160(%rdx) -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm5, 96(%rdx) -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm5, 32(%rdx) -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm5, 192(%rcx) -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm5, 128(%rcx) -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm5, 64(%rcx) -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm5, (%rcx) -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm5, 224(%rcx) -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm5, 160(%rcx) -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm5, 96(%rcx) -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm5, 32(%rcx) -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm5, (%r8) -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm5, 64(%r8) -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm5, 128(%r8) -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm5, 192(%r8) -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm5, 224(%r8) -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm5, 160(%r8) -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm5, 96(%r8) -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm5, 32(%r8) -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm5, 224(%r9) -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm5, 192(%r9) -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm5, 160(%r9) -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm5, 128(%r9) -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm5, 96(%r9) -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm5, 64(%r9) -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm5, 32(%r9) -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm5, (%r9) +; AVX-NEXT: vperm2f128 {{.*#+}} ymm10 = ymm6[2,3,0,1] +; AVX-NEXT: vshufps {{.*#+}} ymm10 = ymm6[3,0],ymm10[0,0],ymm6[7,4],ymm10[4,4] +; AVX-NEXT: vpermilps $238, {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Folded Reload +; AVX-NEXT: # xmm12 = mem[2,3,2,3] +; AVX-NEXT: vblendps $2, {{[-0-9]+}}(%r{{[sb]}}p), %xmm12, %xmm12 # 16-byte Folded Reload +; AVX-NEXT: # xmm12 = xmm12[0],mem[1],xmm12[2,3] +; AVX-NEXT: vinsertf128 $1, %xmm12, %ymm0, %ymm12 +; AVX-NEXT: vshufps {{.*#+}} ymm10 = ymm12[0,1],ymm10[2,0],ymm12[4,5],ymm10[6,4] +; AVX-NEXT: vblendps {{.*#+}} ymm9 = ymm9[0,1,2,3],ymm10[4,5,6,7] +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm6, 192(%rsi) +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm6, 128(%rsi) +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm6, 64(%rsi) +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm6, (%rsi) +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm6, 224(%rsi) +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm10, 160(%rsi) +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm10, 96(%rsi) +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm10, 32(%rsi) +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm6, 192(%rdx) +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm6, 128(%rdx) +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm6, 64(%rdx) +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm6, (%rdx) +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm6, 224(%rdx) +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm6, 160(%rdx) +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm6, 96(%rdx) +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm6, 32(%rdx) +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm6, 192(%rcx) +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm6, 128(%rcx) +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm6, 64(%rcx) +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm6, (%rcx) +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm6, 224(%rcx) +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm6, 160(%rcx) +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm6, 96(%rcx) +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm6, 32(%rcx) +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm6, (%r8) +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm6, 64(%r8) +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm6, 128(%r8) +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm6, 192(%r8) +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm6, 224(%r8) +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm6, 160(%r8) +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm6, 96(%r8) +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm6, 32(%r8) +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm6, 224(%r9) +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm6, 192(%r9) +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm6, 160(%r9) +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm6, 128(%r9) +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm6, 96(%r9) +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm6, 64(%r9) +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm6, 32(%r9) +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm6, (%r9) ; AVX-NEXT: movq {{[0-9]+}}(%rsp), %rax -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm5, 224(%rax) -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm5, 192(%rax) -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm5, 160(%rax) -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm5, 128(%rax) -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm5, 96(%rax) -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm5, 64(%rax) -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm5, 32(%rax) -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm5, (%rax) +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm6, 224(%rax) +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm6, 192(%rax) +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm6, 160(%rax) +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm6, 128(%rax) +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm6, 96(%rax) +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm6, 64(%rax) +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm6, 32(%rax) +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm6, (%rax) ; AVX-NEXT: movq {{[0-9]+}}(%rsp), %rax -; AVX-NEXT: vmovaps %ymm10, 224(%rax) -; AVX-NEXT: vmovaps %ymm4, 192(%rax) -; AVX-NEXT: vmovaps %ymm9, 160(%rax) -; AVX-NEXT: vmovaps %ymm3, 128(%rax) -; AVX-NEXT: vmovaps %ymm1, 96(%rax) +; AVX-NEXT: vmovaps %ymm9, 224(%rax) +; AVX-NEXT: vmovaps %ymm5, 192(%rax) +; AVX-NEXT: vmovaps %ymm4, 160(%rax) +; AVX-NEXT: vmovaps %ymm8, 128(%rax) +; AVX-NEXT: vmovaps %ymm3, 96(%rax) ; AVX-NEXT: vmovaps %ymm0, 64(%rax) -; AVX-NEXT: vmovaps %ymm2, 32(%rax) -; AVX-NEXT: vmovaps %ymm13, (%rax) +; AVX-NEXT: vmovaps %ymm1, 32(%rax) +; AVX-NEXT: vmovaps %ymm2, (%rax) ; AVX-NEXT: addq $3176, %rsp # imm = 0xC68 ; AVX-NEXT: vzeroupper ; AVX-NEXT: retq diff --git a/llvm/test/CodeGen/X86/vector-interleaved-load-i32-stride-8.ll b/llvm/test/CodeGen/X86/vector-interleaved-load-i32-stride-8.ll index cd0891385faf..f0c95f4fa9ef 100644 --- a/llvm/test/CodeGen/X86/vector-interleaved-load-i32-stride-8.ll +++ b/llvm/test/CodeGen/X86/vector-interleaved-load-i32-stride-8.ll @@ -61,13 +61,13 @@ define void @load_i32_stride8_vf2(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr ; AVX-NEXT: vunpcklps {{.*#+}} ymm3 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5] ; AVX-NEXT: vextractf128 $1, %ymm3, %xmm3 ; AVX-NEXT: vshufps {{.*#+}} ymm6 = ymm1[1,0],ymm0[1,0],ymm1[5,4],ymm0[5,4] -; AVX-NEXT: vshufps {{.*#+}} ymm6 = ymm6[2,0,2,3,6,4,6,7] ; AVX-NEXT: vextractf128 $1, %ymm6, %xmm6 +; AVX-NEXT: vshufps {{.*#+}} xmm6 = xmm6[2,0,2,3] ; AVX-NEXT: vunpckhps {{.*#+}} ymm7 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7] ; AVX-NEXT: vextractf128 $1, %ymm7, %xmm7 ; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm1[3,0],ymm0[3,0],ymm1[7,4],ymm0[7,4] -; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm0[2,0,2,3,6,4,6,7] ; AVX-NEXT: vextractf128 $1, %ymm0, %xmm0 +; AVX-NEXT: vshufps {{.*#+}} xmm0 = xmm0[2,0,2,3] ; AVX-NEXT: vmovq %xmm4, (%rsi) ; AVX-NEXT: vmovq %xmm5, (%rdx) ; AVX-NEXT: vmovq %xmm2, (%rcx) @@ -514,50 +514,46 @@ define void @load_i32_stride8_vf4(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr ; AVX-NEXT: vmovaps 32(%rdi), %ymm1 ; AVX-NEXT: vmovaps 64(%rdi), %ymm2 ; AVX-NEXT: vmovaps 96(%rdi), %ymm3 -; AVX-NEXT: vmovaps 32(%rdi), %xmm5 -; AVX-NEXT: vmovaps (%rdi), %xmm6 -; AVX-NEXT: vunpcklps {{.*#+}} xmm4 = xmm6[0],xmm5[0],xmm6[1],xmm5[1] +; AVX-NEXT: vmovaps 32(%rdi), %xmm4 +; AVX-NEXT: vmovaps (%rdi), %xmm5 +; AVX-NEXT: vunpcklps {{.*#+}} xmm6 = xmm5[0],xmm4[0],xmm5[1],xmm4[1] ; AVX-NEXT: vmovaps 96(%rdi), %xmm7 ; AVX-NEXT: vmovaps 64(%rdi), %xmm8 ; AVX-NEXT: vunpcklps {{.*#+}} xmm9 = xmm8[0],xmm7[0],xmm8[1],xmm7[1] -; AVX-NEXT: vmovlhps {{.*#+}} xmm4 = xmm4[0],xmm9[0] -; AVX-NEXT: vshufps {{.*#+}} xmm10 = xmm6[1,1,1,1] -; AVX-NEXT: vblendps {{.*#+}} xmm10 = xmm10[0],xmm5[1],xmm10[2,3] +; AVX-NEXT: vmovlhps {{.*#+}} xmm6 = xmm6[0],xmm9[0] +; AVX-NEXT: vshufps {{.*#+}} xmm10 = xmm5[1,1,1,1] +; AVX-NEXT: vblendps {{.*#+}} xmm10 = xmm10[0],xmm4[1],xmm10[2,3] ; AVX-NEXT: vblendps {{.*#+}} xmm9 = xmm10[0,1],xmm9[2,3] ; AVX-NEXT: vshufps {{.*#+}} xmm10 = xmm7[2,2,2,2] ; AVX-NEXT: vblendps {{.*#+}} xmm10 = xmm8[0,1,2],xmm10[3] -; AVX-NEXT: vunpckhps {{.*#+}} xmm5 = xmm6[2],xmm5[2],xmm6[3],xmm5[3] -; AVX-NEXT: vblendps {{.*#+}} xmm6 = xmm5[0,1],xmm10[2,3] +; AVX-NEXT: vunpckhps {{.*#+}} xmm4 = xmm5[2],xmm4[2],xmm5[3],xmm4[3] +; AVX-NEXT: vblendps {{.*#+}} xmm5 = xmm4[0,1],xmm10[2,3] ; AVX-NEXT: vunpckhps {{.*#+}} xmm7 = xmm8[2],xmm7[2],xmm8[3],xmm7[3] -; AVX-NEXT: vunpckhpd {{.*#+}} xmm5 = xmm5[1],xmm7[1] +; AVX-NEXT: vunpckhpd {{.*#+}} xmm4 = xmm4[1],xmm7[1] ; AVX-NEXT: vunpcklpd {{.*#+}} ymm7 = ymm3[0],ymm2[0],ymm3[2],ymm2[2] -; AVX-NEXT: vshufps {{.*#+}} ymm7 = ymm7[0,1,2,0,4,5,6,4] ; AVX-NEXT: vextractf128 $1, %ymm7, %xmm7 ; AVX-NEXT: vunpcklps {{.*#+}} ymm8 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5] ; AVX-NEXT: vextractf128 $1, %ymm8, %xmm8 -; AVX-NEXT: vblendps {{.*#+}} xmm7 = xmm8[0,1],xmm7[2,3] +; AVX-NEXT: vshufps {{.*#+}} xmm7 = xmm8[0,1],xmm7[2,0] ; AVX-NEXT: vunpcklps {{.*#+}} ymm8 = ymm2[0],ymm3[0],ymm2[1],ymm3[1],ymm2[4],ymm3[4],ymm2[5],ymm3[5] ; AVX-NEXT: vextractf128 $1, %ymm8, %xmm8 ; AVX-NEXT: vshufps {{.*#+}} ymm10 = ymm1[1,0],ymm0[1,0],ymm1[5,4],ymm0[5,4] -; AVX-NEXT: vshufps {{.*#+}} ymm10 = ymm10[2,0,2,3,6,4,6,7] ; AVX-NEXT: vextractf128 $1, %ymm10, %xmm10 -; AVX-NEXT: vblendps {{.*#+}} xmm8 = xmm10[0,1],xmm8[2,3] +; AVX-NEXT: vshufps {{.*#+}} xmm8 = xmm10[2,0],xmm8[2,3] ; AVX-NEXT: vunpckhpd {{.*#+}} ymm10 = ymm3[1],ymm2[1],ymm3[3],ymm2[3] -; AVX-NEXT: vshufps {{.*#+}} ymm10 = ymm10[0,1,2,0,4,5,6,4] ; AVX-NEXT: vextractf128 $1, %ymm10, %xmm10 ; AVX-NEXT: vunpckhps {{.*#+}} ymm11 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7] ; AVX-NEXT: vextractf128 $1, %ymm11, %xmm11 -; AVX-NEXT: vblendps {{.*#+}} xmm10 = xmm11[0,1],xmm10[2,3] +; AVX-NEXT: vshufps {{.*#+}} xmm10 = xmm11[0,1],xmm10[2,0] ; AVX-NEXT: vunpckhps {{.*#+}} ymm2 = ymm2[2],ymm3[2],ymm2[3],ymm3[3],ymm2[6],ymm3[6],ymm2[7],ymm3[7] ; AVX-NEXT: vextractf128 $1, %ymm2, %xmm2 ; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm1[3,0],ymm0[3,0],ymm1[7,4],ymm0[7,4] -; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm0[2,0,2,3,6,4,6,7] ; AVX-NEXT: vextractf128 $1, %ymm0, %xmm0 -; AVX-NEXT: vblendps {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3] -; AVX-NEXT: vmovaps %xmm4, (%rsi) +; AVX-NEXT: vshufps {{.*#+}} xmm0 = xmm0[2,0],xmm2[2,3] +; AVX-NEXT: vmovaps %xmm6, (%rsi) ; AVX-NEXT: vmovaps %xmm9, (%rdx) -; AVX-NEXT: vmovaps %xmm6, (%rcx) -; AVX-NEXT: vmovaps %xmm5, (%r8) +; AVX-NEXT: vmovaps %xmm5, (%rcx) +; AVX-NEXT: vmovaps %xmm4, (%r8) ; AVX-NEXT: vmovaps %xmm7, (%r9) ; AVX-NEXT: vmovaps %xmm8, (%r11) ; AVX-NEXT: vmovaps %xmm10, (%r10) @@ -1189,11 +1185,10 @@ define void @load_i32_stride8_vf8(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr ; AVX-NEXT: vunpcklps {{.*#+}} ymm12 = ymm7[0],ymm6[0],ymm7[1],ymm6[1],ymm7[4],ymm6[4],ymm7[5],ymm6[5] ; AVX-NEXT: vshufps {{.*#+}} ymm8 = ymm12[0,1],ymm8[2,0],ymm12[4,5],ymm8[6,4] ; AVX-NEXT: vunpcklpd {{.*#+}} ymm12 = ymm3[0],ymm2[0],ymm3[2],ymm2[2] -; AVX-NEXT: vshufps {{.*#+}} ymm12 = ymm12[0,1,2,0,4,5,6,4] ; AVX-NEXT: vextractf128 $1, %ymm12, %xmm12 ; AVX-NEXT: vunpcklps {{.*#+}} ymm13 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5] ; AVX-NEXT: vextractf128 $1, %ymm13, %xmm13 -; AVX-NEXT: vblendps {{.*#+}} xmm12 = xmm13[0,1],xmm12[2,3] +; AVX-NEXT: vshufps {{.*#+}} xmm12 = xmm13[0,1],xmm12[2,0] ; AVX-NEXT: vblendps {{.*#+}} ymm12 = ymm12[0,1,2,3],ymm8[4,5,6,7] ; AVX-NEXT: vunpcklps {{.*#+}} ymm8 = ymm11[0],ymm10[0],ymm11[1],ymm10[1],ymm11[4],ymm10[4],ymm11[5],ymm10[5] ; AVX-NEXT: vshufps {{.*#+}} ymm13 = ymm6[1,0],ymm7[1,0],ymm6[5,4],ymm7[5,4] @@ -1201,19 +1196,17 @@ define void @load_i32_stride8_vf8(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr ; AVX-NEXT: vunpcklps {{.*#+}} ymm13 = ymm2[0],ymm3[0],ymm2[1],ymm3[1],ymm2[4],ymm3[4],ymm2[5],ymm3[5] ; AVX-NEXT: vextractf128 $1, %ymm13, %xmm13 ; AVX-NEXT: vshufps {{.*#+}} ymm15 = ymm1[1,0],ymm0[1,0],ymm1[5,4],ymm0[5,4] -; AVX-NEXT: vshufps {{.*#+}} ymm15 = ymm15[2,0,2,3,6,4,6,7] ; AVX-NEXT: vextractf128 $1, %ymm15, %xmm15 -; AVX-NEXT: vblendps {{.*#+}} xmm13 = xmm15[0,1],xmm13[2,3] +; AVX-NEXT: vshufps {{.*#+}} xmm13 = xmm15[2,0],xmm13[2,3] ; AVX-NEXT: vblendps {{.*#+}} ymm8 = ymm13[0,1,2,3],ymm8[4,5,6,7] ; AVX-NEXT: vunpckhpd {{.*#+}} ymm13 = ymm10[1],ymm11[1],ymm10[3],ymm11[3] ; AVX-NEXT: vunpckhps {{.*#+}} ymm15 = ymm7[2],ymm6[2],ymm7[3],ymm6[3],ymm7[6],ymm6[6],ymm7[7],ymm6[7] ; AVX-NEXT: vshufps {{.*#+}} ymm13 = ymm15[0,1],ymm13[2,0],ymm15[4,5],ymm13[6,4] ; AVX-NEXT: vunpckhpd {{.*#+}} ymm15 = ymm3[1],ymm2[1],ymm3[3],ymm2[3] -; AVX-NEXT: vshufps {{.*#+}} ymm15 = ymm15[0,1,2,0,4,5,6,4] ; AVX-NEXT: vextractf128 $1, %ymm15, %xmm15 ; AVX-NEXT: vunpckhps {{.*#+}} ymm4 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7] ; AVX-NEXT: vextractf128 $1, %ymm4, %xmm4 -; AVX-NEXT: vblendps {{.*#+}} xmm4 = xmm4[0,1],xmm15[2,3] +; AVX-NEXT: vshufps {{.*#+}} xmm4 = xmm4[0,1],xmm15[2,0] ; AVX-NEXT: vblendps {{.*#+}} ymm4 = ymm4[0,1,2,3],ymm13[4,5,6,7] ; AVX-NEXT: vunpckhps {{.*#+}} ymm10 = ymm11[2],ymm10[2],ymm11[3],ymm10[3],ymm11[6],ymm10[6],ymm11[7],ymm10[7] ; AVX-NEXT: vshufps {{.*#+}} ymm6 = ymm6[3,0],ymm7[3,0],ymm6[7,4],ymm7[7,4] @@ -1221,9 +1214,8 @@ define void @load_i32_stride8_vf8(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr ; AVX-NEXT: vunpckhps {{.*#+}} ymm2 = ymm2[2],ymm3[2],ymm2[3],ymm3[3],ymm2[6],ymm3[6],ymm2[7],ymm3[7] ; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm1[3,0],ymm0[3,0],ymm1[7,4],ymm0[7,4] ; AVX-NEXT: vextractf128 $1, %ymm2, %xmm1 -; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm0[2,0,2,3,6,4,6,7] ; AVX-NEXT: vextractf128 $1, %ymm0, %xmm0 -; AVX-NEXT: vblendps {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3] +; AVX-NEXT: vshufps {{.*#+}} xmm0 = xmm0[2,0],xmm1[2,3] ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm6[4,5,6,7] ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload ; AVX-NEXT: vmovaps %ymm1, (%rsi) @@ -2302,34 +2294,35 @@ define void @load_i32_stride8_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt ; ; AVX-LABEL: load_i32_stride8_vf16: ; AVX: # %bb.0: -; AVX-NEXT: subq $616, %rsp # imm = 0x268 -; AVX-NEXT: vmovaps 32(%rdi), %xmm14 -; AVX-NEXT: vmovaps (%rdi), %xmm9 -; AVX-NEXT: vunpcklps {{.*#+}} xmm0 = xmm9[0],xmm14[0],xmm9[1],xmm14[1] +; AVX-NEXT: subq $584, %rsp # imm = 0x248 +; AVX-NEXT: vmovaps 32(%rdi), %xmm0 +; AVX-NEXT: vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; AVX-NEXT: vmovaps (%rdi), %xmm12 +; AVX-NEXT: vunpcklps {{.*#+}} xmm0 = xmm12[0],xmm0[0],xmm12[1],xmm0[1] ; AVX-NEXT: vmovaps 96(%rdi), %xmm1 ; AVX-NEXT: vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; AVX-NEXT: vmovaps 64(%rdi), %xmm2 ; AVX-NEXT: vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; AVX-NEXT: vunpcklps {{.*#+}} xmm8 = xmm2[0],xmm1[0],xmm2[1],xmm1[1] -; AVX-NEXT: vmovlhps {{.*#+}} xmm2 = xmm0[0],xmm8[0] -; AVX-NEXT: vmovaps 160(%rdi), %xmm5 +; AVX-NEXT: vunpcklps {{.*#+}} xmm9 = xmm2[0],xmm1[0],xmm2[1],xmm1[1] +; AVX-NEXT: vmovlhps {{.*#+}} xmm4 = xmm0[0],xmm9[0] +; AVX-NEXT: vmovaps 160(%rdi), %xmm8 ; AVX-NEXT: vmovaps 128(%rdi), %xmm10 -; AVX-NEXT: vunpcklps {{.*#+}} xmm0 = xmm10[0],xmm5[0],xmm10[1],xmm5[1] +; AVX-NEXT: vunpcklps {{.*#+}} xmm0 = xmm10[0],xmm8[0],xmm10[1],xmm8[1] ; AVX-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm6 ; AVX-NEXT: vmovaps 224(%rdi), %xmm0 ; AVX-NEXT: vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; AVX-NEXT: vmovaps 192(%rdi), %xmm1 ; AVX-NEXT: vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; AVX-NEXT: vunpcklps {{.*#+}} xmm4 = xmm1[0],xmm0[0],xmm1[1],xmm0[1] -; AVX-NEXT: vshufps {{.*#+}} xmm7 = xmm4[0,1,0,1] +; AVX-NEXT: vunpcklps {{.*#+}} xmm5 = xmm1[0],xmm0[0],xmm1[1],xmm0[1] +; AVX-NEXT: vshufps {{.*#+}} xmm7 = xmm5[0,1,0,1] ; AVX-NEXT: vinsertf128 $1, %xmm7, %ymm0, %ymm7 ; AVX-NEXT: vblendps {{.*#+}} ymm6 = ymm6[0,1,2,3,4,5],ymm7[6,7] -; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm6[4,5,6,7] +; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm4[0,1,2,3],ymm6[4,5,6,7] ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX-NEXT: vmovaps 416(%rdi), %xmm11 -; AVX-NEXT: vmovaps 384(%rdi), %xmm12 -; AVX-NEXT: vunpcklps {{.*#+}} xmm2 = xmm12[0],xmm11[0],xmm12[1],xmm11[1] -; AVX-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm2 +; AVX-NEXT: vmovaps 384(%rdi), %xmm13 +; AVX-NEXT: vunpcklps {{.*#+}} xmm4 = xmm13[0],xmm11[0],xmm13[1],xmm11[1] +; AVX-NEXT: vinsertf128 $1, %xmm4, %ymm0, %ymm4 ; AVX-NEXT: vmovaps 480(%rdi), %xmm0 ; AVX-NEXT: vmovaps %xmm0, (%rsp) # 16-byte Spill ; AVX-NEXT: vmovaps 448(%rdi), %xmm1 @@ -2337,87 +2330,88 @@ define void @load_i32_stride8_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt ; AVX-NEXT: vunpcklps {{.*#+}} xmm3 = xmm1[0],xmm0[0],xmm1[1],xmm0[1] ; AVX-NEXT: vshufps {{.*#+}} xmm6 = xmm3[0,1,0,1] ; AVX-NEXT: vinsertf128 $1, %xmm6, %ymm0, %ymm6 -; AVX-NEXT: vblendps {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm6[6,7] -; AVX-NEXT: vmovaps 288(%rdi), %xmm13 +; AVX-NEXT: vblendps {{.*#+}} ymm2 = ymm4[0,1,2,3,4,5],ymm6[6,7] +; AVX-NEXT: vmovaps 288(%rdi), %xmm14 ; AVX-NEXT: vmovaps 256(%rdi), %xmm15 -; AVX-NEXT: vunpcklps {{.*#+}} xmm1 = xmm15[0],xmm13[0],xmm15[1],xmm13[1] +; AVX-NEXT: vunpcklps {{.*#+}} xmm1 = xmm15[0],xmm14[0],xmm15[1],xmm14[1] ; AVX-NEXT: vmovaps 352(%rdi), %xmm7 ; AVX-NEXT: vmovaps 320(%rdi), %xmm6 ; AVX-NEXT: vunpcklps {{.*#+}} xmm0 = xmm6[0],xmm7[0],xmm6[1],xmm7[1] ; AVX-NEXT: vmovlhps {{.*#+}} xmm1 = xmm1[0],xmm0[0] ; AVX-NEXT: vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7] ; AVX-NEXT: vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vshufps {{.*#+}} xmm1 = xmm9[1,1,1,1] -; AVX-NEXT: vblendps {{.*#+}} xmm1 = xmm1[0],xmm14[1],xmm1[2,3] -; AVX-NEXT: vblendps {{.*#+}} xmm1 = xmm1[0,1],xmm8[2,3] -; AVX-NEXT: vinsertf128 $1, %xmm4, %ymm0, %ymm2 -; AVX-NEXT: vshufps {{.*#+}} xmm4 = xmm10[1,1,1,1] -; AVX-NEXT: vblendps {{.*#+}} xmm4 = xmm4[0],xmm5[1],xmm4[2,3] -; AVX-NEXT: vinsertf128 $1, %xmm4, %ymm0, %ymm4 -; AVX-NEXT: vblendps {{.*#+}} ymm2 = ymm4[0,1,2,3,4,5],ymm2[6,7] +; AVX-NEXT: vshufps {{.*#+}} xmm1 = xmm12[1,1,1,1] +; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload +; AVX-NEXT: vblendps {{.*#+}} xmm1 = xmm1[0],xmm4[1],xmm1[2,3] +; AVX-NEXT: vblendps {{.*#+}} xmm1 = xmm1[0,1],xmm9[2,3] +; AVX-NEXT: vinsertf128 $1, %xmm5, %ymm0, %ymm2 +; AVX-NEXT: vshufps {{.*#+}} xmm5 = xmm10[1,1,1,1] +; AVX-NEXT: vblendps {{.*#+}} xmm5 = xmm5[0],xmm8[1],xmm5[2,3] +; AVX-NEXT: vinsertf128 $1, %xmm5, %ymm0, %ymm5 +; AVX-NEXT: vblendps {{.*#+}} ymm2 = ymm5[0,1,2,3,4,5],ymm2[6,7] ; AVX-NEXT: vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7] ; AVX-NEXT: vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX-NEXT: vshufps {{.*#+}} xmm1 = xmm15[1,1,1,1] -; AVX-NEXT: vblendps {{.*#+}} xmm1 = xmm1[0],xmm13[1],xmm1[2,3] +; AVX-NEXT: vblendps {{.*#+}} xmm1 = xmm1[0],xmm14[1],xmm1[2,3] ; AVX-NEXT: vblendps {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3] ; AVX-NEXT: vinsertf128 $1, %xmm3, %ymm0, %ymm1 -; AVX-NEXT: vshufps {{.*#+}} xmm2 = xmm12[1,1,1,1] +; AVX-NEXT: vshufps {{.*#+}} xmm2 = xmm13[1,1,1,1] ; AVX-NEXT: vblendps {{.*#+}} xmm2 = xmm2[0],xmm11[1],xmm2[2,3] ; AVX-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm2 ; AVX-NEXT: vblendps {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5],ymm1[6,7] ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7] ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vunpckhps {{.*#+}} xmm1 = xmm9[2],xmm14[2],xmm9[3],xmm14[3] -; AVX-NEXT: vunpckhps {{.*#+}} xmm0 = xmm10[2],xmm5[2],xmm10[3],xmm5[3] -; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload -; AVX-NEXT: vshufps {{.*#+}} xmm2 = xmm14[2,2,2,2] -; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload -; AVX-NEXT: vblendps {{.*#+}} xmm2 = xmm10[0,1,2],xmm2[3] +; AVX-NEXT: vunpckhps {{.*#+}} xmm1 = xmm12[2],xmm4[2],xmm12[3],xmm4[3] +; AVX-NEXT: vunpckhps {{.*#+}} xmm4 = xmm10[2],xmm8[2],xmm10[3],xmm8[3] +; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload +; AVX-NEXT: vshufps {{.*#+}} xmm2 = xmm0[2,2,2,2] +; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Reload +; AVX-NEXT: vblendps {{.*#+}} xmm2 = xmm12[0,1,2],xmm2[3] ; AVX-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm2 -; AVX-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm3 +; AVX-NEXT: vinsertf128 $1, %xmm4, %ymm0, %ymm3 ; AVX-NEXT: vblendps {{.*#+}} ymm2 = ymm3[0,1,2,3,4,5],ymm2[6,7] +; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload +; AVX-NEXT: vshufps {{.*#+}} xmm3 = xmm10[2,2,2,2] ; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload -; AVX-NEXT: vshufps {{.*#+}} xmm3 = xmm9[2,2,2,2] -; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload -; AVX-NEXT: vblendps {{.*#+}} xmm3 = xmm8[0,1,2],xmm3[3] +; AVX-NEXT: vblendps {{.*#+}} xmm3 = xmm9[0,1,2],xmm3[3] ; AVX-NEXT: vblendps {{.*#+}} xmm3 = xmm1[0,1],xmm3[2,3] ; AVX-NEXT: vblendps {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7] ; AVX-NEXT: vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vunpckhps {{.*#+}} xmm2 = xmm15[2],xmm13[2],xmm15[3],xmm13[3] -; AVX-NEXT: vunpckhps {{.*#+}} xmm3 = xmm12[2],xmm11[2],xmm12[3],xmm11[3] -; AVX-NEXT: vmovaps (%rsp), %xmm15 # 16-byte Reload -; AVX-NEXT: vshufps {{.*#+}} xmm4 = xmm15[2,2,2,2] +; AVX-NEXT: vunpckhps {{.*#+}} xmm2 = xmm15[2],xmm14[2],xmm15[3],xmm14[3] +; AVX-NEXT: vunpckhps {{.*#+}} xmm3 = xmm13[2],xmm11[2],xmm13[3],xmm11[3] +; AVX-NEXT: vmovaps (%rsp), %xmm14 # 16-byte Reload +; AVX-NEXT: vshufps {{.*#+}} xmm5 = xmm14[2,2,2,2] ; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload -; AVX-NEXT: vblendps {{.*#+}} xmm4 = xmm11[0,1,2],xmm4[3] -; AVX-NEXT: vinsertf128 $1, %xmm4, %ymm0, %ymm4 -; AVX-NEXT: vinsertf128 $1, %xmm3, %ymm0, %ymm5 -; AVX-NEXT: vblendps {{.*#+}} ymm4 = ymm5[0,1,2,3,4,5],ymm4[6,7] -; AVX-NEXT: vshufps {{.*#+}} xmm5 = xmm7[2,2,2,2] -; AVX-NEXT: vblendps {{.*#+}} xmm5 = xmm6[0,1,2],xmm5[3] -; AVX-NEXT: vblendps {{.*#+}} xmm5 = xmm2[0,1],xmm5[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm4 = ymm5[0,1,2,3],ymm4[4,5,6,7] -; AVX-NEXT: vmovups %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vunpckhps {{.*#+}} xmm4 = xmm8[2],xmm9[2],xmm8[3],xmm9[3] -; AVX-NEXT: vunpckhpd {{.*#+}} xmm1 = xmm1[1],xmm4[1] -; AVX-NEXT: vunpckhps {{.*#+}} xmm4 = xmm10[2],xmm14[2],xmm10[3],xmm14[3] -; AVX-NEXT: vmovaps 288(%rdi), %ymm5 +; AVX-NEXT: vblendps {{.*#+}} xmm5 = xmm11[0,1,2],xmm5[3] +; AVX-NEXT: vinsertf128 $1, %xmm5, %ymm0, %ymm5 +; AVX-NEXT: vinsertf128 $1, %xmm3, %ymm0, %ymm8 +; AVX-NEXT: vblendps {{.*#+}} ymm5 = ymm8[0,1,2,3,4,5],ymm5[6,7] +; AVX-NEXT: vshufps {{.*#+}} xmm8 = xmm7[2,2,2,2] +; AVX-NEXT: vblendps {{.*#+}} xmm8 = xmm6[0,1,2],xmm8[3] +; AVX-NEXT: vblendps {{.*#+}} xmm8 = xmm2[0,1],xmm8[2,3] +; AVX-NEXT: vblendps {{.*#+}} ymm5 = ymm8[0,1,2,3],ymm5[4,5,6,7] ; AVX-NEXT: vmovups %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vinsertf128 $1, %xmm4, %ymm0, %ymm4 -; AVX-NEXT: vshufps {{.*#+}} xmm0 = xmm0[2,3,2,3] +; AVX-NEXT: vunpckhps {{.*#+}} xmm5 = xmm9[2],xmm10[2],xmm9[3],xmm10[3] +; AVX-NEXT: vunpckhpd {{.*#+}} xmm1 = xmm1[1],xmm5[1] +; AVX-NEXT: vunpckhps {{.*#+}} xmm5 = xmm12[2],xmm0[2],xmm12[3],xmm0[3] +; AVX-NEXT: vmovaps 320(%rdi), %ymm8 +; AVX-NEXT: vinsertf128 $1, %xmm5, %ymm0, %ymm5 +; AVX-NEXT: vshufps {{.*#+}} xmm0 = xmm4[2,3,2,3] ; AVX-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0 -; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm4[6,7] -; AVX-NEXT: vmovaps 256(%rdi), %ymm4 -; AVX-NEXT: vmovups %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm5[6,7] +; AVX-NEXT: vmovaps 352(%rdi), %ymm5 +; AVX-NEXT: vmovups %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7] ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vmovaps 416(%rdi), %ymm8 -; AVX-NEXT: vmovups %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vmovaps 416(%rdi), %ymm4 +; AVX-NEXT: vmovups %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX-NEXT: vunpckhps {{.*#+}} xmm0 = xmm6[2],xmm7[2],xmm6[3],xmm7[3] ; AVX-NEXT: vmovaps 384(%rdi), %ymm6 +; AVX-NEXT: vmovups %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX-NEXT: vunpckhpd {{.*#+}} xmm0 = xmm2[1],xmm0[1] ; AVX-NEXT: vmovaps 448(%rdi), %ymm7 ; AVX-NEXT: vmovups %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vunpckhps {{.*#+}} xmm1 = xmm11[2],xmm15[2],xmm11[3],xmm15[3] +; AVX-NEXT: vunpckhps {{.*#+}} xmm1 = xmm11[2],xmm14[2],xmm11[3],xmm14[3] ; AVX-NEXT: vmovaps 480(%rdi), %ymm9 ; AVX-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm1 ; AVX-NEXT: vshufps {{.*#+}} xmm2 = xmm3[2,3,2,3] @@ -2426,124 +2420,116 @@ define void @load_i32_stride8_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7] ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX-NEXT: vunpcklpd {{.*#+}} ymm0 = ymm9[0],ymm7[0],ymm9[2],ymm7[2] -; AVX-NEXT: vunpcklps {{.*#+}} ymm1 = ymm6[0],ymm8[0],ymm6[1],ymm8[1],ymm6[4],ymm8[4],ymm6[5],ymm8[5] -; AVX-NEXT: vmovups %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vmovaps %ymm9, %ymm3 +; AVX-NEXT: vunpcklps {{.*#+}} ymm1 = ymm6[0],ymm4[0],ymm6[1],ymm4[1],ymm6[4],ymm4[4],ymm6[5],ymm4[5] ; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,0],ymm1[4,5],ymm0[6,4] -; AVX-NEXT: vunpcklps {{.*#+}} ymm1 = ymm4[0],ymm5[0],ymm4[1],ymm5[1],ymm4[4],ymm5[4],ymm4[5],ymm5[5] +; AVX-NEXT: vunpcklpd {{.*#+}} ymm1 = ymm5[0],ymm8[0],ymm5[2],ymm8[2] +; AVX-NEXT: vmovaps %ymm8, %ymm6 ; AVX-NEXT: vextractf128 $1, %ymm1, %xmm1 -; AVX-NEXT: vmovaps 320(%rdi), %ymm10 -; AVX-NEXT: vmovaps 352(%rdi), %ymm11 -; AVX-NEXT: vunpcklpd {{.*#+}} ymm2 = ymm11[0],ymm10[0],ymm11[2],ymm10[2] -; AVX-NEXT: vmovups %ymm11, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vshufps {{.*#+}} ymm2 = ymm2[0,1,2,0,4,5,6,4] +; AVX-NEXT: vmovaps 288(%rdi), %ymm7 +; AVX-NEXT: vmovaps 256(%rdi), %ymm9 +; AVX-NEXT: vunpcklps {{.*#+}} ymm2 = ymm9[0],ymm7[0],ymm9[1],ymm7[1],ymm9[4],ymm7[4],ymm9[5],ymm7[5] ; AVX-NEXT: vextractf128 $1, %ymm2, %xmm2 -; AVX-NEXT: vblendps {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3] +; AVX-NEXT: vshufps {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,0] ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7] ; AVX-NEXT: vmovups %ymm0, (%rsp) # 32-byte Spill -; AVX-NEXT: vmovaps 160(%rdi), %ymm7 -; AVX-NEXT: vmovaps 128(%rdi), %ymm5 -; AVX-NEXT: vmovaps 192(%rdi), %ymm1 +; AVX-NEXT: vmovaps 160(%rdi), %ymm11 +; AVX-NEXT: vmovaps 128(%rdi), %ymm12 +; AVX-NEXT: vmovaps 192(%rdi), %ymm10 +; AVX-NEXT: vmovaps 224(%rdi), %ymm13 +; AVX-NEXT: vunpcklpd {{.*#+}} ymm0 = ymm13[0],ymm10[0],ymm13[2],ymm10[2] +; AVX-NEXT: vmovups %ymm10, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vunpcklps {{.*#+}} ymm1 = ymm12[0],ymm11[0],ymm12[1],ymm11[1],ymm12[4],ymm11[4],ymm12[5],ymm11[5] +; AVX-NEXT: vmovups %ymm12, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vmovups %ymm11, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vshufps {{.*#+}} ymm4 = ymm1[0,1],ymm0[2,0],ymm1[4,5],ymm0[6,4] +; AVX-NEXT: vmovaps 64(%rdi), %ymm1 ; AVX-NEXT: vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vmovaps 224(%rdi), %ymm0 +; AVX-NEXT: vmovaps 96(%rdi), %ymm0 ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX-NEXT: vunpcklpd {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[2],ymm1[2] -; AVX-NEXT: vunpcklps {{.*#+}} ymm1 = ymm5[0],ymm7[0],ymm5[1],ymm7[1],ymm5[4],ymm7[4],ymm5[5],ymm7[5] -; AVX-NEXT: vmovups %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vshufps {{.*#+}} ymm2 = ymm1[0,1],ymm0[2,0],ymm1[4,5],ymm0[6,4] -; AVX-NEXT: vmovaps 64(%rdi), %ymm0 -; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vmovaps 96(%rdi), %ymm14 -; AVX-NEXT: vunpcklpd {{.*#+}} ymm0 = ymm14[0],ymm0[0],ymm14[2],ymm0[2] -; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm0[0,1,2,0,4,5,6,4] ; AVX-NEXT: vextractf128 $1, %ymm0, %xmm0 -; AVX-NEXT: vmovaps (%rdi), %ymm13 -; AVX-NEXT: vmovaps 32(%rdi), %ymm12 -; AVX-NEXT: vunpcklps {{.*#+}} ymm15 = ymm13[0],ymm12[0],ymm13[1],ymm12[1],ymm13[4],ymm12[4],ymm13[5],ymm12[5] -; AVX-NEXT: vextractf128 $1, %ymm15, %xmm15 -; AVX-NEXT: vblendps {{.*#+}} xmm0 = xmm15[0,1],xmm0[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm2[4,5,6,7] +; AVX-NEXT: vmovaps (%rdi), %ymm1 +; AVX-NEXT: vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vmovaps 32(%rdi), %ymm15 +; AVX-NEXT: vunpcklps {{.*#+}} ymm14 = ymm1[0],ymm15[0],ymm1[1],ymm15[1],ymm1[4],ymm15[4],ymm1[5],ymm15[5] +; AVX-NEXT: vextractf128 $1, %ymm14, %xmm14 +; AVX-NEXT: vshufps {{.*#+}} xmm0 = xmm14[0,1],xmm0[2,0] +; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm4[4,5,6,7] ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload -; AVX-NEXT: vunpcklps {{.*#+}} ymm0 = ymm4[0],ymm9[0],ymm4[1],ymm9[1],ymm4[4],ymm9[4],ymm4[5],ymm9[5] -; AVX-NEXT: vmovaps %ymm9, %ymm8 -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Reload -; AVX-NEXT: vshufps {{.*#+}} ymm2 = ymm9[1,0],ymm6[1,0],ymm9[5,4],ymm6[5,4] -; AVX-NEXT: vshufps {{.*#+}} ymm6 = ymm2[2,0],ymm0[2,3],ymm2[6,4],ymm0[6,7] -; AVX-NEXT: vunpcklps {{.*#+}} ymm2 = ymm10[0],ymm11[0],ymm10[1],ymm11[1],ymm10[4],ymm11[4],ymm10[5],ymm11[5] -; AVX-NEXT: vextractf128 $1, %ymm2, %xmm2 ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload -; AVX-NEXT: vshufps {{.*#+}} ymm15 = ymm0[1,0],ymm1[1,0],ymm0[5,4],ymm1[5,4] -; AVX-NEXT: vshufps {{.*#+}} ymm15 = ymm15[2,0,2,3,6,4,6,7] -; AVX-NEXT: vextractf128 $1, %ymm15, %xmm15 -; AVX-NEXT: vblendps {{.*#+}} xmm2 = xmm15[0,1],xmm2[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm6[4,5,6,7] -; AVX-NEXT: vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vunpcklps {{.*#+}} ymm2 = ymm1[0],ymm3[0],ymm1[1],ymm3[1],ymm1[4],ymm3[4],ymm1[5],ymm3[5] +; AVX-NEXT: vmovaps %ymm3, %ymm8 ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload -; AVX-NEXT: vunpcklps {{.*#+}} ymm15 = ymm3[0],ymm6[0],ymm3[1],ymm6[1],ymm3[4],ymm6[4],ymm3[5],ymm6[5] -; AVX-NEXT: vshufps {{.*#+}} ymm2 = ymm7[1,0],ymm5[1,0],ymm7[5,4],ymm5[5,4] -; AVX-NEXT: vmovaps %ymm7, %ymm11 -; AVX-NEXT: vshufps {{.*#+}} ymm7 = ymm2[2,0],ymm15[2,3],ymm2[6,4],ymm15[6,7] -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload -; AVX-NEXT: vunpcklps {{.*#+}} ymm2 = ymm5[0],ymm14[0],ymm5[1],ymm14[1],ymm5[4],ymm14[4],ymm5[5],ymm14[5] -; AVX-NEXT: vextractf128 $1, %ymm2, %xmm2 -; AVX-NEXT: vshufps {{.*#+}} ymm15 = ymm12[1,0],ymm13[1,0],ymm12[5,4],ymm13[5,4] -; AVX-NEXT: vshufps {{.*#+}} ymm15 = ymm15[2,0,2,3,6,4,6,7] -; AVX-NEXT: vextractf128 $1, %ymm15, %xmm15 -; AVX-NEXT: vblendps {{.*#+}} xmm2 = xmm15[0,1],xmm2[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm7[4,5,6,7] -; AVX-NEXT: vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vunpckhpd {{.*#+}} ymm7 = ymm8[1],ymm4[1],ymm8[3],ymm4[3] -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload -; AVX-NEXT: vunpckhps {{.*#+}} ymm2 = ymm4[2],ymm9[2],ymm4[3],ymm9[3],ymm4[6],ymm9[6],ymm4[7],ymm9[7] -; AVX-NEXT: vshufps {{.*#+}} ymm9 = ymm2[0,1],ymm7[2,0],ymm2[4,5],ymm7[6,4] -; AVX-NEXT: vunpckhps {{.*#+}} ymm2 = ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[6],ymm0[6],ymm1[7],ymm0[7] -; AVX-NEXT: vextractf128 $1, %ymm2, %xmm2 +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload +; AVX-NEXT: vshufps {{.*#+}} ymm4 = ymm0[1,0],ymm3[1,0],ymm0[5,4],ymm3[5,4] +; AVX-NEXT: vshufps {{.*#+}} ymm5 = ymm4[2,0],ymm2[2,3],ymm4[6,4],ymm2[6,7] +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload +; AVX-NEXT: vunpcklps {{.*#+}} ymm4 = ymm6[0],ymm2[0],ymm6[1],ymm2[1],ymm6[4],ymm2[4],ymm6[5],ymm2[5] +; AVX-NEXT: vextractf128 $1, %ymm4, %xmm4 +; AVX-NEXT: vshufps {{.*#+}} ymm14 = ymm7[1,0],ymm9[1,0],ymm7[5,4],ymm9[5,4] +; AVX-NEXT: vextractf128 $1, %ymm14, %xmm14 +; AVX-NEXT: vshufps {{.*#+}} xmm4 = xmm14[2,0],xmm4[2,3] +; AVX-NEXT: vblendps {{.*#+}} ymm4 = ymm4[0,1,2,3],ymm5[4,5,6,7] +; AVX-NEXT: vmovups %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vunpcklps {{.*#+}} ymm5 = ymm10[0],ymm13[0],ymm10[1],ymm13[1],ymm10[4],ymm13[4],ymm10[5],ymm13[5] +; AVX-NEXT: vshufps {{.*#+}} ymm4 = ymm11[1,0],ymm12[1,0],ymm11[5,4],ymm12[5,4] +; AVX-NEXT: vshufps {{.*#+}} ymm5 = ymm4[2,0],ymm5[2,3],ymm4[6,4],ymm5[6,7] +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Reload +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm12 # 32-byte Reload +; AVX-NEXT: vunpcklps {{.*#+}} ymm4 = ymm10[0],ymm12[0],ymm10[1],ymm12[1],ymm10[4],ymm12[4],ymm10[5],ymm12[5] +; AVX-NEXT: vextractf128 $1, %ymm4, %xmm4 +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm11 # 32-byte Reload +; AVX-NEXT: vshufps {{.*#+}} ymm14 = ymm15[1,0],ymm11[1,0],ymm15[5,4],ymm11[5,4] +; AVX-NEXT: vextractf128 $1, %ymm14, %xmm14 +; AVX-NEXT: vshufps {{.*#+}} xmm4 = xmm14[2,0],xmm4[2,3] +; AVX-NEXT: vblendps {{.*#+}} ymm4 = ymm4[0,1,2,3],ymm5[4,5,6,7] +; AVX-NEXT: vmovups %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vunpckhpd {{.*#+}} ymm5 = ymm8[1],ymm1[1],ymm8[3],ymm1[3] +; AVX-NEXT: vunpckhps {{.*#+}} ymm4 = ymm3[2],ymm0[2],ymm3[3],ymm0[3],ymm3[6],ymm0[6],ymm3[7],ymm0[7] +; AVX-NEXT: vshufps {{.*#+}} ymm1 = ymm4[0,1],ymm5[2,0],ymm4[4,5],ymm5[6,4] +; AVX-NEXT: vmovaps %ymm6, %ymm3 +; AVX-NEXT: vunpckhpd {{.*#+}} ymm4 = ymm2[1],ymm6[1],ymm2[3],ymm6[3] +; AVX-NEXT: vextractf128 $1, %ymm4, %xmm4 +; AVX-NEXT: vmovaps %ymm7, %ymm2 +; AVX-NEXT: vunpckhps {{.*#+}} ymm14 = ymm9[2],ymm7[2],ymm9[3],ymm7[3],ymm9[6],ymm7[6],ymm9[7],ymm7[7] +; AVX-NEXT: vextractf128 $1, %ymm14, %xmm14 +; AVX-NEXT: vshufps {{.*#+}} xmm4 = xmm14[0,1],xmm4[2,0] +; AVX-NEXT: vblendps {{.*#+}} ymm5 = ymm4[0,1,2,3],ymm1[4,5,6,7] ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload -; AVX-NEXT: vunpckhpd {{.*#+}} ymm15 = ymm7[1],ymm10[1],ymm7[3],ymm10[3] -; AVX-NEXT: vshufps {{.*#+}} ymm15 = ymm15[0,1,2,0,4,5,6,4] -; AVX-NEXT: vextractf128 $1, %ymm15, %xmm15 -; AVX-NEXT: vblendps {{.*#+}} xmm2 = xmm2[0,1],xmm15[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm9[4,5,6,7] -; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vunpckhpd {{.*#+}} ymm0 = ymm6[1],ymm3[1],ymm6[3],ymm3[3] -; AVX-NEXT: vmovaps %ymm3, %ymm9 -; AVX-NEXT: vmovaps %ymm11, %ymm1 -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload -; AVX-NEXT: vunpckhps {{.*#+}} ymm2 = ymm3[2],ymm11[2],ymm3[3],ymm11[3],ymm3[6],ymm11[6],ymm3[7],ymm11[7] -; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm2[0,1],ymm0[2,0],ymm2[4,5],ymm0[6,4] -; AVX-NEXT: vunpckhpd {{.*#+}} ymm2 = ymm14[1],ymm5[1],ymm14[3],ymm5[3] -; AVX-NEXT: vshufps {{.*#+}} ymm2 = ymm2[0,1,2,0,4,5,6,4] -; AVX-NEXT: vextractf128 $1, %ymm2, %xmm2 -; AVX-NEXT: vunpckhps {{.*#+}} ymm15 = ymm13[2],ymm12[2],ymm13[3],ymm12[3],ymm13[6],ymm12[6],ymm13[7],ymm12[7] -; AVX-NEXT: vextractf128 $1, %ymm15, %xmm15 -; AVX-NEXT: vblendps {{.*#+}} xmm2 = xmm15[0,1],xmm2[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm11 = ymm2[0,1,2,3],ymm0[4,5,6,7] +; AVX-NEXT: vunpckhpd {{.*#+}} ymm0 = ymm13[1],ymm7[1],ymm13[3],ymm7[3] +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload +; AVX-NEXT: vunpckhps {{.*#+}} ymm4 = ymm6[2],ymm1[2],ymm6[3],ymm1[3],ymm6[6],ymm1[6],ymm6[7],ymm1[7] +; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm4[0,1],ymm0[2,0],ymm4[4,5],ymm0[6,4] +; AVX-NEXT: vunpckhpd {{.*#+}} ymm4 = ymm12[1],ymm10[1],ymm12[3],ymm10[3] +; AVX-NEXT: vextractf128 $1, %ymm4, %xmm4 +; AVX-NEXT: vunpckhps {{.*#+}} ymm14 = ymm11[2],ymm15[2],ymm11[3],ymm15[3],ymm11[6],ymm15[6],ymm11[7],ymm15[7] +; AVX-NEXT: vextractf128 $1, %ymm14, %xmm14 +; AVX-NEXT: vshufps {{.*#+}} xmm4 = xmm14[0,1],xmm4[2,0] +; AVX-NEXT: vblendps {{.*#+}} ymm14 = ymm4[0,1,2,3],ymm0[4,5,6,7] ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload ; AVX-NEXT: vunpckhps {{.*#+}} ymm0 = ymm0[2],ymm8[2],ymm0[3],ymm8[3],ymm0[6],ymm8[6],ymm0[7],ymm8[7] -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload -; AVX-NEXT: vshufps {{.*#+}} ymm2 = ymm2[3,0],ymm4[3,0],ymm2[7,4],ymm4[7,4] -; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm2[2,0],ymm0[2,3],ymm2[6,4],ymm0[6,7] -; AVX-NEXT: vunpckhps {{.*#+}} ymm2 = ymm10[2],ymm7[2],ymm10[3],ymm7[3],ymm10[6],ymm7[6],ymm10[7],ymm7[7] ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload -; AVX-NEXT: vshufps $51, {{[-0-9]+}}(%r{{[sb]}}p), %ymm4, %ymm7 # 32-byte Folded Reload -; AVX-NEXT: # ymm7 = ymm4[3,0],mem[3,0],ymm4[7,4],mem[7,4] -; AVX-NEXT: vextractf128 $1, %ymm2, %xmm2 -; AVX-NEXT: vshufps {{.*#+}} ymm7 = ymm7[2,0,2,3,6,4,6,7] -; AVX-NEXT: vextractf128 $1, %ymm7, %xmm7 -; AVX-NEXT: vblendps {{.*#+}} xmm2 = xmm7[0,1],xmm2[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm0[4,5,6,7] -; AVX-NEXT: vunpckhps {{.*#+}} ymm2 = ymm9[2],ymm6[2],ymm9[3],ymm6[3],ymm9[6],ymm6[6],ymm9[7],ymm6[7] -; AVX-NEXT: vshufps {{.*#+}} ymm6 = ymm1[3,0],ymm3[3,0],ymm1[7,4],ymm3[7,4] -; AVX-NEXT: vshufps {{.*#+}} ymm2 = ymm6[2,0],ymm2[2,3],ymm6[6,4],ymm2[6,7] -; AVX-NEXT: vunpckhps {{.*#+}} ymm4 = ymm5[2],ymm14[2],ymm5[3],ymm14[3],ymm5[6],ymm14[6],ymm5[7],ymm14[7] -; AVX-NEXT: vshufps {{.*#+}} ymm1 = ymm12[3,0],ymm13[3,0],ymm12[7,4],ymm13[7,4] -; AVX-NEXT: vextractf128 $1, %ymm4, %xmm3 -; AVX-NEXT: vshufps {{.*#+}} ymm1 = ymm1[2,0,2,3,6,4,6,7] +; AVX-NEXT: vshufps $51, {{[-0-9]+}}(%r{{[sb]}}p), %ymm4, %ymm4 # 32-byte Folded Reload +; AVX-NEXT: # ymm4 = ymm4[3,0],mem[3,0],ymm4[7,4],mem[7,4] +; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm4[2,0],ymm0[2,3],ymm4[6,4],ymm0[6,7] +; AVX-NEXT: vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %ymm3, %ymm4 # 32-byte Folded Reload +; AVX-NEXT: # ymm4 = ymm3[2],mem[2],ymm3[3],mem[3],ymm3[6],mem[6],ymm3[7],mem[7] +; AVX-NEXT: vshufps {{.*#+}} ymm9 = ymm2[3,0],ymm9[3,0],ymm2[7,4],ymm9[7,4] +; AVX-NEXT: vextractf128 $1, %ymm4, %xmm4 +; AVX-NEXT: vextractf128 $1, %ymm9, %xmm9 +; AVX-NEXT: vshufps {{.*#+}} xmm4 = xmm9[2,0],xmm4[2,3] +; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm4[0,1,2,3],ymm0[4,5,6,7] +; AVX-NEXT: vunpckhps {{.*#+}} ymm4 = ymm7[2],ymm13[2],ymm7[3],ymm13[3],ymm7[6],ymm13[6],ymm7[7],ymm13[7] +; AVX-NEXT: vshufps {{.*#+}} ymm6 = ymm1[3,0],ymm6[3,0],ymm1[7,4],ymm6[7,4] +; AVX-NEXT: vshufps {{.*#+}} ymm4 = ymm6[2,0],ymm4[2,3],ymm6[6,4],ymm4[6,7] +; AVX-NEXT: vunpckhps {{.*#+}} ymm3 = ymm10[2],ymm12[2],ymm10[3],ymm12[3],ymm10[6],ymm12[6],ymm10[7],ymm12[7] +; AVX-NEXT: vshufps {{.*#+}} ymm1 = ymm15[3,0],ymm11[3,0],ymm15[7,4],ymm11[7,4] +; AVX-NEXT: vextractf128 $1, %ymm3, %xmm2 ; AVX-NEXT: vextractf128 $1, %ymm1, %xmm1 -; AVX-NEXT: vblendps {{.*#+}} xmm1 = xmm1[0,1],xmm3[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7] +; AVX-NEXT: vshufps {{.*#+}} xmm1 = xmm1[2,0],xmm2[2,3] +; AVX-NEXT: vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm4[4,5,6,7] ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload ; AVX-NEXT: vmovaps %ymm2, 32(%rsi) ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload @@ -2570,13 +2556,12 @@ define void @load_i32_stride8_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload ; AVX-NEXT: vmovaps %ymm2, (%rax) ; AVX-NEXT: movq {{[0-9]+}}(%rsp), %rax -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm2, 32(%rax) -; AVX-NEXT: vmovaps %ymm11, (%rax) +; AVX-NEXT: vmovaps %ymm5, 32(%rax) +; AVX-NEXT: vmovaps %ymm14, (%rax) ; AVX-NEXT: movq {{[0-9]+}}(%rsp), %rax ; AVX-NEXT: vmovaps %ymm0, 32(%rax) ; AVX-NEXT: vmovaps %ymm1, (%rax) -; AVX-NEXT: addq $616, %rsp # imm = 0x268 +; AVX-NEXT: addq $584, %rsp # imm = 0x248 ; AVX-NEXT: vzeroupper ; AVX-NEXT: retq ; @@ -4892,7 +4877,7 @@ define void @load_i32_stride8_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt ; ; AVX-LABEL: load_i32_stride8_vf32: ; AVX: # %bb.0: -; AVX-NEXT: subq $1800, %rsp # imm = 0x708 +; AVX-NEXT: subq $1768, %rsp # imm = 0x6E8 ; AVX-NEXT: vmovaps 288(%rdi), %xmm14 ; AVX-NEXT: vmovaps 256(%rdi), %xmm10 ; AVX-NEXT: vunpcklps {{.*#+}} xmm0 = xmm10[0],xmm14[0],xmm10[1],xmm14[1] @@ -5175,84 +5160,81 @@ define void @load_i32_stride8_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt ; AVX-NEXT: vblendps {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5],ymm1[6,7] ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7] ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vmovaps 416(%rdi), %ymm13 -; AVX-NEXT: vmovaps 384(%rdi), %ymm5 -; AVX-NEXT: vmovaps 448(%rdi), %ymm9 -; AVX-NEXT: vmovaps 480(%rdi), %ymm12 -; AVX-NEXT: vunpcklpd {{.*#+}} ymm0 = ymm12[0],ymm9[0],ymm12[2],ymm9[2] +; AVX-NEXT: vmovaps 416(%rdi), %ymm12 +; AVX-NEXT: vmovaps 384(%rdi), %ymm9 +; AVX-NEXT: vmovaps 448(%rdi), %ymm7 +; AVX-NEXT: vmovaps 480(%rdi), %ymm11 +; AVX-NEXT: vunpcklpd {{.*#+}} ymm0 = ymm11[0],ymm7[0],ymm11[2],ymm7[2] +; AVX-NEXT: vmovups %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vunpcklps {{.*#+}} ymm1 = ymm9[0],ymm12[0],ymm9[1],ymm12[1],ymm9[4],ymm12[4],ymm9[5],ymm12[5] ; AVX-NEXT: vmovups %ymm9, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vunpcklps {{.*#+}} ymm1 = ymm5[0],ymm13[0],ymm5[1],ymm13[1],ymm5[4],ymm13[4],ymm5[5],ymm13[5] -; AVX-NEXT: vmovups %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,0],ymm1[4,5],ymm0[6,4] -; AVX-NEXT: vmovaps 288(%rdi), %ymm2 +; AVX-NEXT: vmovaps 320(%rdi), %ymm2 ; AVX-NEXT: vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vmovaps 256(%rdi), %ymm1 +; AVX-NEXT: vmovaps 352(%rdi), %ymm1 ; AVX-NEXT: vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vunpcklps {{.*#+}} ymm1 = ymm1[0],ymm2[0],ymm1[1],ymm2[1],ymm1[4],ymm2[4],ymm1[5],ymm2[5] +; AVX-NEXT: vunpcklpd {{.*#+}} ymm1 = ymm1[0],ymm2[0],ymm1[2],ymm2[2] ; AVX-NEXT: vextractf128 $1, %ymm1, %xmm1 -; AVX-NEXT: vmovaps 320(%rdi), %ymm2 +; AVX-NEXT: vmovaps 288(%rdi), %ymm2 ; AVX-NEXT: vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vmovaps 352(%rdi), %ymm11 -; AVX-NEXT: vunpcklpd {{.*#+}} ymm2 = ymm11[0],ymm2[0],ymm11[2],ymm2[2] -; AVX-NEXT: vmovups %ymm11, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vshufps {{.*#+}} ymm2 = ymm2[0,1,2,0,4,5,6,4] +; AVX-NEXT: vmovaps 256(%rdi), %ymm10 +; AVX-NEXT: vunpcklps {{.*#+}} ymm2 = ymm10[0],ymm2[0],ymm10[1],ymm2[1],ymm10[4],ymm2[4],ymm10[5],ymm2[5] +; AVX-NEXT: vmovups %ymm10, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX-NEXT: vextractf128 $1, %ymm2, %xmm2 -; AVX-NEXT: vblendps {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3] +; AVX-NEXT: vshufps {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,0] ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7] ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vmovaps 928(%rdi), %ymm2 -; AVX-NEXT: vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vmovaps 928(%rdi), %ymm5 ; AVX-NEXT: vmovaps 896(%rdi), %ymm3 -; AVX-NEXT: vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vmovups %ymm3, (%rsp) # 32-byte Spill ; AVX-NEXT: vmovaps 960(%rdi), %ymm1 ; AVX-NEXT: vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX-NEXT: vmovaps 992(%rdi), %ymm0 -; AVX-NEXT: vmovups %ymm0, (%rsp) # 32-byte Spill +; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX-NEXT: vunpcklpd {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[2],ymm1[2] -; AVX-NEXT: vunpcklps {{.*#+}} ymm1 = ymm3[0],ymm2[0],ymm3[1],ymm2[1],ymm3[4],ymm2[4],ymm3[5],ymm2[5] +; AVX-NEXT: vunpcklps {{.*#+}} ymm1 = ymm3[0],ymm5[0],ymm3[1],ymm5[1],ymm3[4],ymm5[4],ymm3[5],ymm5[5] +; AVX-NEXT: vmovups %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,0],ymm1[4,5],ymm0[6,4] -; AVX-NEXT: vmovaps 800(%rdi), %ymm1 -; AVX-NEXT: vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vmovaps 768(%rdi), %ymm2 +; AVX-NEXT: vmovaps 832(%rdi), %ymm2 ; AVX-NEXT: vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vunpcklps {{.*#+}} ymm1 = ymm2[0],ymm1[0],ymm2[1],ymm1[1],ymm2[4],ymm1[4],ymm2[5],ymm1[5] +; AVX-NEXT: vmovaps 864(%rdi), %ymm1 +; AVX-NEXT: vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vunpcklpd {{.*#+}} ymm1 = ymm1[0],ymm2[0],ymm1[2],ymm2[2] ; AVX-NEXT: vextractf128 $1, %ymm1, %xmm1 -; AVX-NEXT: vmovaps 832(%rdi), %ymm3 +; AVX-NEXT: vmovaps 800(%rdi), %ymm3 ; AVX-NEXT: vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vmovaps 864(%rdi), %ymm2 +; AVX-NEXT: vmovaps 768(%rdi), %ymm2 ; AVX-NEXT: vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vunpcklpd {{.*#+}} ymm2 = ymm2[0],ymm3[0],ymm2[2],ymm3[2] -; AVX-NEXT: vshufps {{.*#+}} ymm2 = ymm2[0,1,2,0,4,5,6,4] +; AVX-NEXT: vunpcklps {{.*#+}} ymm2 = ymm2[0],ymm3[0],ymm2[1],ymm3[1],ymm2[4],ymm3[4],ymm2[5],ymm3[5] ; AVX-NEXT: vextractf128 $1, %ymm2, %xmm2 -; AVX-NEXT: vblendps {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3] +; AVX-NEXT: vshufps {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,0] ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7] ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX-NEXT: vmovaps 672(%rdi), %ymm2 ; AVX-NEXT: vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vmovaps 640(%rdi), %ymm1 +; AVX-NEXT: vmovaps 640(%rdi), %ymm3 +; AVX-NEXT: vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vmovaps 704(%rdi), %ymm13 +; AVX-NEXT: vmovaps 736(%rdi), %ymm0 +; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vunpcklpd {{.*#+}} ymm0 = ymm0[0],ymm13[0],ymm0[2],ymm13[2] +; AVX-NEXT: vmovups %ymm13, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vunpcklps {{.*#+}} ymm1 = ymm3[0],ymm2[0],ymm3[1],ymm2[1],ymm3[4],ymm2[4],ymm3[5],ymm2[5] +; AVX-NEXT: vshufps {{.*#+}} ymm4 = ymm1[0,1],ymm0[2,0],ymm1[4,5],ymm0[6,4] +; AVX-NEXT: vmovaps 576(%rdi), %ymm1 ; AVX-NEXT: vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vmovaps 704(%rdi), %ymm0 +; AVX-NEXT: vmovaps 608(%rdi), %ymm0 ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vmovaps 736(%rdi), %ymm10 -; AVX-NEXT: vunpcklpd {{.*#+}} ymm0 = ymm10[0],ymm0[0],ymm10[2],ymm0[2] -; AVX-NEXT: vmovups %ymm10, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vunpcklps {{.*#+}} ymm1 = ymm1[0],ymm2[0],ymm1[1],ymm2[1],ymm1[4],ymm2[4],ymm1[5],ymm2[5] -; AVX-NEXT: vshufps {{.*#+}} ymm3 = ymm1[0,1],ymm0[2,0],ymm1[4,5],ymm0[6,4] +; AVX-NEXT: vunpcklpd {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[2],ymm1[2] +; AVX-NEXT: vextractf128 $1, %ymm0, %xmm6 ; AVX-NEXT: vmovaps 544(%rdi), %ymm0 ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX-NEXT: vmovaps 512(%rdi), %ymm1 ; AVX-NEXT: vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vunpcklps {{.*#+}} ymm0 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[4],ymm0[4],ymm1[5],ymm0[5] -; AVX-NEXT: vextractf128 $1, %ymm0, %xmm6 -; AVX-NEXT: vmovaps 576(%rdi), %ymm0 -; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vmovaps 608(%rdi), %ymm1 -; AVX-NEXT: vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vunpcklpd {{.*#+}} ymm8 = ymm1[0],ymm0[0],ymm1[2],ymm0[2] -; AVX-NEXT: vshufps {{.*#+}} ymm8 = ymm8[0,1,2,0,4,5,6,4] +; AVX-NEXT: vunpcklps {{.*#+}} ymm8 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[4],ymm0[4],ymm1[5],ymm0[5] ; AVX-NEXT: vextractf128 $1, %ymm8, %xmm8 -; AVX-NEXT: vblendps {{.*#+}} xmm6 = xmm6[0,1],xmm8[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm6[0,1,2,3],ymm3[4,5,6,7] +; AVX-NEXT: vshufps {{.*#+}} xmm6 = xmm8[0,1],xmm6[2,0] +; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm6[0,1,2,3],ymm4[4,5,6,7] ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX-NEXT: vmovaps 160(%rdi), %ymm0 ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill @@ -5262,67 +5244,63 @@ define void @load_i32_stride8_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt ; AVX-NEXT: vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX-NEXT: vmovaps 224(%rdi), %ymm3 ; AVX-NEXT: vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vunpcklpd {{.*#+}} ymm3 = ymm3[0],ymm2[0],ymm3[2],ymm2[2] +; AVX-NEXT: vunpcklpd {{.*#+}} ymm4 = ymm3[0],ymm2[0],ymm3[2],ymm2[2] ; AVX-NEXT: vunpcklps {{.*#+}} ymm6 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[4],ymm0[4],ymm1[5],ymm0[5] -; AVX-NEXT: vshufps {{.*#+}} ymm1 = ymm6[0,1],ymm3[2,0],ymm6[4,5],ymm3[6,4] -; AVX-NEXT: vmovaps 64(%rdi), %ymm14 +; AVX-NEXT: vshufps {{.*#+}} ymm1 = ymm6[0,1],ymm4[2,0],ymm6[4,5],ymm4[6,4] +; AVX-NEXT: vmovaps 64(%rdi), %ymm0 +; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX-NEXT: vmovaps 96(%rdi), %ymm2 ; AVX-NEXT: vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vunpcklpd {{.*#+}} ymm3 = ymm2[0],ymm14[0],ymm2[2],ymm14[2] -; AVX-NEXT: vmovups %ymm14, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vshufps {{.*#+}} ymm3 = ymm3[0,1,2,0,4,5,6,4] -; AVX-NEXT: vextractf128 $1, %ymm3, %xmm0 +; AVX-NEXT: vunpcklpd {{.*#+}} ymm4 = ymm2[0],ymm0[0],ymm2[2],ymm0[2] +; AVX-NEXT: vextractf128 $1, %ymm4, %xmm0 ; AVX-NEXT: vmovaps (%rdi), %ymm2 ; AVX-NEXT: vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vmovaps 32(%rdi), %ymm3 -; AVX-NEXT: vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vunpcklps {{.*#+}} ymm15 = ymm2[0],ymm3[0],ymm2[1],ymm3[1],ymm2[4],ymm3[4],ymm2[5],ymm3[5] +; AVX-NEXT: vmovaps 32(%rdi), %ymm14 +; AVX-NEXT: vunpcklps {{.*#+}} ymm15 = ymm2[0],ymm14[0],ymm2[1],ymm14[1],ymm2[4],ymm14[4],ymm2[5],ymm14[5] ; AVX-NEXT: vextractf128 $1, %ymm15, %xmm15 -; AVX-NEXT: vblendps {{.*#+}} xmm0 = xmm15[0,1],xmm0[2,3] +; AVX-NEXT: vshufps {{.*#+}} xmm0 = xmm15[0,1],xmm0[2,0] ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7] ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vmovaps %ymm11, %ymm6 +; AVX-NEXT: vmovups %ymm11, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vunpcklps {{.*#+}} ymm0 = ymm7[0],ymm11[0],ymm7[1],ymm11[1],ymm7[4],ymm11[4],ymm7[5],ymm11[5] +; AVX-NEXT: vmovaps %ymm12, %ymm7 ; AVX-NEXT: vmovups %ymm12, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vunpcklps {{.*#+}} ymm0 = ymm9[0],ymm12[0],ymm9[1],ymm12[1],ymm9[4],ymm12[4],ymm9[5],ymm12[5] -; AVX-NEXT: vmovaps %ymm13, %ymm8 -; AVX-NEXT: vmovups %ymm13, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vshufps {{.*#+}} ymm1 = ymm13[1,0],ymm5[1,0],ymm13[5,4],ymm5[5,4] +; AVX-NEXT: vshufps {{.*#+}} ymm1 = ymm12[1,0],ymm9[1,0],ymm12[5,4],ymm9[5,4] ; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm1[2,0],ymm0[2,3],ymm1[6,4],ymm0[6,7] -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm13 # 32-byte Reload -; AVX-NEXT: vunpcklps {{.*#+}} ymm1 = ymm13[0],ymm11[0],ymm13[1],ymm11[1],ymm13[4],ymm11[4],ymm13[5],ymm11[5] -; AVX-NEXT: vextractf128 $1, %ymm1, %xmm1 -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Reload ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload -; AVX-NEXT: vshufps {{.*#+}} ymm15 = ymm3[1,0],ymm5[1,0],ymm3[5,4],ymm5[5,4] -; AVX-NEXT: vshufps {{.*#+}} ymm15 = ymm15[2,0,2,3,6,4,6,7] +; AVX-NEXT: vunpcklps {{.*#+}} ymm1 = ymm3[0],ymm9[0],ymm3[1],ymm9[1],ymm3[4],ymm9[4],ymm3[5],ymm9[5] +; AVX-NEXT: vextractf128 $1, %ymm1, %xmm1 +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm11 # 32-byte Reload +; AVX-NEXT: vshufps {{.*#+}} ymm15 = ymm11[1,0],ymm10[1,0],ymm11[5,4],ymm10[5,4] ; AVX-NEXT: vextractf128 $1, %ymm15, %xmm15 -; AVX-NEXT: vblendps {{.*#+}} xmm1 = xmm15[0,1],xmm1[2,3] +; AVX-NEXT: vshufps {{.*#+}} xmm1 = xmm15[2,0],xmm1[2,3] ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7] ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vmovups (%rsp), %ymm11 # 32-byte Reload -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload -; AVX-NEXT: vunpcklps {{.*#+}} ymm0 = ymm7[0],ymm11[0],ymm7[1],ymm11[1],ymm7[4],ymm11[4],ymm7[5],ymm11[5] -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Reload -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload -; AVX-NEXT: vshufps {{.*#+}} ymm1 = ymm1[1,0],ymm9[1,0],ymm1[5,4],ymm9[5,4] +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Reload +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload +; AVX-NEXT: vunpcklps {{.*#+}} ymm0 = ymm2[0],ymm10[0],ymm2[1],ymm10[1],ymm2[4],ymm10[4],ymm2[5],ymm10[5] +; AVX-NEXT: vmovups (%rsp), %ymm12 # 32-byte Reload +; AVX-NEXT: vshufps {{.*#+}} ymm1 = ymm5[1,0],ymm12[1,0],ymm5[5,4],ymm12[5,4] ; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm1[2,0],ymm0[2,3],ymm1[6,4],ymm0[6,7] -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload -; AVX-NEXT: vunpcklps {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm1 # 32-byte Folded Reload -; AVX-NEXT: # ymm1 = ymm1[0],mem[0],ymm1[1],mem[1],ymm1[4],mem[4],ymm1[5],mem[5] +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload +; AVX-NEXT: vunpcklps {{[-0-9]+}}(%r{{[sb]}}p), %ymm5, %ymm1 # 32-byte Folded Reload +; AVX-NEXT: # ymm1 = ymm5[0],mem[0],ymm5[1],mem[1],ymm5[4],mem[4],ymm5[5],mem[5] ; AVX-NEXT: vextractf128 $1, %ymm1, %xmm1 -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload -; AVX-NEXT: vshufps {{.*#+}} ymm15 = ymm6[1,0],ymm4[1,0],ymm6[5,4],ymm4[5,4] -; AVX-NEXT: vshufps {{.*#+}} ymm15 = ymm15[2,0,2,3,6,4,6,7] +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload +; AVX-NEXT: vshufps $17, {{[-0-9]+}}(%r{{[sb]}}p), %ymm8, %ymm15 # 32-byte Folded Reload +; AVX-NEXT: # ymm15 = ymm8[1,0],mem[1,0],ymm8[5,4],mem[5,4] ; AVX-NEXT: vextractf128 $1, %ymm15, %xmm15 -; AVX-NEXT: vblendps {{.*#+}} xmm1 = xmm15[0,1],xmm1[2,3] +; AVX-NEXT: vshufps {{.*#+}} xmm1 = xmm15[2,0],xmm1[2,3] ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7] ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload -; AVX-NEXT: vunpcklps {{.*#+}} ymm0 = ymm0[0],ymm10[0],ymm0[1],ymm10[1],ymm0[4],ymm10[4],ymm0[5],ymm10[5] -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Reload -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload -; AVX-NEXT: vshufps {{.*#+}} ymm1 = ymm6[1,0],ymm10[1,0],ymm6[5,4],ymm10[5,4] -; AVX-NEXT: vshufps {{.*#+}} ymm2 = ymm1[2,0],ymm0[2,3],ymm1[6,4],ymm0[6,7] +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload +; AVX-NEXT: vunpcklps {{.*#+}} ymm0 = ymm13[0],ymm4[0],ymm13[1],ymm4[1],ymm13[4],ymm4[4],ymm13[5],ymm4[5] +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm13 # 32-byte Reload +; AVX-NEXT: vshufps $17, {{[-0-9]+}}(%r{{[sb]}}p), %ymm13, %ymm1 # 32-byte Folded Reload +; AVX-NEXT: # ymm1 = ymm13[1,0],mem[1,0],ymm13[5,4],mem[5,4] +; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm1[2,0],ymm0[2,3],ymm1[6,4],ymm0[6,7] ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload ; AVX-NEXT: vunpcklps {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm1 # 32-byte Folded Reload ; AVX-NEXT: # ymm1 = ymm1[0],mem[0],ymm1[1],mem[1],ymm1[4],mem[4],ymm1[5],mem[5] @@ -5330,10 +5308,9 @@ define void @load_i32_stride8_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm15 # 32-byte Reload ; AVX-NEXT: vshufps $17, {{[-0-9]+}}(%r{{[sb]}}p), %ymm15, %ymm15 # 32-byte Folded Reload ; AVX-NEXT: # ymm15 = ymm15[1,0],mem[1,0],ymm15[5,4],mem[5,4] -; AVX-NEXT: vshufps {{.*#+}} ymm15 = ymm15[2,0,2,3,6,4,6,7] ; AVX-NEXT: vextractf128 $1, %ymm15, %xmm15 -; AVX-NEXT: vblendps {{.*#+}} xmm1 = xmm15[0,1],xmm1[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm2[4,5,6,7] +; AVX-NEXT: vshufps {{.*#+}} xmm1 = xmm15[2,0],xmm1[2,3] +; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7] ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload ; AVX-NEXT: vunpcklps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload @@ -5342,78 +5319,73 @@ define void @load_i32_stride8_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt ; AVX-NEXT: vshufps $17, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm1 # 32-byte Folded Reload ; AVX-NEXT: # ymm1 = ymm1[1,0],mem[1,0],ymm1[5,4],mem[5,4] ; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm1[2,0],ymm0[2,3],ymm1[6,4],ymm0[6,7] -; AVX-NEXT: vunpcklps {{[-0-9]+}}(%r{{[sb]}}p), %ymm14, %ymm1 # 32-byte Folded Reload -; AVX-NEXT: # ymm1 = ymm14[0],mem[0],ymm14[1],mem[1],ymm14[4],mem[4],ymm14[5],mem[5] +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload +; AVX-NEXT: vunpcklps {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm1 # 32-byte Folded Reload +; AVX-NEXT: # ymm1 = ymm1[0],mem[0],ymm1[1],mem[1],ymm1[4],mem[4],ymm1[5],mem[5] ; AVX-NEXT: vextractf128 $1, %ymm1, %xmm1 -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm14 # 32-byte Reload -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload -; AVX-NEXT: vshufps {{.*#+}} ymm15 = ymm14[1,0],ymm2[1,0],ymm14[5,4],ymm2[5,4] -; AVX-NEXT: vshufps {{.*#+}} ymm15 = ymm15[2,0,2,3,6,4,6,7] +; AVX-NEXT: vshufps $17, {{[-0-9]+}}(%r{{[sb]}}p), %ymm14, %ymm15 # 32-byte Folded Reload +; AVX-NEXT: # ymm15 = ymm14[1,0],mem[1,0],ymm14[5,4],mem[5,4] ; AVX-NEXT: vextractf128 $1, %ymm15, %xmm15 -; AVX-NEXT: vblendps {{.*#+}} xmm1 = xmm15[0,1],xmm1[2,3] +; AVX-NEXT: vshufps {{.*#+}} xmm1 = xmm15[2,0],xmm1[2,3] ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7] ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vunpckhpd {{[-0-9]+}}(%r{{[sb]}}p), %ymm12, %ymm0 # 32-byte Folded Reload -; AVX-NEXT: # ymm0 = ymm12[1],mem[1],ymm12[3],mem[3] +; AVX-NEXT: vunpckhpd {{[-0-9]+}}(%r{{[sb]}}p), %ymm6, %ymm0 # 32-byte Folded Reload +; AVX-NEXT: # ymm0 = ymm6[1],mem[1],ymm6[3],mem[3] ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload -; AVX-NEXT: vunpckhps {{.*#+}} ymm1 = ymm1[2],ymm8[2],ymm1[3],ymm8[3],ymm1[6],ymm8[6],ymm1[7],ymm8[7] +; AVX-NEXT: vunpckhps {{.*#+}} ymm1 = ymm1[2],ymm7[2],ymm1[3],ymm7[3],ymm1[6],ymm7[6],ymm1[7],ymm7[7] ; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,0],ymm1[4,5],ymm0[6,4] -; AVX-NEXT: vunpckhps {{.*#+}} ymm1 = ymm5[2],ymm3[2],ymm5[3],ymm3[3],ymm5[6],ymm3[6],ymm5[7],ymm3[7] +; AVX-NEXT: vunpckhpd {{.*#+}} ymm1 = ymm9[1],ymm3[1],ymm9[3],ymm3[3] ; AVX-NEXT: vextractf128 $1, %ymm1, %xmm1 -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm12 # 32-byte Reload -; AVX-NEXT: vunpckhpd {{.*#+}} ymm15 = ymm12[1],ymm13[1],ymm12[3],ymm13[3] -; AVX-NEXT: vshufps {{.*#+}} ymm15 = ymm15[0,1,2,0,4,5,6,4] +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Reload +; AVX-NEXT: vunpckhps {{.*#+}} ymm15 = ymm9[2],ymm11[2],ymm9[3],ymm11[3],ymm9[6],ymm11[6],ymm9[7],ymm11[7] ; AVX-NEXT: vextractf128 $1, %ymm15, %xmm15 -; AVX-NEXT: vblendps {{.*#+}} xmm1 = xmm1[0,1],xmm15[2,3] +; AVX-NEXT: vshufps {{.*#+}} xmm1 = xmm15[0,1],xmm1[2,0] ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7] ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vunpckhpd {{.*#+}} ymm0 = ymm11[1],ymm7[1],ymm11[3],ymm7[3] -; AVX-NEXT: vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %ymm9, %ymm1 # 32-byte Folded Reload -; AVX-NEXT: # ymm1 = ymm9[2],mem[2],ymm9[3],mem[3],ymm9[6],mem[6],ymm9[7],mem[7] +; AVX-NEXT: vunpckhpd {{.*#+}} ymm0 = ymm10[1],ymm2[1],ymm10[3],ymm2[3] +; AVX-NEXT: vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %ymm12, %ymm1 # 32-byte Folded Reload +; AVX-NEXT: # ymm1 = ymm12[2],mem[2],ymm12[3],mem[3],ymm12[6],mem[6],ymm12[7],mem[7] ; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,0],ymm1[4,5],ymm0[6,4] -; AVX-NEXT: vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %ymm4, %ymm1 # 32-byte Folded Reload -; AVX-NEXT: # ymm1 = ymm4[2],mem[2],ymm4[3],mem[3],ymm4[6],mem[6],ymm4[7],mem[7] +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload +; AVX-NEXT: vunpckhpd {{.*#+}} ymm1 = ymm1[1],ymm5[1],ymm1[3],ymm5[3] ; AVX-NEXT: vextractf128 $1, %ymm1, %xmm1 -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload -; AVX-NEXT: vunpckhpd {{[-0-9]+}}(%r{{[sb]}}p), %ymm3, %ymm15 # 32-byte Folded Reload -; AVX-NEXT: # ymm15 = ymm3[1],mem[1],ymm3[3],mem[3] -; AVX-NEXT: vshufps {{.*#+}} ymm15 = ymm15[0,1,2,0,4,5,6,4] +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload +; AVX-NEXT: vunpckhps {{.*#+}} ymm15 = ymm2[2],ymm8[2],ymm2[3],ymm8[3],ymm2[6],ymm8[6],ymm2[7],ymm8[7] ; AVX-NEXT: vextractf128 $1, %ymm15, %xmm15 -; AVX-NEXT: vblendps {{.*#+}} xmm1 = xmm1[0,1],xmm15[2,3] +; AVX-NEXT: vshufps {{.*#+}} xmm1 = xmm15[0,1],xmm1[2,0] ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7] ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload -; AVX-NEXT: vunpckhpd {{.*#+}} ymm0 = ymm5[1],ymm7[1],ymm5[3],ymm7[3] -; AVX-NEXT: vunpckhps {{.*#+}} ymm1 = ymm10[2],ymm6[2],ymm10[3],ymm6[3],ymm10[6],ymm6[6],ymm10[7],ymm6[7] +; AVX-NEXT: vunpckhpd {{.*#+}} ymm0 = ymm4[1],ymm5[1],ymm4[3],ymm5[3] +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload +; AVX-NEXT: vunpckhps {{.*#+}} ymm1 = ymm4[2],ymm13[2],ymm4[3],ymm13[3],ymm4[6],ymm13[6],ymm4[7],ymm13[7] ; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,0],ymm1[4,5],ymm0[6,4] -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload -; AVX-NEXT: vunpckhps {{.*#+}} ymm1 = ymm6[2],ymm8[2],ymm6[3],ymm8[3],ymm6[6],ymm8[6],ymm6[7],ymm8[7] +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload +; AVX-NEXT: vunpckhpd {{.*#+}} ymm1 = ymm6[1],ymm2[1],ymm6[3],ymm2[3] ; AVX-NEXT: vextractf128 $1, %ymm1, %xmm1 -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload -; AVX-NEXT: vunpckhpd {{.*#+}} ymm15 = ymm4[1],ymm3[1],ymm4[3],ymm3[3] -; AVX-NEXT: vshufps {{.*#+}} ymm15 = ymm15[0,1,2,0,4,5,6,4] +; AVX-NEXT: vunpckhps {{.*#+}} ymm15 = ymm7[2],ymm3[2],ymm7[3],ymm3[3],ymm7[6],ymm3[6],ymm7[7],ymm3[7] ; AVX-NEXT: vextractf128 $1, %ymm15, %xmm15 -; AVX-NEXT: vblendps {{.*#+}} xmm1 = xmm1[0,1],xmm15[2,3] +; AVX-NEXT: vshufps {{.*#+}} xmm1 = xmm15[0,1],xmm1[2,0] ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7] ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm11 # 32-byte Reload +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload -; AVX-NEXT: vunpckhpd {{.*#+}} ymm0 = ymm0[1],ymm11[1],ymm0[3],ymm11[3] -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm13 # 32-byte Reload +; AVX-NEXT: vunpckhpd {{.*#+}} ymm0 = ymm0[1],ymm8[1],ymm0[3],ymm8[3] +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm12 # 32-byte Reload ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Reload -; AVX-NEXT: vunpckhps {{.*#+}} ymm1 = ymm13[2],ymm10[2],ymm13[3],ymm10[3],ymm13[6],ymm10[6],ymm13[7],ymm10[7] +; AVX-NEXT: vunpckhps {{.*#+}} ymm1 = ymm12[2],ymm10[2],ymm12[3],ymm10[3],ymm12[6],ymm10[6],ymm12[7],ymm10[7] ; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,0],ymm1[4,5],ymm0[6,4] -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Reload -; AVX-NEXT: vunpckhpd {{[-0-9]+}}(%r{{[sb]}}p), %ymm9, %ymm1 # 32-byte Folded Reload -; AVX-NEXT: # ymm1 = ymm9[1],mem[1],ymm9[3],mem[3] -; AVX-NEXT: vshufps {{.*#+}} ymm1 = ymm1[0,1,2,0,4,5,6,4] +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm11 # 32-byte Reload +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload +; AVX-NEXT: vunpckhpd {{.*#+}} ymm1 = ymm1[1],ymm11[1],ymm1[3],ymm11[3] ; AVX-NEXT: vextractf128 $1, %ymm1, %xmm1 -; AVX-NEXT: vunpckhps {{.*#+}} ymm15 = ymm2[2],ymm14[2],ymm2[3],ymm14[3],ymm2[6],ymm14[6],ymm2[7],ymm14[7] +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm13 # 32-byte Reload +; AVX-NEXT: vunpckhps {{.*#+}} ymm15 = ymm13[2],ymm14[2],ymm13[3],ymm14[3],ymm13[6],ymm14[6],ymm13[7],ymm14[7] ; AVX-NEXT: vextractf128 $1, %ymm15, %xmm15 -; AVX-NEXT: vblendps {{.*#+}} xmm1 = xmm15[0,1],xmm1[2,3] +; AVX-NEXT: vshufps {{.*#+}} xmm1 = xmm15[0,1],xmm1[2,0] ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7] ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload @@ -5424,57 +5396,52 @@ define void @load_i32_stride8_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt ; AVX-NEXT: # ymm1 = ymm1[3,0],mem[3,0],ymm1[7,4],mem[7,4] ; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm1[2,0],ymm0[2,3],ymm1[6,4],ymm0[6,7] ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload -; AVX-NEXT: vunpckhps {{.*#+}} ymm1 = ymm1[2],ymm12[2],ymm1[3],ymm12[3],ymm1[6],ymm12[6],ymm1[7],ymm12[7] -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload -; AVX-NEXT: vshufps $51, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm15 # 32-byte Folded Reload -; AVX-NEXT: # ymm15 = ymm2[3,0],mem[3,0],ymm2[7,4],mem[7,4] +; AVX-NEXT: vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm1 # 32-byte Folded Reload +; AVX-NEXT: # ymm1 = ymm1[2],mem[2],ymm1[3],mem[3],ymm1[6],mem[6],ymm1[7],mem[7] +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm15 # 32-byte Reload +; AVX-NEXT: vshufps {{.*#+}} ymm15 = ymm15[3,0],ymm9[3,0],ymm15[7,4],ymm9[7,4] ; AVX-NEXT: vextractf128 $1, %ymm1, %xmm1 -; AVX-NEXT: vshufps {{.*#+}} ymm15 = ymm15[2,0,2,3,6,4,6,7] ; AVX-NEXT: vextractf128 $1, %ymm15, %xmm15 -; AVX-NEXT: vblendps {{.*#+}} xmm1 = xmm15[0,1],xmm1[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm12 = ymm1[0,1,2,3],ymm0[4,5,6,7] -; AVX-NEXT: vunpckhps {{.*#+}} ymm0 = ymm7[2],ymm5[2],ymm7[3],ymm5[3],ymm7[6],ymm5[6],ymm7[7],ymm5[7] +; AVX-NEXT: vshufps {{.*#+}} xmm1 = xmm15[2,0],xmm1[2,3] +; AVX-NEXT: vblendps {{.*#+}} ymm9 = ymm1[0,1,2,3],ymm0[4,5,6,7] +; AVX-NEXT: vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %ymm5, %ymm0 # 32-byte Folded Reload +; AVX-NEXT: # ymm0 = ymm5[2],mem[2],ymm5[3],mem[3],ymm5[6],mem[6],ymm5[7],mem[7] ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload -; AVX-NEXT: vshufps $51, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm1 # 32-byte Folded Reload -; AVX-NEXT: # ymm1 = ymm1[3,0],mem[3,0],ymm1[7,4],mem[7,4] +; AVX-NEXT: vshufps {{.*#+}} ymm1 = ymm1[3,0],ymm4[3,0],ymm1[7,4],ymm4[7,4] ; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm1[2,0],ymm0[2,3],ymm1[6,4],ymm0[6,7] -; AVX-NEXT: vunpckhps {{.*#+}} ymm1 = ymm3[2],ymm4[2],ymm3[3],ymm4[3],ymm3[6],ymm4[6],ymm3[7],ymm4[7] -; AVX-NEXT: vshufps {{.*#+}} ymm2 = ymm8[3,0],ymm6[3,0],ymm8[7,4],ymm6[7,4] +; AVX-NEXT: vunpckhps {{.*#+}} ymm1 = ymm2[2],ymm6[2],ymm2[3],ymm6[3],ymm2[6],ymm6[6],ymm2[7],ymm6[7] +; AVX-NEXT: vshufps {{.*#+}} ymm2 = ymm3[3,0],ymm7[3,0],ymm3[7,4],ymm7[7,4] ; AVX-NEXT: vextractf128 $1, %ymm1, %xmm1 -; AVX-NEXT: vshufps {{.*#+}} ymm2 = ymm2[2,0,2,3,6,4,6,7] ; AVX-NEXT: vextractf128 $1, %ymm2, %xmm2 -; AVX-NEXT: vblendps {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3] +; AVX-NEXT: vshufps {{.*#+}} xmm1 = xmm2[2,0],xmm1[2,3] ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7] ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload -; AVX-NEXT: vunpckhps (%rsp), %ymm1, %ymm1 # 32-byte Folded Reload +; AVX-NEXT: vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm1 # 32-byte Folded Reload ; AVX-NEXT: # ymm1 = ymm1[2],mem[2],ymm1[3],mem[3],ymm1[6],mem[6],ymm1[7],mem[7] ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload -; AVX-NEXT: vshufps $51, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm2 # 32-byte Folded Reload +; AVX-NEXT: vshufps $51, (%rsp), %ymm2, %ymm2 # 32-byte Folded Reload ; AVX-NEXT: # ymm2 = ymm2[3,0],mem[3,0],ymm2[7,4],mem[7,4] ; AVX-NEXT: vshufps {{.*#+}} ymm1 = ymm2[2,0],ymm1[2,3],ymm2[6,4],ymm1[6,7] ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload ; AVX-NEXT: vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm2 # 32-byte Folded Reload ; AVX-NEXT: # ymm2 = ymm2[2],mem[2],ymm2[3],mem[3],ymm2[6],mem[6],ymm2[7],mem[7] ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload -; AVX-NEXT: vshufps $51, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3, %ymm4 # 32-byte Folded Reload -; AVX-NEXT: # ymm4 = ymm3[3,0],mem[3,0],ymm3[7,4],mem[7,4] +; AVX-NEXT: vshufps $51, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3, %ymm3 # 32-byte Folded Reload +; AVX-NEXT: # ymm3 = ymm3[3,0],mem[3,0],ymm3[7,4],mem[7,4] ; AVX-NEXT: vextractf128 $1, %ymm2, %xmm2 -; AVX-NEXT: vshufps {{.*#+}} ymm4 = ymm4[2,0,2,3,6,4,6,7] -; AVX-NEXT: vextractf128 $1, %ymm4, %xmm4 -; AVX-NEXT: vblendps {{.*#+}} xmm2 = xmm4[0,1],xmm2[2,3] +; AVX-NEXT: vextractf128 $1, %ymm3, %xmm3 +; AVX-NEXT: vshufps {{.*#+}} xmm2 = xmm3[2,0],xmm2[2,3] ; AVX-NEXT: vblendps {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7] -; AVX-NEXT: vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %ymm11, %ymm2 # 32-byte Folded Reload -; AVX-NEXT: # ymm2 = ymm11[2],mem[2],ymm11[3],mem[3],ymm11[6],mem[6],ymm11[7],mem[7] -; AVX-NEXT: vshufps {{.*#+}} ymm4 = ymm10[3,0],ymm13[3,0],ymm10[7,4],ymm13[7,4] -; AVX-NEXT: vshufps {{.*#+}} ymm2 = ymm4[2,0],ymm2[2,3],ymm4[6,4],ymm2[6,7] -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload -; AVX-NEXT: vunpckhps {{.*#+}} ymm4 = ymm3[2],ymm9[2],ymm3[3],ymm9[3],ymm3[6],ymm9[6],ymm3[7],ymm9[7] -; AVX-NEXT: vshufps $51, {{[-0-9]+}}(%r{{[sb]}}p), %ymm14, %ymm3 # 32-byte Folded Reload -; AVX-NEXT: # ymm3 = ymm14[3,0],mem[3,0],ymm14[7,4],mem[7,4] -; AVX-NEXT: vextractf128 $1, %ymm4, %xmm4 -; AVX-NEXT: vshufps {{.*#+}} ymm3 = ymm3[2,0,2,3,6,4,6,7] +; AVX-NEXT: vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %ymm8, %ymm2 # 32-byte Folded Reload +; AVX-NEXT: # ymm2 = ymm8[2],mem[2],ymm8[3],mem[3],ymm8[6],mem[6],ymm8[7],mem[7] +; AVX-NEXT: vshufps {{.*#+}} ymm3 = ymm10[3,0],ymm12[3,0],ymm10[7,4],ymm12[7,4] +; AVX-NEXT: vshufps {{.*#+}} ymm2 = ymm3[2,0],ymm2[2,3],ymm3[6,4],ymm2[6,7] +; AVX-NEXT: vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %ymm11, %ymm3 # 32-byte Folded Reload +; AVX-NEXT: # ymm3 = ymm11[2],mem[2],ymm11[3],mem[3],ymm11[6],mem[6],ymm11[7],mem[7] +; AVX-NEXT: vshufps {{.*#+}} ymm4 = ymm14[3,0],ymm13[3,0],ymm14[7,4],ymm13[7,4] ; AVX-NEXT: vextractf128 $1, %ymm3, %xmm3 -; AVX-NEXT: vblendps {{.*#+}} xmm3 = xmm3[0,1],xmm4[2,3] +; AVX-NEXT: vextractf128 $1, %ymm4, %xmm4 +; AVX-NEXT: vshufps {{.*#+}} xmm3 = xmm4[2,0],xmm3[2,3] ; AVX-NEXT: vblendps {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7] ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload ; AVX-NEXT: vmovaps %ymm3, 64(%rsi) @@ -5537,9 +5504,9 @@ define void @load_i32_stride8_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt ; AVX-NEXT: movq {{[0-9]+}}(%rsp), %rax ; AVX-NEXT: vmovaps %ymm1, 96(%rax) ; AVX-NEXT: vmovaps %ymm0, 64(%rax) -; AVX-NEXT: vmovaps %ymm12, 32(%rax) +; AVX-NEXT: vmovaps %ymm9, 32(%rax) ; AVX-NEXT: vmovaps %ymm2, (%rax) -; AVX-NEXT: addq $1800, %rsp # imm = 0x708 +; AVX-NEXT: addq $1768, %rsp # imm = 0x6E8 ; AVX-NEXT: vzeroupper ; AVX-NEXT: retq ; @@ -10431,7 +10398,7 @@ define void @load_i32_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt ; AVX-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm1 ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7] ; AVX-NEXT: vmovaps 32(%rdi), %xmm1 -; AVX-NEXT: vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; AVX-NEXT: vmovaps %xmm1, (%rsp) # 16-byte Spill ; AVX-NEXT: vmovaps (%rdi), %xmm4 ; AVX-NEXT: vunpcklps {{.*#+}} xmm1 = xmm4[0],xmm1[0],xmm4[1],xmm1[1] ; AVX-NEXT: vmovaps %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill @@ -10474,7 +10441,7 @@ define void @load_i32_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt ; AVX-NEXT: vmovaps 1184(%rdi), %xmm0 ; AVX-NEXT: vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; AVX-NEXT: vmovaps 1152(%rdi), %xmm6 -; AVX-NEXT: vmovaps %xmm6, (%rsp) # 16-byte Spill +; AVX-NEXT: vmovaps %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; AVX-NEXT: vunpcklps {{.*#+}} xmm6 = xmm6[0],xmm0[0],xmm6[1],xmm0[1] ; AVX-NEXT: vinsertf128 $1, %xmm6, %ymm0, %ymm7 ; AVX-NEXT: vmovaps 1248(%rdi), %xmm0 @@ -10584,7 +10551,7 @@ define void @load_i32_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm5[4,5,6,7] ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX-NEXT: vshufps {{.*#+}} xmm0 = xmm4[1,1,1,1] -; AVX-NEXT: vblendps $2, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload +; AVX-NEXT: vblendps $2, (%rsp), %xmm0, %xmm0 # 16-byte Folded Reload ; AVX-NEXT: # xmm0 = xmm0[0],mem[1],xmm0[2,3] ; AVX-NEXT: vblendps {{.*#+}} xmm0 = xmm0[0,1],xmm3[2,3] ; AVX-NEXT: vinsertf128 $1, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm3 # 16-byte Folded Reload @@ -10616,7 +10583,7 @@ define void @load_i32_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt ; AVX-NEXT: vblendps $12, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload ; AVX-NEXT: # xmm0 = xmm0[0,1],mem[2,3] ; AVX-NEXT: vinsertf128 $1, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm1 # 16-byte Folded Reload -; AVX-NEXT: vpermilps $85, (%rsp), %xmm2 # 16-byte Folded Reload +; AVX-NEXT: vpermilps $85, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload ; AVX-NEXT: # xmm2 = mem[1,1,1,1] ; AVX-NEXT: vblendps $2, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2, %xmm2 # 16-byte Folded Reload ; AVX-NEXT: # xmm2 = xmm2[0],mem[1],xmm2[2,3] @@ -10719,9 +10686,9 @@ define void @load_i32_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7] ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload -; AVX-NEXT: vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm8 # 16-byte Folded Reload +; AVX-NEXT: vunpckhps (%rsp), %xmm0, %xmm8 # 16-byte Folded Reload ; AVX-NEXT: # xmm8 = xmm0[2],mem[2],xmm0[3],mem[3] -; AVX-NEXT: vmovaps %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; AVX-NEXT: vmovaps %xmm8, (%rsp) # 16-byte Spill ; AVX-NEXT: vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %xmm10, %xmm1 # 16-byte Folded Reload ; AVX-NEXT: # xmm1 = xmm10[2],mem[2],xmm10[3],mem[3] ; AVX-NEXT: vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill @@ -10765,7 +10732,7 @@ define void @load_i32_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt ; AVX-NEXT: vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %xmm8, %xmm0 # 16-byte Folded Reload ; AVX-NEXT: # xmm0 = xmm8[2],mem[2],xmm8[3],mem[3] ; AVX-NEXT: vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; AVX-NEXT: vmovaps (%rsp), %xmm8 # 16-byte Reload +; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload ; AVX-NEXT: vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %xmm8, %xmm8 # 16-byte Folded Reload ; AVX-NEXT: # xmm8 = xmm8[2],mem[2],xmm8[3],mem[3] ; AVX-NEXT: vmovaps %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill @@ -10786,7 +10753,7 @@ define void @load_i32_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt ; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload ; AVX-NEXT: vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %xmm8, %xmm0 # 16-byte Folded Reload ; AVX-NEXT: # xmm0 = xmm8[2],mem[2],xmm8[3],mem[3] -; AVX-NEXT: vmovaps %xmm0, (%rsp) # 16-byte Spill +; AVX-NEXT: vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; AVX-NEXT: vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %xmm9, %xmm8 # 16-byte Folded Reload ; AVX-NEXT: # xmm8 = xmm9[2],mem[2],xmm9[3],mem[3] ; AVX-NEXT: vmovaps %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill @@ -10861,7 +10828,7 @@ define void @load_i32_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt ; AVX-NEXT: vmovups %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX-NEXT: vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %xmm3, %xmm4 # 16-byte Folded Reload ; AVX-NEXT: # xmm4 = xmm3[2],mem[2],xmm3[3],mem[3] -; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload +; AVX-NEXT: vmovaps (%rsp), %xmm2 # 16-byte Reload ; AVX-NEXT: vunpckhpd {{.*#+}} xmm2 = xmm2[1],xmm4[1] ; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload ; AVX-NEXT: vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %xmm3, %xmm4 # 16-byte Folded Reload @@ -10901,7 +10868,7 @@ define void @load_i32_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7] ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX-NEXT: vunpckhps {{.*#+}} xmm0 = xmm10[2],xmm12[2],xmm10[3],xmm12[3] -; AVX-NEXT: vmovaps (%rsp), %xmm1 # 16-byte Reload +; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload ; AVX-NEXT: vunpckhpd {{.*#+}} xmm0 = xmm1[1],xmm0[1] ; AVX-NEXT: vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %xmm13, %xmm1 # 16-byte Folded Reload ; AVX-NEXT: # xmm1 = xmm13[2],mem[2],xmm13[3],mem[3] @@ -10914,50 +10881,46 @@ define void @load_i32_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX-NEXT: vmovaps 416(%rdi), %ymm2 ; AVX-NEXT: vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vmovaps 384(%rdi), %ymm15 +; AVX-NEXT: vmovaps 384(%rdi), %ymm3 +; AVX-NEXT: vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX-NEXT: vmovaps 448(%rdi), %ymm4 ; AVX-NEXT: vmovaps 480(%rdi), %ymm0 ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX-NEXT: vunpcklpd {{.*#+}} ymm0 = ymm0[0],ymm4[0],ymm0[2],ymm4[2] -; AVX-NEXT: vunpcklps {{.*#+}} ymm1 = ymm15[0],ymm2[0],ymm15[1],ymm2[1],ymm15[4],ymm2[4],ymm15[5],ymm2[5] +; AVX-NEXT: vunpcklps {{.*#+}} ymm1 = ymm3[0],ymm2[0],ymm3[1],ymm2[1],ymm3[4],ymm2[4],ymm3[5],ymm2[5] ; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,0],ymm1[4,5],ymm0[6,4] -; AVX-NEXT: vmovaps 288(%rdi), %ymm6 -; AVX-NEXT: vmovaps 256(%rdi), %ymm1 -; AVX-NEXT: vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vunpcklps {{.*#+}} ymm1 = ymm1[0],ymm6[0],ymm1[1],ymm6[1],ymm1[4],ymm6[4],ymm1[5],ymm6[5] -; AVX-NEXT: vmovups %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vmovaps 320(%rdi), %ymm6 +; AVX-NEXT: vmovaps 352(%rdi), %ymm13 +; AVX-NEXT: vunpcklpd {{.*#+}} ymm1 = ymm13[0],ymm6[0],ymm13[2],ymm6[2] +; AVX-NEXT: vmovups %ymm13, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX-NEXT: vextractf128 $1, %ymm1, %xmm1 -; AVX-NEXT: vmovaps 320(%rdi), %ymm2 -; AVX-NEXT: vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vmovaps 352(%rdi), %ymm9 -; AVX-NEXT: vunpcklpd {{.*#+}} ymm2 = ymm9[0],ymm2[0],ymm9[2],ymm2[2] -; AVX-NEXT: vshufps {{.*#+}} ymm2 = ymm2[0,1,2,0,4,5,6,4] +; AVX-NEXT: vmovaps 288(%rdi), %ymm8 +; AVX-NEXT: vmovaps 256(%rdi), %ymm7 +; AVX-NEXT: vunpcklps {{.*#+}} ymm2 = ymm7[0],ymm8[0],ymm7[1],ymm8[1],ymm7[4],ymm8[4],ymm7[5],ymm8[5] ; AVX-NEXT: vextractf128 $1, %ymm2, %xmm2 -; AVX-NEXT: vblendps {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3] +; AVX-NEXT: vshufps {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,0] ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7] ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vmovaps 672(%rdi), %ymm12 -; AVX-NEXT: vmovaps 640(%rdi), %ymm8 -; AVX-NEXT: vmovaps 704(%rdi), %ymm13 -; AVX-NEXT: vmovaps 736(%rdi), %ymm0 -; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vunpcklpd {{.*#+}} ymm0 = ymm0[0],ymm13[0],ymm0[2],ymm13[2] -; AVX-NEXT: vmovups %ymm13, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vunpcklps {{.*#+}} ymm1 = ymm8[0],ymm12[0],ymm8[1],ymm12[1],ymm8[4],ymm12[4],ymm8[5],ymm12[5] -; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,0],ymm1[4,5],ymm0[6,4] -; AVX-NEXT: vmovaps 544(%rdi), %ymm1 -; AVX-NEXT: vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vmovaps 512(%rdi), %ymm2 +; AVX-NEXT: vmovaps 672(%rdi), %ymm2 ; AVX-NEXT: vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vunpcklps {{.*#+}} ymm1 = ymm2[0],ymm1[0],ymm2[1],ymm1[1],ymm2[4],ymm1[4],ymm2[5],ymm1[5] +; AVX-NEXT: vmovaps 640(%rdi), %ymm9 +; AVX-NEXT: vmovaps 704(%rdi), %ymm12 +; AVX-NEXT: vmovaps 736(%rdi), %ymm10 +; AVX-NEXT: vunpcklpd {{.*#+}} ymm0 = ymm10[0],ymm12[0],ymm10[2],ymm12[2] +; AVX-NEXT: vunpcklps {{.*#+}} ymm1 = ymm9[0],ymm2[0],ymm9[1],ymm2[1],ymm9[4],ymm2[4],ymm9[5],ymm2[5] +; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,0],ymm1[4,5],ymm0[6,4] +; AVX-NEXT: vmovaps 576(%rdi), %ymm14 +; AVX-NEXT: vmovaps 608(%rdi), %ymm11 +; AVX-NEXT: vunpcklpd {{.*#+}} ymm1 = ymm11[0],ymm14[0],ymm11[2],ymm14[2] +; AVX-NEXT: vmovups %ymm14, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX-NEXT: vextractf128 $1, %ymm1, %xmm1 -; AVX-NEXT: vmovaps 576(%rdi), %ymm11 -; AVX-NEXT: vmovaps 608(%rdi), %ymm7 -; AVX-NEXT: vunpcklpd {{.*#+}} ymm2 = ymm7[0],ymm11[0],ymm7[2],ymm11[2] -; AVX-NEXT: vmovups %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vshufps {{.*#+}} ymm2 = ymm2[0,1,2,0,4,5,6,4] +; AVX-NEXT: vmovaps 544(%rdi), %ymm2 +; AVX-NEXT: vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vmovaps 512(%rdi), %ymm3 +; AVX-NEXT: vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vunpcklps {{.*#+}} ymm2 = ymm3[0],ymm2[0],ymm3[1],ymm2[1],ymm3[4],ymm2[4],ymm3[5],ymm2[5] ; AVX-NEXT: vextractf128 $1, %ymm2, %xmm2 -; AVX-NEXT: vblendps {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3] +; AVX-NEXT: vshufps {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,0] ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7] ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX-NEXT: vmovaps 928(%rdi), %ymm2 @@ -10971,20 +10934,19 @@ define void @load_i32_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt ; AVX-NEXT: vunpcklpd {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[2],ymm1[2] ; AVX-NEXT: vunpcklps {{.*#+}} ymm1 = ymm3[0],ymm2[0],ymm3[1],ymm2[1],ymm3[4],ymm2[4],ymm3[5],ymm2[5] ; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,0],ymm1[4,5],ymm0[6,4] -; AVX-NEXT: vmovaps 800(%rdi), %ymm2 +; AVX-NEXT: vmovaps 832(%rdi), %ymm2 ; AVX-NEXT: vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vmovaps 768(%rdi), %ymm1 +; AVX-NEXT: vmovaps 864(%rdi), %ymm1 ; AVX-NEXT: vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vunpcklps {{.*#+}} ymm1 = ymm1[0],ymm2[0],ymm1[1],ymm2[1],ymm1[4],ymm2[4],ymm1[5],ymm2[5] +; AVX-NEXT: vunpcklpd {{.*#+}} ymm1 = ymm1[0],ymm2[0],ymm1[2],ymm2[2] ; AVX-NEXT: vextractf128 $1, %ymm1, %xmm1 -; AVX-NEXT: vmovaps 832(%rdi), %ymm3 +; AVX-NEXT: vmovaps 800(%rdi), %ymm3 ; AVX-NEXT: vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vmovaps 864(%rdi), %ymm2 +; AVX-NEXT: vmovaps 768(%rdi), %ymm2 ; AVX-NEXT: vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vunpcklpd {{.*#+}} ymm2 = ymm2[0],ymm3[0],ymm2[2],ymm3[2] -; AVX-NEXT: vshufps {{.*#+}} ymm2 = ymm2[0,1,2,0,4,5,6,4] +; AVX-NEXT: vunpcklps {{.*#+}} ymm2 = ymm2[0],ymm3[0],ymm2[1],ymm3[1],ymm2[4],ymm3[4],ymm2[5],ymm3[5] ; AVX-NEXT: vextractf128 $1, %ymm2, %xmm2 -; AVX-NEXT: vblendps {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3] +; AVX-NEXT: vshufps {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,0] ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7] ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX-NEXT: vmovaps 1184(%rdi), %ymm2 @@ -10998,20 +10960,19 @@ define void @load_i32_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt ; AVX-NEXT: vunpcklpd {{.*#+}} ymm0 = ymm3[0],ymm0[0],ymm3[2],ymm0[2] ; AVX-NEXT: vunpcklps {{.*#+}} ymm1 = ymm1[0],ymm2[0],ymm1[1],ymm2[1],ymm1[4],ymm2[4],ymm1[5],ymm2[5] ; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,0],ymm1[4,5],ymm0[6,4] -; AVX-NEXT: vmovaps 1056(%rdi), %ymm1 -; AVX-NEXT: vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vmovaps 1024(%rdi), %ymm2 +; AVX-NEXT: vmovaps 1088(%rdi), %ymm2 ; AVX-NEXT: vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vunpcklps {{.*#+}} ymm1 = ymm2[0],ymm1[0],ymm2[1],ymm1[1],ymm2[4],ymm1[4],ymm2[5],ymm1[5] +; AVX-NEXT: vmovaps 1120(%rdi), %ymm1 +; AVX-NEXT: vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vunpcklpd {{.*#+}} ymm1 = ymm1[0],ymm2[0],ymm1[2],ymm2[2] ; AVX-NEXT: vextractf128 $1, %ymm1, %xmm1 -; AVX-NEXT: vmovaps 1088(%rdi), %ymm3 +; AVX-NEXT: vmovaps 1056(%rdi), %ymm3 ; AVX-NEXT: vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vmovaps 1120(%rdi), %ymm2 +; AVX-NEXT: vmovaps 1024(%rdi), %ymm2 ; AVX-NEXT: vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vunpcklpd {{.*#+}} ymm2 = ymm2[0],ymm3[0],ymm2[2],ymm3[2] -; AVX-NEXT: vshufps {{.*#+}} ymm2 = ymm2[0,1,2,0,4,5,6,4] +; AVX-NEXT: vunpcklps {{.*#+}} ymm2 = ymm2[0],ymm3[0],ymm2[1],ymm3[1],ymm2[4],ymm3[4],ymm2[5],ymm3[5] ; AVX-NEXT: vextractf128 $1, %ymm2, %xmm2 -; AVX-NEXT: vblendps {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3] +; AVX-NEXT: vshufps {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,0] ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7] ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX-NEXT: vmovaps 1440(%rdi), %ymm2 @@ -11025,20 +10986,19 @@ define void @load_i32_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt ; AVX-NEXT: vunpcklpd {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[2],ymm1[2] ; AVX-NEXT: vunpcklps {{.*#+}} ymm1 = ymm3[0],ymm2[0],ymm3[1],ymm2[1],ymm3[4],ymm2[4],ymm3[5],ymm2[5] ; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,0],ymm1[4,5],ymm0[6,4] -; AVX-NEXT: vmovaps 1312(%rdi), %ymm2 +; AVX-NEXT: vmovaps 1344(%rdi), %ymm2 ; AVX-NEXT: vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vmovaps 1280(%rdi), %ymm1 +; AVX-NEXT: vmovaps 1376(%rdi), %ymm1 ; AVX-NEXT: vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vunpcklps {{.*#+}} ymm1 = ymm1[0],ymm2[0],ymm1[1],ymm2[1],ymm1[4],ymm2[4],ymm1[5],ymm2[5] +; AVX-NEXT: vunpcklpd {{.*#+}} ymm1 = ymm1[0],ymm2[0],ymm1[2],ymm2[2] ; AVX-NEXT: vextractf128 $1, %ymm1, %xmm1 -; AVX-NEXT: vmovaps 1344(%rdi), %ymm3 +; AVX-NEXT: vmovaps 1312(%rdi), %ymm3 ; AVX-NEXT: vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vmovaps 1376(%rdi), %ymm2 +; AVX-NEXT: vmovaps 1280(%rdi), %ymm2 ; AVX-NEXT: vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vunpcklpd {{.*#+}} ymm2 = ymm2[0],ymm3[0],ymm2[2],ymm3[2] -; AVX-NEXT: vshufps {{.*#+}} ymm2 = ymm2[0,1,2,0,4,5,6,4] +; AVX-NEXT: vunpcklps {{.*#+}} ymm2 = ymm2[0],ymm3[0],ymm2[1],ymm3[1],ymm2[4],ymm3[4],ymm2[5],ymm3[5] ; AVX-NEXT: vextractf128 $1, %ymm2, %xmm2 -; AVX-NEXT: vblendps {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3] +; AVX-NEXT: vshufps {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,0] ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7] ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX-NEXT: vmovaps 1696(%rdi), %ymm2 @@ -11052,20 +11012,19 @@ define void @load_i32_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt ; AVX-NEXT: vunpcklpd {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[2],ymm1[2] ; AVX-NEXT: vunpcklps {{.*#+}} ymm1 = ymm3[0],ymm2[0],ymm3[1],ymm2[1],ymm3[4],ymm2[4],ymm3[5],ymm2[5] ; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,0],ymm1[4,5],ymm0[6,4] -; AVX-NEXT: vmovaps 1568(%rdi), %ymm2 +; AVX-NEXT: vmovaps 1600(%rdi), %ymm2 ; AVX-NEXT: vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vmovaps 1536(%rdi), %ymm1 +; AVX-NEXT: vmovaps 1632(%rdi), %ymm1 ; AVX-NEXT: vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vunpcklps {{.*#+}} ymm1 = ymm1[0],ymm2[0],ymm1[1],ymm2[1],ymm1[4],ymm2[4],ymm1[5],ymm2[5] +; AVX-NEXT: vunpcklpd {{.*#+}} ymm1 = ymm1[0],ymm2[0],ymm1[2],ymm2[2] ; AVX-NEXT: vextractf128 $1, %ymm1, %xmm1 -; AVX-NEXT: vmovaps 1600(%rdi), %ymm3 +; AVX-NEXT: vmovaps 1568(%rdi), %ymm3 ; AVX-NEXT: vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vmovaps 1632(%rdi), %ymm2 +; AVX-NEXT: vmovaps 1536(%rdi), %ymm2 ; AVX-NEXT: vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vunpcklpd {{.*#+}} ymm2 = ymm2[0],ymm3[0],ymm2[2],ymm3[2] -; AVX-NEXT: vshufps {{.*#+}} ymm2 = ymm2[0,1,2,0,4,5,6,4] +; AVX-NEXT: vunpcklps {{.*#+}} ymm2 = ymm2[0],ymm3[0],ymm2[1],ymm3[1],ymm2[4],ymm3[4],ymm2[5],ymm3[5] ; AVX-NEXT: vextractf128 $1, %ymm2, %xmm2 -; AVX-NEXT: vblendps {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3] +; AVX-NEXT: vshufps {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,0] ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7] ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX-NEXT: vmovaps 1952(%rdi), %ymm2 @@ -11079,20 +11038,19 @@ define void @load_i32_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt ; AVX-NEXT: vunpcklpd {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[2],ymm1[2] ; AVX-NEXT: vunpcklps {{.*#+}} ymm1 = ymm3[0],ymm2[0],ymm3[1],ymm2[1],ymm3[4],ymm2[4],ymm3[5],ymm2[5] ; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,0],ymm1[4,5],ymm0[6,4] -; AVX-NEXT: vmovaps 1824(%rdi), %ymm2 +; AVX-NEXT: vmovaps 1856(%rdi), %ymm2 ; AVX-NEXT: vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vmovaps 1792(%rdi), %ymm1 +; AVX-NEXT: vmovaps 1888(%rdi), %ymm1 ; AVX-NEXT: vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vunpcklps {{.*#+}} ymm1 = ymm1[0],ymm2[0],ymm1[1],ymm2[1],ymm1[4],ymm2[4],ymm1[5],ymm2[5] +; AVX-NEXT: vunpcklpd {{.*#+}} ymm1 = ymm1[0],ymm2[0],ymm1[2],ymm2[2] ; AVX-NEXT: vextractf128 $1, %ymm1, %xmm1 -; AVX-NEXT: vmovaps 1856(%rdi), %ymm3 +; AVX-NEXT: vmovaps 1824(%rdi), %ymm3 ; AVX-NEXT: vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vmovaps 1888(%rdi), %ymm2 -; AVX-NEXT: vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vunpcklpd {{.*#+}} ymm2 = ymm2[0],ymm3[0],ymm2[2],ymm3[2] -; AVX-NEXT: vshufps {{.*#+}} ymm2 = ymm2[0,1,2,0,4,5,6,4] +; AVX-NEXT: vmovaps 1792(%rdi), %ymm2 +; AVX-NEXT: vmovups %ymm2, (%rsp) # 32-byte Spill +; AVX-NEXT: vunpcklps {{.*#+}} ymm2 = ymm2[0],ymm3[0],ymm2[1],ymm3[1],ymm2[4],ymm3[4],ymm2[5],ymm3[5] ; AVX-NEXT: vextractf128 $1, %ymm2, %xmm2 -; AVX-NEXT: vblendps {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3] +; AVX-NEXT: vshufps {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,0] ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7] ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX-NEXT: vmovaps 160(%rdi), %ymm2 @@ -11102,449 +11060,423 @@ define void @load_i32_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt ; AVX-NEXT: vmovaps 192(%rdi), %ymm1 ; AVX-NEXT: vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX-NEXT: vmovaps 224(%rdi), %ymm0 -; AVX-NEXT: vmovups %ymm0, (%rsp) # 32-byte Spill +; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX-NEXT: vunpcklpd {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[2],ymm1[2] ; AVX-NEXT: vunpcklps {{.*#+}} ymm1 = ymm3[0],ymm2[0],ymm3[1],ymm2[1],ymm3[4],ymm2[4],ymm3[5],ymm2[5] -; AVX-NEXT: vshufps {{.*#+}} ymm10 = ymm1[0,1],ymm0[2,0],ymm1[4,5],ymm0[6,4] -; AVX-NEXT: vmovaps 64(%rdi), %ymm1 -; AVX-NEXT: vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vmovaps 96(%rdi), %ymm0 +; AVX-NEXT: vshufps {{.*#+}} ymm2 = ymm1[0,1],ymm0[2,0],ymm1[4,5],ymm0[6,4] +; AVX-NEXT: vmovaps 64(%rdi), %ymm0 ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vunpcklpd {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[2],ymm1[2] -; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm0[0,1,2,0,4,5,6,4] -; AVX-NEXT: vextractf128 $1, %ymm0, %xmm0 +; AVX-NEXT: vmovaps 96(%rdi), %ymm1 +; AVX-NEXT: vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vunpcklpd {{.*#+}} ymm1 = ymm1[0],ymm0[0],ymm1[2],ymm0[2] +; AVX-NEXT: vextractf128 $1, %ymm1, %xmm0 ; AVX-NEXT: vmovaps (%rdi), %ymm1 ; AVX-NEXT: vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vmovaps 32(%rdi), %ymm2 -; AVX-NEXT: vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vunpcklps {{.*#+}} ymm14 = ymm1[0],ymm2[0],ymm1[1],ymm2[1],ymm1[4],ymm2[4],ymm1[5],ymm2[5] -; AVX-NEXT: vextractf128 $1, %ymm14, %xmm14 -; AVX-NEXT: vblendps {{.*#+}} xmm0 = xmm14[0,1],xmm0[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm10[4,5,6,7] +; AVX-NEXT: vmovaps 32(%rdi), %ymm3 +; AVX-NEXT: vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vunpcklps {{.*#+}} ymm15 = ymm1[0],ymm3[0],ymm1[1],ymm3[1],ymm1[4],ymm3[4],ymm1[5],ymm3[5] +; AVX-NEXT: vextractf128 $1, %ymm15, %xmm15 +; AVX-NEXT: vshufps {{.*#+}} xmm0 = xmm15[0,1],xmm0[2,0] +; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm2[4,5,6,7] ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload ; AVX-NEXT: vmovaps %ymm4, %ymm5 ; AVX-NEXT: vmovups %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX-NEXT: vunpcklps {{.*#+}} ymm0 = ymm4[0],ymm3[0],ymm4[1],ymm3[1],ymm4[4],ymm3[4],ymm4[5],ymm3[5] -; AVX-NEXT: vmovaps %ymm15, %ymm4 -; AVX-NEXT: vmovups %ymm15, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload -; AVX-NEXT: vshufps {{.*#+}} ymm10 = ymm1[1,0],ymm15[1,0],ymm1[5,4],ymm15[5,4] -; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm10[2,0],ymm0[2,3],ymm10[6,4],ymm0[6,7] -; AVX-NEXT: vmovaps %ymm9, %ymm15 -; AVX-NEXT: vmovups %ymm9, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Reload -; AVX-NEXT: vunpcklps {{.*#+}} ymm10 = ymm9[0],ymm15[0],ymm9[1],ymm15[1],ymm9[4],ymm15[4],ymm9[5],ymm15[5] -; AVX-NEXT: vextractf128 $1, %ymm10, %xmm10 -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload -; AVX-NEXT: vshufps {{.*#+}} ymm14 = ymm6[1,0],ymm2[1,0],ymm6[5,4],ymm2[5,4] -; AVX-NEXT: vshufps {{.*#+}} ymm14 = ymm14[2,0,2,3,6,4,6,7] -; AVX-NEXT: vextractf128 $1, %ymm14, %xmm14 -; AVX-NEXT: vblendps {{.*#+}} xmm10 = xmm14[0,1],xmm10[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm10[0,1,2,3],ymm0[4,5,6,7] -; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload -; AVX-NEXT: vunpcklps {{.*#+}} ymm0 = ymm13[0],ymm6[0],ymm13[1],ymm6[1],ymm13[4],ymm6[4],ymm13[5],ymm6[5] +; AVX-NEXT: vshufps {{.*#+}} ymm2 = ymm1[1,0],ymm4[1,0],ymm1[5,4],ymm4[5,4] +; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm2[2,0],ymm0[2,3],ymm2[6,4],ymm0[6,7] +; AVX-NEXT: vmovups %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vunpcklps {{.*#+}} ymm2 = ymm6[0],ymm13[0],ymm6[1],ymm13[1],ymm6[4],ymm13[4],ymm6[5],ymm13[5] +; AVX-NEXT: vextractf128 $1, %ymm2, %xmm2 +; AVX-NEXT: vmovups %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX-NEXT: vmovups %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vshufps {{.*#+}} ymm15 = ymm8[1,0],ymm7[1,0],ymm8[5,4],ymm7[5,4] +; AVX-NEXT: vextractf128 $1, %ymm15, %xmm15 +; AVX-NEXT: vshufps {{.*#+}} xmm2 = xmm15[2,0],xmm2[2,3] +; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm0[4,5,6,7] +; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vmovups %ymm10, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vmovaps %ymm12, %ymm13 ; AVX-NEXT: vmovups %ymm12, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vshufps {{.*#+}} ymm10 = ymm12[1,0],ymm8[1,0],ymm12[5,4],ymm8[5,4] -; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm10[2,0],ymm0[2,3],ymm10[6,4],ymm0[6,7] -; AVX-NEXT: vmovaps %ymm11, %ymm13 +; AVX-NEXT: vunpcklps {{.*#+}} ymm0 = ymm12[0],ymm10[0],ymm12[1],ymm10[1],ymm12[4],ymm10[4],ymm12[5],ymm10[5] +; AVX-NEXT: vmovups %ymm9, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm12 # 32-byte Reload +; AVX-NEXT: vshufps {{.*#+}} ymm2 = ymm12[1,0],ymm9[1,0],ymm12[5,4],ymm9[5,4] +; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm2[2,0],ymm0[2,3],ymm2[6,4],ymm0[6,7] ; AVX-NEXT: vmovups %ymm11, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vunpcklps {{.*#+}} ymm10 = ymm11[0],ymm7[0],ymm11[1],ymm7[1],ymm11[4],ymm7[4],ymm11[5],ymm7[5] -; AVX-NEXT: vextractf128 $1, %ymm10, %xmm10 -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm11 # 32-byte Reload -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload -; AVX-NEXT: vshufps {{.*#+}} ymm14 = ymm7[1,0],ymm11[1,0],ymm7[5,4],ymm11[5,4] -; AVX-NEXT: vshufps {{.*#+}} ymm14 = ymm14[2,0,2,3,6,4,6,7] -; AVX-NEXT: vextractf128 $1, %ymm14, %xmm14 -; AVX-NEXT: vblendps {{.*#+}} xmm10 = xmm14[0,1],xmm10[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm10[0,1,2,3],ymm0[4,5,6,7] +; AVX-NEXT: vunpcklps {{.*#+}} ymm2 = ymm14[0],ymm11[0],ymm14[1],ymm11[1],ymm14[4],ymm11[4],ymm14[5],ymm11[5] +; AVX-NEXT: vextractf128 $1, %ymm2, %xmm2 +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm14 # 32-byte Reload +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm15 # 32-byte Reload +; AVX-NEXT: vshufps {{.*#+}} ymm15 = ymm15[1,0],ymm14[1,0],ymm15[5,4],ymm14[5,4] +; AVX-NEXT: vextractf128 $1, %ymm15, %xmm15 +; AVX-NEXT: vshufps {{.*#+}} xmm2 = xmm15[2,0],xmm2[2,3] +; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm0[4,5,6,7] ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload ; AVX-NEXT: vunpcklps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload ; AVX-NEXT: # ymm0 = ymm0[0],mem[0],ymm0[1],mem[1],ymm0[4],mem[4],ymm0[5],mem[5] -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Reload -; AVX-NEXT: vshufps {{.*#+}} ymm10 = ymm10[1,0],ymm7[1,0],ymm10[5,4],ymm7[5,4] -; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm10[2,0],ymm0[2,3],ymm10[6,4],ymm0[6,7] -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Reload -; AVX-NEXT: vunpcklps {{[-0-9]+}}(%r{{[sb]}}p), %ymm10, %ymm10 # 32-byte Folded Reload -; AVX-NEXT: # ymm10 = ymm10[0],mem[0],ymm10[1],mem[1],ymm10[4],mem[4],ymm10[5],mem[5] -; AVX-NEXT: vextractf128 $1, %ymm10, %xmm10 -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm14 # 32-byte Reload -; AVX-NEXT: vshufps $17, {{[-0-9]+}}(%r{{[sb]}}p), %ymm14, %ymm14 # 32-byte Folded Reload -; AVX-NEXT: # ymm14 = ymm14[1,0],mem[1,0],ymm14[5,4],mem[5,4] -; AVX-NEXT: vshufps {{.*#+}} ymm14 = ymm14[2,0,2,3,6,4,6,7] -; AVX-NEXT: vextractf128 $1, %ymm14, %xmm14 -; AVX-NEXT: vblendps {{.*#+}} xmm10 = xmm14[0,1],xmm10[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm10[0,1,2,3],ymm0[4,5,6,7] +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload +; AVX-NEXT: vshufps $17, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm2 # 32-byte Folded Reload +; AVX-NEXT: # ymm2 = ymm2[1,0],mem[1,0],ymm2[5,4],mem[5,4] +; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm2[2,0],ymm0[2,3],ymm2[6,4],ymm0[6,7] +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload +; AVX-NEXT: vunpcklps {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm2 # 32-byte Folded Reload +; AVX-NEXT: # ymm2 = ymm2[0],mem[0],ymm2[1],mem[1],ymm2[4],mem[4],ymm2[5],mem[5] +; AVX-NEXT: vextractf128 $1, %ymm2, %xmm2 +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm15 # 32-byte Reload +; AVX-NEXT: vshufps $17, {{[-0-9]+}}(%r{{[sb]}}p), %ymm15, %ymm15 # 32-byte Folded Reload +; AVX-NEXT: # ymm15 = ymm15[1,0],mem[1,0],ymm15[5,4],mem[5,4] +; AVX-NEXT: vextractf128 $1, %ymm15, %xmm15 +; AVX-NEXT: vshufps {{.*#+}} xmm2 = xmm15[2,0],xmm2[2,3] +; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm0[4,5,6,7] ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload ; AVX-NEXT: vunpcklps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload ; AVX-NEXT: # ymm0 = ymm0[0],mem[0],ymm0[1],mem[1],ymm0[4],mem[4],ymm0[5],mem[5] -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Reload -; AVX-NEXT: vshufps $17, {{[-0-9]+}}(%r{{[sb]}}p), %ymm10, %ymm10 # 32-byte Folded Reload -; AVX-NEXT: # ymm10 = ymm10[1,0],mem[1,0],ymm10[5,4],mem[5,4] -; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm10[2,0],ymm0[2,3],ymm10[6,4],ymm0[6,7] -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Reload -; AVX-NEXT: vunpcklps {{[-0-9]+}}(%r{{[sb]}}p), %ymm10, %ymm10 # 32-byte Folded Reload -; AVX-NEXT: # ymm10 = ymm10[0],mem[0],ymm10[1],mem[1],ymm10[4],mem[4],ymm10[5],mem[5] -; AVX-NEXT: vextractf128 $1, %ymm10, %xmm10 -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm14 # 32-byte Reload -; AVX-NEXT: vshufps $17, {{[-0-9]+}}(%r{{[sb]}}p), %ymm14, %ymm14 # 32-byte Folded Reload -; AVX-NEXT: # ymm14 = ymm14[1,0],mem[1,0],ymm14[5,4],mem[5,4] -; AVX-NEXT: vshufps {{.*#+}} ymm14 = ymm14[2,0,2,3,6,4,6,7] -; AVX-NEXT: vextractf128 $1, %ymm14, %xmm14 -; AVX-NEXT: vblendps {{.*#+}} xmm10 = xmm14[0,1],xmm10[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm10[0,1,2,3],ymm0[4,5,6,7] +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload +; AVX-NEXT: vshufps $17, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm2 # 32-byte Folded Reload +; AVX-NEXT: # ymm2 = ymm2[1,0],mem[1,0],ymm2[5,4],mem[5,4] +; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm2[2,0],ymm0[2,3],ymm2[6,4],ymm0[6,7] +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload +; AVX-NEXT: vunpcklps {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm2 # 32-byte Folded Reload +; AVX-NEXT: # ymm2 = ymm2[0],mem[0],ymm2[1],mem[1],ymm2[4],mem[4],ymm2[5],mem[5] +; AVX-NEXT: vextractf128 $1, %ymm2, %xmm2 +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm15 # 32-byte Reload +; AVX-NEXT: vshufps $17, {{[-0-9]+}}(%r{{[sb]}}p), %ymm15, %ymm15 # 32-byte Folded Reload +; AVX-NEXT: # ymm15 = ymm15[1,0],mem[1,0],ymm15[5,4],mem[5,4] +; AVX-NEXT: vextractf128 $1, %ymm15, %xmm15 +; AVX-NEXT: vshufps {{.*#+}} xmm2 = xmm15[2,0],xmm2[2,3] +; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm0[4,5,6,7] ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload ; AVX-NEXT: vunpcklps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload ; AVX-NEXT: # ymm0 = ymm0[0],mem[0],ymm0[1],mem[1],ymm0[4],mem[4],ymm0[5],mem[5] -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Reload -; AVX-NEXT: vshufps $17, {{[-0-9]+}}(%r{{[sb]}}p), %ymm10, %ymm10 # 32-byte Folded Reload -; AVX-NEXT: # ymm10 = ymm10[1,0],mem[1,0],ymm10[5,4],mem[5,4] -; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm10[2,0],ymm0[2,3],ymm10[6,4],ymm0[6,7] -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Reload -; AVX-NEXT: vunpcklps {{[-0-9]+}}(%r{{[sb]}}p), %ymm10, %ymm10 # 32-byte Folded Reload -; AVX-NEXT: # ymm10 = ymm10[0],mem[0],ymm10[1],mem[1],ymm10[4],mem[4],ymm10[5],mem[5] -; AVX-NEXT: vextractf128 $1, %ymm10, %xmm10 -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm14 # 32-byte Reload -; AVX-NEXT: vshufps $17, {{[-0-9]+}}(%r{{[sb]}}p), %ymm14, %ymm14 # 32-byte Folded Reload -; AVX-NEXT: # ymm14 = ymm14[1,0],mem[1,0],ymm14[5,4],mem[5,4] -; AVX-NEXT: vshufps {{.*#+}} ymm14 = ymm14[2,0,2,3,6,4,6,7] -; AVX-NEXT: vextractf128 $1, %ymm14, %xmm14 -; AVX-NEXT: vblendps {{.*#+}} xmm10 = xmm14[0,1],xmm10[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm10[0,1,2,3],ymm0[4,5,6,7] +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload +; AVX-NEXT: vshufps $17, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm2 # 32-byte Folded Reload +; AVX-NEXT: # ymm2 = ymm2[1,0],mem[1,0],ymm2[5,4],mem[5,4] +; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm2[2,0],ymm0[2,3],ymm2[6,4],ymm0[6,7] +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload +; AVX-NEXT: vunpcklps {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm2 # 32-byte Folded Reload +; AVX-NEXT: # ymm2 = ymm2[0],mem[0],ymm2[1],mem[1],ymm2[4],mem[4],ymm2[5],mem[5] +; AVX-NEXT: vextractf128 $1, %ymm2, %xmm2 +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm15 # 32-byte Reload +; AVX-NEXT: vshufps $17, {{[-0-9]+}}(%r{{[sb]}}p), %ymm15, %ymm15 # 32-byte Folded Reload +; AVX-NEXT: # ymm15 = ymm15[1,0],mem[1,0],ymm15[5,4],mem[5,4] +; AVX-NEXT: vextractf128 $1, %ymm15, %xmm15 +; AVX-NEXT: vshufps {{.*#+}} xmm2 = xmm15[2,0],xmm2[2,3] +; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm0[4,5,6,7] ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload ; AVX-NEXT: vunpcklps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload ; AVX-NEXT: # ymm0 = ymm0[0],mem[0],ymm0[1],mem[1],ymm0[4],mem[4],ymm0[5],mem[5] -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Reload -; AVX-NEXT: vshufps $17, {{[-0-9]+}}(%r{{[sb]}}p), %ymm10, %ymm10 # 32-byte Folded Reload -; AVX-NEXT: # ymm10 = ymm10[1,0],mem[1,0],ymm10[5,4],mem[5,4] -; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm10[2,0],ymm0[2,3],ymm10[6,4],ymm0[6,7] -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Reload -; AVX-NEXT: vunpcklps {{[-0-9]+}}(%r{{[sb]}}p), %ymm10, %ymm10 # 32-byte Folded Reload -; AVX-NEXT: # ymm10 = ymm10[0],mem[0],ymm10[1],mem[1],ymm10[4],mem[4],ymm10[5],mem[5] -; AVX-NEXT: vextractf128 $1, %ymm10, %xmm10 -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm14 # 32-byte Reload -; AVX-NEXT: vshufps $17, {{[-0-9]+}}(%r{{[sb]}}p), %ymm14, %ymm14 # 32-byte Folded Reload -; AVX-NEXT: # ymm14 = ymm14[1,0],mem[1,0],ymm14[5,4],mem[5,4] -; AVX-NEXT: vshufps {{.*#+}} ymm14 = ymm14[2,0,2,3,6,4,6,7] -; AVX-NEXT: vextractf128 $1, %ymm14, %xmm14 -; AVX-NEXT: vblendps {{.*#+}} xmm10 = xmm14[0,1],xmm10[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm10[0,1,2,3],ymm0[4,5,6,7] +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload +; AVX-NEXT: vshufps $17, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm2 # 32-byte Folded Reload +; AVX-NEXT: # ymm2 = ymm2[1,0],mem[1,0],ymm2[5,4],mem[5,4] +; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm2[2,0],ymm0[2,3],ymm2[6,4],ymm0[6,7] +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload +; AVX-NEXT: vunpcklps {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm2 # 32-byte Folded Reload +; AVX-NEXT: # ymm2 = ymm2[0],mem[0],ymm2[1],mem[1],ymm2[4],mem[4],ymm2[5],mem[5] +; AVX-NEXT: vextractf128 $1, %ymm2, %xmm2 +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm15 # 32-byte Reload +; AVX-NEXT: vshufps $17, {{[-0-9]+}}(%r{{[sb]}}p), %ymm15, %ymm15 # 32-byte Folded Reload +; AVX-NEXT: # ymm15 = ymm15[1,0],mem[1,0],ymm15[5,4],mem[5,4] +; AVX-NEXT: vextractf128 $1, %ymm15, %xmm15 +; AVX-NEXT: vshufps {{.*#+}} xmm2 = xmm15[2,0],xmm2[2,3] +; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm0[4,5,6,7] ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload ; AVX-NEXT: vunpcklps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload ; AVX-NEXT: # ymm0 = ymm0[0],mem[0],ymm0[1],mem[1],ymm0[4],mem[4],ymm0[5],mem[5] -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Reload -; AVX-NEXT: vshufps $17, {{[-0-9]+}}(%r{{[sb]}}p), %ymm10, %ymm10 # 32-byte Folded Reload -; AVX-NEXT: # ymm10 = ymm10[1,0],mem[1,0],ymm10[5,4],mem[5,4] -; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm10[2,0],ymm0[2,3],ymm10[6,4],ymm0[6,7] -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Reload -; AVX-NEXT: vunpcklps {{[-0-9]+}}(%r{{[sb]}}p), %ymm10, %ymm10 # 32-byte Folded Reload -; AVX-NEXT: # ymm10 = ymm10[0],mem[0],ymm10[1],mem[1],ymm10[4],mem[4],ymm10[5],mem[5] -; AVX-NEXT: vextractf128 $1, %ymm10, %xmm10 -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm14 # 32-byte Reload -; AVX-NEXT: vshufps $17, {{[-0-9]+}}(%r{{[sb]}}p), %ymm14, %ymm14 # 32-byte Folded Reload -; AVX-NEXT: # ymm14 = ymm14[1,0],mem[1,0],ymm14[5,4],mem[5,4] -; AVX-NEXT: vshufps {{.*#+}} ymm14 = ymm14[2,0,2,3,6,4,6,7] -; AVX-NEXT: vextractf128 $1, %ymm14, %xmm14 -; AVX-NEXT: vblendps {{.*#+}} xmm10 = xmm14[0,1],xmm10[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm10[0,1,2,3],ymm0[4,5,6,7] +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload +; AVX-NEXT: vshufps $17, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm2 # 32-byte Folded Reload +; AVX-NEXT: # ymm2 = ymm2[1,0],mem[1,0],ymm2[5,4],mem[5,4] +; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm2[2,0],ymm0[2,3],ymm2[6,4],ymm0[6,7] +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload +; AVX-NEXT: vunpcklps {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm2 # 32-byte Folded Reload +; AVX-NEXT: # ymm2 = ymm2[0],mem[0],ymm2[1],mem[1],ymm2[4],mem[4],ymm2[5],mem[5] +; AVX-NEXT: vextractf128 $1, %ymm2, %xmm2 +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm15 # 32-byte Reload +; AVX-NEXT: vshufps $17, (%rsp), %ymm15, %ymm15 # 32-byte Folded Reload +; AVX-NEXT: # ymm15 = ymm15[1,0],mem[1,0],ymm15[5,4],mem[5,4] +; AVX-NEXT: vextractf128 $1, %ymm15, %xmm15 +; AVX-NEXT: vshufps {{.*#+}} xmm2 = xmm15[2,0],xmm2[2,3] +; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm0[4,5,6,7] ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload -; AVX-NEXT: vunpcklps (%rsp), %ymm0, %ymm0 # 32-byte Folded Reload +; AVX-NEXT: vunpcklps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload ; AVX-NEXT: # ymm0 = ymm0[0],mem[0],ymm0[1],mem[1],ymm0[4],mem[4],ymm0[5],mem[5] -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Reload -; AVX-NEXT: vshufps $17, {{[-0-9]+}}(%r{{[sb]}}p), %ymm10, %ymm10 # 32-byte Folded Reload -; AVX-NEXT: # ymm10 = ymm10[1,0],mem[1,0],ymm10[5,4],mem[5,4] -; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm10[2,0],ymm0[2,3],ymm10[6,4],ymm0[6,7] -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Reload -; AVX-NEXT: vunpcklps {{[-0-9]+}}(%r{{[sb]}}p), %ymm10, %ymm10 # 32-byte Folded Reload -; AVX-NEXT: # ymm10 = ymm10[0],mem[0],ymm10[1],mem[1],ymm10[4],mem[4],ymm10[5],mem[5] -; AVX-NEXT: vextractf128 $1, %ymm10, %xmm10 -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm14 # 32-byte Reload -; AVX-NEXT: vshufps $17, {{[-0-9]+}}(%r{{[sb]}}p), %ymm14, %ymm14 # 32-byte Folded Reload -; AVX-NEXT: # ymm14 = ymm14[1,0],mem[1,0],ymm14[5,4],mem[5,4] -; AVX-NEXT: vshufps {{.*#+}} ymm14 = ymm14[2,0,2,3,6,4,6,7] -; AVX-NEXT: vextractf128 $1, %ymm14, %xmm14 -; AVX-NEXT: vblendps {{.*#+}} xmm10 = xmm14[0,1],xmm10[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm10[0,1,2,3],ymm0[4,5,6,7] +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload +; AVX-NEXT: vshufps $17, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm2 # 32-byte Folded Reload +; AVX-NEXT: # ymm2 = ymm2[1,0],mem[1,0],ymm2[5,4],mem[5,4] +; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm2[2,0],ymm0[2,3],ymm2[6,4],ymm0[6,7] +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload +; AVX-NEXT: vunpcklps {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm2 # 32-byte Folded Reload +; AVX-NEXT: # ymm2 = ymm2[0],mem[0],ymm2[1],mem[1],ymm2[4],mem[4],ymm2[5],mem[5] +; AVX-NEXT: vextractf128 $1, %ymm2, %xmm2 +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm15 # 32-byte Reload +; AVX-NEXT: vshufps $17, {{[-0-9]+}}(%r{{[sb]}}p), %ymm15, %ymm15 # 32-byte Folded Reload +; AVX-NEXT: # ymm15 = ymm15[1,0],mem[1,0],ymm15[5,4],mem[5,4] +; AVX-NEXT: vextractf128 $1, %ymm15, %xmm15 +; AVX-NEXT: vshufps {{.*#+}} xmm2 = xmm15[2,0],xmm2[2,3] +; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm0[4,5,6,7] ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX-NEXT: vunpckhpd {{.*#+}} ymm0 = ymm3[1],ymm5[1],ymm3[3],ymm5[3] -; AVX-NEXT: vunpckhps {{.*#+}} ymm10 = ymm4[2],ymm1[2],ymm4[3],ymm1[3],ymm4[6],ymm1[6],ymm4[7],ymm1[7] -; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm10[0,1],ymm0[2,0],ymm10[4,5],ymm0[6,4] -; AVX-NEXT: vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm10 # 32-byte Folded Reload -; AVX-NEXT: # ymm10 = ymm2[2],mem[2],ymm2[3],mem[3],ymm2[6],mem[6],ymm2[7],mem[7] -; AVX-NEXT: vextractf128 $1, %ymm10, %xmm10 -; AVX-NEXT: vunpckhpd {{.*#+}} ymm14 = ymm15[1],ymm9[1],ymm15[3],ymm9[3] -; AVX-NEXT: vshufps {{.*#+}} ymm14 = ymm14[0,1,2,0,4,5,6,4] -; AVX-NEXT: vextractf128 $1, %ymm14, %xmm14 -; AVX-NEXT: vblendps {{.*#+}} xmm10 = xmm10[0,1],xmm14[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm10[0,1,2,3],ymm0[4,5,6,7] +; AVX-NEXT: vunpckhps {{.*#+}} ymm2 = ymm4[2],ymm1[2],ymm4[3],ymm1[3],ymm4[6],ymm1[6],ymm4[7],ymm1[7] +; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm2[0,1],ymm0[2,0],ymm2[4,5],ymm0[6,4] +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload +; AVX-NEXT: vunpckhpd {{.*#+}} ymm2 = ymm4[1],ymm6[1],ymm4[3],ymm6[3] +; AVX-NEXT: vextractf128 $1, %ymm2, %xmm2 +; AVX-NEXT: vunpckhps {{.*#+}} ymm15 = ymm7[2],ymm8[2],ymm7[3],ymm8[3],ymm7[6],ymm8[6],ymm7[7],ymm8[7] +; AVX-NEXT: vextractf128 $1, %ymm15, %xmm15 +; AVX-NEXT: vshufps {{.*#+}} xmm2 = xmm15[0,1],xmm2[2,0] +; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm0[4,5,6,7] ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vunpckhpd {{[-0-9]+}}(%r{{[sb]}}p), %ymm6, %ymm0 # 32-byte Folded Reload -; AVX-NEXT: # ymm0 = ymm6[1],mem[1],ymm6[3],mem[3] -; AVX-NEXT: vunpckhps {{.*#+}} ymm10 = ymm8[2],ymm12[2],ymm8[3],ymm12[3],ymm8[6],ymm12[6],ymm8[7],ymm12[7] -; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm10[0,1],ymm0[2,0],ymm10[4,5],ymm0[6,4] -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm12 # 32-byte Reload -; AVX-NEXT: vunpckhps {{.*#+}} ymm10 = ymm11[2],ymm12[2],ymm11[3],ymm12[3],ymm11[6],ymm12[6],ymm11[7],ymm12[7] -; AVX-NEXT: vextractf128 $1, %ymm10, %xmm10 +; AVX-NEXT: vunpckhpd {{.*#+}} ymm0 = ymm10[1],ymm13[1],ymm10[3],ymm13[3] +; AVX-NEXT: vunpckhps {{.*#+}} ymm2 = ymm9[2],ymm12[2],ymm9[3],ymm12[3],ymm9[6],ymm12[6],ymm9[7],ymm12[7] +; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm2[0,1],ymm0[2,0],ymm2[4,5],ymm0[6,4] +; AVX-NEXT: vunpckhpd {{[-0-9]+}}(%r{{[sb]}}p), %ymm11, %ymm2 # 32-byte Folded Reload +; AVX-NEXT: # ymm2 = ymm11[1],mem[1],ymm11[3],mem[3] +; AVX-NEXT: vextractf128 $1, %ymm2, %xmm2 ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload -; AVX-NEXT: vunpckhpd {{.*#+}} ymm14 = ymm1[1],ymm13[1],ymm1[3],ymm13[3] -; AVX-NEXT: vshufps {{.*#+}} ymm14 = ymm14[0,1,2,0,4,5,6,4] -; AVX-NEXT: vextractf128 $1, %ymm14, %xmm14 -; AVX-NEXT: vblendps {{.*#+}} xmm10 = xmm10[0,1],xmm14[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm10[0,1,2,3],ymm0[4,5,6,7] +; AVX-NEXT: vunpckhps {{.*#+}} ymm15 = ymm14[2],ymm1[2],ymm14[3],ymm1[3],ymm14[6],ymm1[6],ymm14[7],ymm1[7] +; AVX-NEXT: vextractf128 $1, %ymm15, %xmm15 +; AVX-NEXT: vshufps {{.*#+}} xmm2 = xmm15[0,1],xmm2[2,0] +; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm0[4,5,6,7] ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload -; AVX-NEXT: vunpckhpd {{.*#+}} ymm0 = ymm6[1],ymm4[1],ymm6[3],ymm4[3] -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload -; AVX-NEXT: vunpckhps {{.*#+}} ymm10 = ymm7[2],ymm5[2],ymm7[3],ymm5[3],ymm7[6],ymm5[6],ymm7[7],ymm5[7] -; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm10[0,1],ymm0[2,0],ymm10[4,5],ymm0[6,4] -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm13 # 32-byte Reload ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm11 # 32-byte Reload -; AVX-NEXT: vunpckhps {{.*#+}} ymm10 = ymm13[2],ymm11[2],ymm13[3],ymm11[3],ymm13[6],ymm11[6],ymm13[7],ymm11[7] -; AVX-NEXT: vextractf128 $1, %ymm10, %xmm10 +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload +; AVX-NEXT: vunpckhpd {{.*#+}} ymm0 = ymm11[1],ymm5[1],ymm11[3],ymm5[3] +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Reload ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload -; AVX-NEXT: vunpckhpd {{.*#+}} ymm14 = ymm3[1],ymm2[1],ymm3[3],ymm2[3] -; AVX-NEXT: vshufps {{.*#+}} ymm14 = ymm14[0,1,2,0,4,5,6,4] -; AVX-NEXT: vextractf128 $1, %ymm14, %xmm14 -; AVX-NEXT: vblendps {{.*#+}} xmm10 = xmm10[0,1],xmm14[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm10[0,1,2,3],ymm0[4,5,6,7] -; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vunpckhps {{.*#+}} ymm2 = ymm9[2],ymm3[2],ymm9[3],ymm3[3],ymm9[6],ymm3[6],ymm9[7],ymm3[7] +; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm2[0,1],ymm0[2,0],ymm2[4,5],ymm0[6,4] ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload -; AVX-NEXT: vunpckhpd {{[-0-9]+}}(%r{{[sb]}}p), %ymm8, %ymm0 # 32-byte Folded Reload -; AVX-NEXT: # ymm0 = ymm8[1],mem[1],ymm8[3],mem[3] -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Reload +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload +; AVX-NEXT: vunpckhpd {{.*#+}} ymm2 = ymm8[1],ymm6[1],ymm8[3],ymm6[3] +; AVX-NEXT: vextractf128 $1, %ymm2, %xmm2 +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Reload ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload -; AVX-NEXT: vunpckhps {{.*#+}} ymm10 = ymm9[2],ymm7[2],ymm9[3],ymm7[3],ymm9[6],ymm7[6],ymm9[7],ymm7[7] -; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm10[0,1],ymm0[2,0],ymm10[4,5],ymm0[6,4] -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm15 # 32-byte Reload -; AVX-NEXT: vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %ymm15, %ymm10 # 32-byte Folded Reload -; AVX-NEXT: # ymm10 = ymm15[2],mem[2],ymm15[3],mem[3],ymm15[6],mem[6],ymm15[7],mem[7] -; AVX-NEXT: vextractf128 $1, %ymm10, %xmm10 +; AVX-NEXT: vunpckhps {{.*#+}} ymm15 = ymm10[2],ymm7[2],ymm10[3],ymm7[3],ymm10[6],ymm7[6],ymm10[7],ymm7[7] +; AVX-NEXT: vextractf128 $1, %ymm15, %xmm15 +; AVX-NEXT: vshufps {{.*#+}} xmm2 = xmm15[0,1],xmm2[2,0] +; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm0[4,5,6,7] +; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm13 # 32-byte Reload +; AVX-NEXT: vunpckhpd {{[-0-9]+}}(%r{{[sb]}}p), %ymm13, %ymm0 # 32-byte Folded Reload +; AVX-NEXT: # ymm0 = ymm13[1],mem[1],ymm13[3],mem[3] ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm14 # 32-byte Reload -; AVX-NEXT: vunpckhpd {{[-0-9]+}}(%r{{[sb]}}p), %ymm14, %ymm14 # 32-byte Folded Reload -; AVX-NEXT: # ymm14 = ymm14[1],mem[1],ymm14[3],mem[3] -; AVX-NEXT: vshufps {{.*#+}} ymm14 = ymm14[0,1,2,0,4,5,6,4] -; AVX-NEXT: vextractf128 $1, %ymm14, %xmm14 -; AVX-NEXT: vblendps {{.*#+}} xmm10 = xmm10[0,1],xmm14[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm10[0,1,2,3],ymm0[4,5,6,7] +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm12 # 32-byte Reload +; AVX-NEXT: vunpckhps {{.*#+}} ymm2 = ymm14[2],ymm12[2],ymm14[3],ymm12[3],ymm14[6],ymm12[6],ymm14[7],ymm12[7] +; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm2[0,1],ymm0[2,0],ymm2[4,5],ymm0[6,4] +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload +; AVX-NEXT: vunpckhpd {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm2 # 32-byte Folded Reload +; AVX-NEXT: # ymm2 = ymm2[1],mem[1],ymm2[3],mem[3] +; AVX-NEXT: vextractf128 $1, %ymm2, %xmm2 +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm15 # 32-byte Reload +; AVX-NEXT: vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %ymm15, %ymm15 # 32-byte Folded Reload +; AVX-NEXT: # ymm15 = ymm15[2],mem[2],ymm15[3],mem[3],ymm15[6],mem[6],ymm15[7],mem[7] +; AVX-NEXT: vextractf128 $1, %ymm15, %xmm15 +; AVX-NEXT: vshufps {{.*#+}} xmm2 = xmm15[0,1],xmm2[2,0] +; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm0[4,5,6,7] ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload ; AVX-NEXT: vunpckhpd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload ; AVX-NEXT: # ymm0 = ymm0[1],mem[1],ymm0[3],mem[3] -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Reload -; AVX-NEXT: vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %ymm10, %ymm10 # 32-byte Folded Reload -; AVX-NEXT: # ymm10 = ymm10[2],mem[2],ymm10[3],mem[3],ymm10[6],mem[6],ymm10[7],mem[7] -; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm10[0,1],ymm0[2,0],ymm10[4,5],ymm0[6,4] -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Reload -; AVX-NEXT: vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %ymm10, %ymm10 # 32-byte Folded Reload -; AVX-NEXT: # ymm10 = ymm10[2],mem[2],ymm10[3],mem[3],ymm10[6],mem[6],ymm10[7],mem[7] -; AVX-NEXT: vextractf128 $1, %ymm10, %xmm10 -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm14 # 32-byte Reload -; AVX-NEXT: vunpckhpd {{[-0-9]+}}(%r{{[sb]}}p), %ymm14, %ymm14 # 32-byte Folded Reload -; AVX-NEXT: # ymm14 = ymm14[1],mem[1],ymm14[3],mem[3] -; AVX-NEXT: vshufps {{.*#+}} ymm14 = ymm14[0,1,2,0,4,5,6,4] -; AVX-NEXT: vextractf128 $1, %ymm14, %xmm14 -; AVX-NEXT: vblendps {{.*#+}} xmm10 = xmm10[0,1],xmm14[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm10[0,1,2,3],ymm0[4,5,6,7] +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload +; AVX-NEXT: vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm2 # 32-byte Folded Reload +; AVX-NEXT: # ymm2 = ymm2[2],mem[2],ymm2[3],mem[3],ymm2[6],mem[6],ymm2[7],mem[7] +; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm2[0,1],ymm0[2,0],ymm2[4,5],ymm0[6,4] +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload +; AVX-NEXT: vunpckhpd {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm2 # 32-byte Folded Reload +; AVX-NEXT: # ymm2 = ymm2[1],mem[1],ymm2[3],mem[3] +; AVX-NEXT: vextractf128 $1, %ymm2, %xmm2 +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm15 # 32-byte Reload +; AVX-NEXT: vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %ymm15, %ymm15 # 32-byte Folded Reload +; AVX-NEXT: # ymm15 = ymm15[2],mem[2],ymm15[3],mem[3],ymm15[6],mem[6],ymm15[7],mem[7] +; AVX-NEXT: vextractf128 $1, %ymm15, %xmm15 +; AVX-NEXT: vshufps {{.*#+}} xmm2 = xmm15[0,1],xmm2[2,0] +; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm0[4,5,6,7] ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload ; AVX-NEXT: vunpckhpd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload ; AVX-NEXT: # ymm0 = ymm0[1],mem[1],ymm0[3],mem[3] -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Reload -; AVX-NEXT: vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %ymm10, %ymm10 # 32-byte Folded Reload -; AVX-NEXT: # ymm10 = ymm10[2],mem[2],ymm10[3],mem[3],ymm10[6],mem[6],ymm10[7],mem[7] -; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm10[0,1],ymm0[2,0],ymm10[4,5],ymm0[6,4] -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Reload -; AVX-NEXT: vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %ymm10, %ymm10 # 32-byte Folded Reload -; AVX-NEXT: # ymm10 = ymm10[2],mem[2],ymm10[3],mem[3],ymm10[6],mem[6],ymm10[7],mem[7] -; AVX-NEXT: vextractf128 $1, %ymm10, %xmm10 -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm14 # 32-byte Reload -; AVX-NEXT: vunpckhpd {{[-0-9]+}}(%r{{[sb]}}p), %ymm14, %ymm14 # 32-byte Folded Reload -; AVX-NEXT: # ymm14 = ymm14[1],mem[1],ymm14[3],mem[3] -; AVX-NEXT: vshufps {{.*#+}} ymm14 = ymm14[0,1,2,0,4,5,6,4] -; AVX-NEXT: vextractf128 $1, %ymm14, %xmm14 -; AVX-NEXT: vblendps {{.*#+}} xmm10 = xmm10[0,1],xmm14[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm10[0,1,2,3],ymm0[4,5,6,7] +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload +; AVX-NEXT: vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm2 # 32-byte Folded Reload +; AVX-NEXT: # ymm2 = ymm2[2],mem[2],ymm2[3],mem[3],ymm2[6],mem[6],ymm2[7],mem[7] +; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm2[0,1],ymm0[2,0],ymm2[4,5],ymm0[6,4] +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload +; AVX-NEXT: vunpckhpd {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm2 # 32-byte Folded Reload +; AVX-NEXT: # ymm2 = ymm2[1],mem[1],ymm2[3],mem[3] +; AVX-NEXT: vextractf128 $1, %ymm2, %xmm2 +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm15 # 32-byte Reload +; AVX-NEXT: vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %ymm15, %ymm15 # 32-byte Folded Reload +; AVX-NEXT: # ymm15 = ymm15[2],mem[2],ymm15[3],mem[3],ymm15[6],mem[6],ymm15[7],mem[7] +; AVX-NEXT: vextractf128 $1, %ymm15, %xmm15 +; AVX-NEXT: vshufps {{.*#+}} xmm2 = xmm15[0,1],xmm2[2,0] +; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm0[4,5,6,7] ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload ; AVX-NEXT: vunpckhpd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload ; AVX-NEXT: # ymm0 = ymm0[1],mem[1],ymm0[3],mem[3] -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Reload -; AVX-NEXT: vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %ymm10, %ymm10 # 32-byte Folded Reload -; AVX-NEXT: # ymm10 = ymm10[2],mem[2],ymm10[3],mem[3],ymm10[6],mem[6],ymm10[7],mem[7] -; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm10[0,1],ymm0[2,0],ymm10[4,5],ymm0[6,4] -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Reload -; AVX-NEXT: vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %ymm10, %ymm10 # 32-byte Folded Reload -; AVX-NEXT: # ymm10 = ymm10[2],mem[2],ymm10[3],mem[3],ymm10[6],mem[6],ymm10[7],mem[7] -; AVX-NEXT: vextractf128 $1, %ymm10, %xmm10 -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm14 # 32-byte Reload -; AVX-NEXT: vunpckhpd {{[-0-9]+}}(%r{{[sb]}}p), %ymm14, %ymm14 # 32-byte Folded Reload -; AVX-NEXT: # ymm14 = ymm14[1],mem[1],ymm14[3],mem[3] -; AVX-NEXT: vshufps {{.*#+}} ymm14 = ymm14[0,1,2,0,4,5,6,4] -; AVX-NEXT: vextractf128 $1, %ymm14, %xmm14 -; AVX-NEXT: vblendps {{.*#+}} xmm10 = xmm10[0,1],xmm14[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm10[0,1,2,3],ymm0[4,5,6,7] +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload +; AVX-NEXT: vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm2 # 32-byte Folded Reload +; AVX-NEXT: # ymm2 = ymm2[2],mem[2],ymm2[3],mem[3],ymm2[6],mem[6],ymm2[7],mem[7] +; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm2[0,1],ymm0[2,0],ymm2[4,5],ymm0[6,4] +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload +; AVX-NEXT: vunpckhpd {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm2 # 32-byte Folded Reload +; AVX-NEXT: # ymm2 = ymm2[1],mem[1],ymm2[3],mem[3] +; AVX-NEXT: vextractf128 $1, %ymm2, %xmm2 +; AVX-NEXT: vmovups (%rsp), %ymm15 # 32-byte Reload +; AVX-NEXT: vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %ymm15, %ymm15 # 32-byte Folded Reload +; AVX-NEXT: # ymm15 = ymm15[2],mem[2],ymm15[3],mem[3],ymm15[6],mem[6],ymm15[7],mem[7] +; AVX-NEXT: vextractf128 $1, %ymm15, %xmm15 +; AVX-NEXT: vshufps {{.*#+}} xmm2 = xmm15[0,1],xmm2[2,0] +; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm0[4,5,6,7] ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vmovups (%rsp), %ymm0 # 32-byte Reload +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload ; AVX-NEXT: vunpckhpd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload ; AVX-NEXT: # ymm0 = ymm0[1],mem[1],ymm0[3],mem[3] -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Reload -; AVX-NEXT: vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %ymm10, %ymm10 # 32-byte Folded Reload -; AVX-NEXT: # ymm10 = ymm10[2],mem[2],ymm10[3],mem[3],ymm10[6],mem[6],ymm10[7],mem[7] -; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm10[0,1],ymm0[2,0],ymm10[4,5],ymm0[6,4] -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Reload -; AVX-NEXT: vunpckhpd {{[-0-9]+}}(%r{{[sb]}}p), %ymm10, %ymm10 # 32-byte Folded Reload -; AVX-NEXT: # ymm10 = ymm10[1],mem[1],ymm10[3],mem[3] -; AVX-NEXT: vshufps {{.*#+}} ymm10 = ymm10[0,1,2,0,4,5,6,4] -; AVX-NEXT: vextractf128 $1, %ymm10, %xmm10 -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm14 # 32-byte Reload -; AVX-NEXT: vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %ymm14, %ymm14 # 32-byte Folded Reload -; AVX-NEXT: # ymm14 = ymm14[2],mem[2],ymm14[3],mem[3],ymm14[6],mem[6],ymm14[7],mem[7] -; AVX-NEXT: vextractf128 $1, %ymm14, %xmm14 -; AVX-NEXT: vblendps {{.*#+}} xmm10 = xmm14[0,1],xmm10[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm10[0,1,2,3],ymm0[4,5,6,7] +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload +; AVX-NEXT: vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm2 # 32-byte Folded Reload +; AVX-NEXT: # ymm2 = ymm2[2],mem[2],ymm2[3],mem[3],ymm2[6],mem[6],ymm2[7],mem[7] +; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm2[0,1],ymm0[2,0],ymm2[4,5],ymm0[6,4] +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload +; AVX-NEXT: vunpckhpd {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm2 # 32-byte Folded Reload +; AVX-NEXT: # ymm2 = ymm2[1],mem[1],ymm2[3],mem[3] +; AVX-NEXT: vextractf128 $1, %ymm2, %xmm2 +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm15 # 32-byte Reload +; AVX-NEXT: vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %ymm15, %ymm15 # 32-byte Folded Reload +; AVX-NEXT: # ymm15 = ymm15[2],mem[2],ymm15[3],mem[3],ymm15[6],mem[6],ymm15[7],mem[7] +; AVX-NEXT: vextractf128 $1, %ymm15, %xmm15 +; AVX-NEXT: vshufps {{.*#+}} xmm2 = xmm15[0,1],xmm2[2,0] +; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm0[4,5,6,7] ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload ; AVX-NEXT: vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload ; AVX-NEXT: # ymm0 = ymm0[2],mem[2],ymm0[3],mem[3],ymm0[6],mem[6],ymm0[7],mem[7] -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Reload -; AVX-NEXT: vshufps $51, {{[-0-9]+}}(%r{{[sb]}}p), %ymm10, %ymm10 # 32-byte Folded Reload -; AVX-NEXT: # ymm10 = ymm10[3,0],mem[3,0],ymm10[7,4],mem[7,4] -; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm10[2,0],ymm0[2,3],ymm10[6,4],ymm0[6,7] -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Reload -; AVX-NEXT: vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %ymm10, %ymm10 # 32-byte Folded Reload -; AVX-NEXT: # ymm10 = ymm10[2],mem[2],ymm10[3],mem[3],ymm10[6],mem[6],ymm10[7],mem[7] -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm14 # 32-byte Reload -; AVX-NEXT: vshufps $51, {{[-0-9]+}}(%r{{[sb]}}p), %ymm14, %ymm14 # 32-byte Folded Reload -; AVX-NEXT: # ymm14 = ymm14[3,0],mem[3,0],ymm14[7,4],mem[7,4] -; AVX-NEXT: vextractf128 $1, %ymm10, %xmm10 -; AVX-NEXT: vshufps {{.*#+}} ymm14 = ymm14[2,0,2,3,6,4,6,7] -; AVX-NEXT: vextractf128 $1, %ymm14, %xmm14 -; AVX-NEXT: vblendps {{.*#+}} xmm10 = xmm14[0,1],xmm10[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm10[0,1,2,3],ymm0[4,5,6,7] -; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload +; AVX-NEXT: vshufps $51, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm2 # 32-byte Folded Reload +; AVX-NEXT: # ymm2 = ymm2[3,0],mem[3,0],ymm2[7,4],mem[7,4] +; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm2[2,0],ymm0[2,3],ymm2[6,4],ymm0[6,7] +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload +; AVX-NEXT: vunpckhps {{.*#+}} ymm2 = ymm2[2],ymm4[2],ymm2[3],ymm4[3],ymm2[6],ymm4[6],ymm2[7],ymm4[7] +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload +; AVX-NEXT: vshufps $51, {{[-0-9]+}}(%r{{[sb]}}p), %ymm4, %ymm15 # 32-byte Folded Reload +; AVX-NEXT: # ymm15 = ymm4[3,0],mem[3,0],ymm4[7,4],mem[7,4] +; AVX-NEXT: vextractf128 $1, %ymm2, %xmm2 +; AVX-NEXT: vextractf128 $1, %ymm15, %xmm15 +; AVX-NEXT: vshufps {{.*#+}} xmm2 = xmm15[2,0],xmm2[2,3] +; AVX-NEXT: vblendps {{.*#+}} ymm4 = ymm2[0,1,2,3],ymm0[4,5,6,7] ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload ; AVX-NEXT: vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload ; AVX-NEXT: # ymm0 = ymm0[2],mem[2],ymm0[3],mem[3],ymm0[6],mem[6],ymm0[7],mem[7] -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Reload -; AVX-NEXT: vshufps $51, {{[-0-9]+}}(%r{{[sb]}}p), %ymm10, %ymm10 # 32-byte Folded Reload -; AVX-NEXT: # ymm10 = ymm10[3,0],mem[3,0],ymm10[7,4],mem[7,4] -; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm10[2,0],ymm0[2,3],ymm10[6,4],ymm0[6,7] -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Reload -; AVX-NEXT: vunpckhps {{.*#+}} ymm10 = ymm10[2],ymm1[2],ymm10[3],ymm1[3],ymm10[6],ymm1[6],ymm10[7],ymm1[7] -; AVX-NEXT: vshufps $51, {{[-0-9]+}}(%r{{[sb]}}p), %ymm12, %ymm14 # 32-byte Folded Reload -; AVX-NEXT: # ymm14 = ymm12[3,0],mem[3,0],ymm12[7,4],mem[7,4] -; AVX-NEXT: vextractf128 $1, %ymm10, %xmm10 -; AVX-NEXT: vshufps {{.*#+}} ymm14 = ymm14[2,0,2,3,6,4,6,7] -; AVX-NEXT: vextractf128 $1, %ymm14, %xmm14 -; AVX-NEXT: vblendps {{.*#+}} xmm10 = xmm14[0,1],xmm10[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm12 = ymm10[0,1,2,3],ymm0[4,5,6,7] -; AVX-NEXT: vunpckhps {{.*#+}} ymm0 = ymm4[2],ymm6[2],ymm4[3],ymm6[3],ymm4[6],ymm6[6],ymm4[7],ymm6[7] -; AVX-NEXT: vshufps $51, {{[-0-9]+}}(%r{{[sb]}}p), %ymm5, %ymm10 # 32-byte Folded Reload -; AVX-NEXT: # ymm10 = ymm5[3,0],mem[3,0],ymm5[7,4],mem[7,4] -; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm10[2,0],ymm0[2,3],ymm10[6,4],ymm0[6,7] -; AVX-NEXT: vunpckhps {{.*#+}} ymm10 = ymm2[2],ymm3[2],ymm2[3],ymm3[3],ymm2[6],ymm3[6],ymm2[7],ymm3[7] -; AVX-NEXT: vshufps {{.*#+}} ymm14 = ymm11[3,0],ymm13[3,0],ymm11[7,4],ymm13[7,4] -; AVX-NEXT: vextractf128 $1, %ymm10, %xmm10 -; AVX-NEXT: vshufps {{.*#+}} ymm14 = ymm14[2,0,2,3,6,4,6,7] -; AVX-NEXT: vextractf128 $1, %ymm14, %xmm14 -; AVX-NEXT: vblendps {{.*#+}} xmm10 = xmm14[0,1],xmm10[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm3 = ymm10[0,1,2,3],ymm0[4,5,6,7] +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload +; AVX-NEXT: vshufps $51, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm2 # 32-byte Folded Reload +; AVX-NEXT: # ymm2 = ymm2[3,0],mem[3,0],ymm2[7,4],mem[7,4] +; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm2[2,0],ymm0[2,3],ymm2[6,4],ymm0[6,7] +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload +; AVX-NEXT: vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm2 # 32-byte Folded Reload +; AVX-NEXT: # ymm2 = ymm2[2],mem[2],ymm2[3],mem[3],ymm2[6],mem[6],ymm2[7],mem[7] +; AVX-NEXT: vshufps $51, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm15 # 32-byte Folded Reload +; AVX-NEXT: # ymm15 = ymm1[3,0],mem[3,0],ymm1[7,4],mem[7,4] +; AVX-NEXT: vextractf128 $1, %ymm2, %xmm2 +; AVX-NEXT: vextractf128 $1, %ymm15, %xmm15 +; AVX-NEXT: vshufps {{.*#+}} xmm2 = xmm15[2,0],xmm2[2,3] +; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm0[4,5,6,7] +; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vunpckhps {{.*#+}} ymm0 = ymm5[2],ymm11[2],ymm5[3],ymm11[3],ymm5[6],ymm11[6],ymm5[7],ymm11[7] +; AVX-NEXT: vshufps {{.*#+}} ymm2 = ymm3[3,0],ymm9[3,0],ymm3[7,4],ymm9[7,4] +; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm2[2,0],ymm0[2,3],ymm2[6,4],ymm0[6,7] +; AVX-NEXT: vunpckhps {{.*#+}} ymm2 = ymm6[2],ymm8[2],ymm6[3],ymm8[3],ymm6[6],ymm8[6],ymm6[7],ymm8[7] +; AVX-NEXT: vshufps {{.*#+}} ymm15 = ymm7[3,0],ymm10[3,0],ymm7[7,4],ymm10[7,4] +; AVX-NEXT: vextractf128 $1, %ymm2, %xmm2 +; AVX-NEXT: vextractf128 $1, %ymm15, %xmm15 +; AVX-NEXT: vshufps {{.*#+}} xmm2 = xmm15[2,0],xmm2[2,3] +; AVX-NEXT: vblendps {{.*#+}} ymm3 = ymm2[0,1,2,3],ymm0[4,5,6,7] ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload -; AVX-NEXT: vunpckhps {{.*#+}} ymm0 = ymm0[2],ymm8[2],ymm0[3],ymm8[3],ymm0[6],ymm8[6],ymm0[7],ymm8[7] -; AVX-NEXT: vshufps {{.*#+}} ymm10 = ymm7[3,0],ymm9[3,0],ymm7[7,4],ymm9[7,4] -; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm10[2,0],ymm0[2,3],ymm10[6,4],ymm0[6,7] +; AVX-NEXT: vunpckhps {{.*#+}} ymm0 = ymm0[2],ymm13[2],ymm0[3],ymm13[3],ymm0[6],ymm13[6],ymm0[7],ymm13[7] +; AVX-NEXT: vshufps {{.*#+}} ymm2 = ymm12[3,0],ymm14[3,0],ymm12[7,4],ymm14[7,4] +; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm2[2,0],ymm0[2,3],ymm2[6,4],ymm0[6,7] ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload -; AVX-NEXT: vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm10 # 32-byte Folded Reload -; AVX-NEXT: # ymm10 = ymm1[2],mem[2],ymm1[3],mem[3],ymm1[6],mem[6],ymm1[7],mem[7] +; AVX-NEXT: vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm2 # 32-byte Folded Reload +; AVX-NEXT: # ymm2 = ymm1[2],mem[2],ymm1[3],mem[3],ymm1[6],mem[6],ymm1[7],mem[7] ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload -; AVX-NEXT: vshufps {{.*#+}} ymm14 = ymm1[3,0],ymm15[3,0],ymm1[7,4],ymm15[7,4] -; AVX-NEXT: vextractf128 $1, %ymm10, %xmm10 -; AVX-NEXT: vshufps {{.*#+}} ymm14 = ymm14[2,0,2,3,6,4,6,7] -; AVX-NEXT: vextractf128 $1, %ymm14, %xmm14 -; AVX-NEXT: vblendps {{.*#+}} xmm10 = xmm14[0,1],xmm10[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm4 = ymm10[0,1,2,3],ymm0[4,5,6,7] +; AVX-NEXT: vshufps $51, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm15 # 32-byte Folded Reload +; AVX-NEXT: # ymm15 = ymm1[3,0],mem[3,0],ymm1[7,4],mem[7,4] +; AVX-NEXT: vextractf128 $1, %ymm2, %xmm2 +; AVX-NEXT: vextractf128 $1, %ymm15, %xmm15 +; AVX-NEXT: vshufps {{.*#+}} xmm2 = xmm15[2,0],xmm2[2,3] +; AVX-NEXT: vblendps {{.*#+}} ymm5 = ymm2[0,1,2,3],ymm0[4,5,6,7] ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload ; AVX-NEXT: vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload ; AVX-NEXT: # ymm0 = ymm0[2],mem[2],ymm0[3],mem[3],ymm0[6],mem[6],ymm0[7],mem[7] ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload -; AVX-NEXT: vshufps $51, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm10 # 32-byte Folded Reload -; AVX-NEXT: # ymm10 = ymm1[3,0],mem[3,0],ymm1[7,4],mem[7,4] -; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm10[2,0],ymm0[2,3],ymm10[6,4],ymm0[6,7] +; AVX-NEXT: vshufps $51, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm2 # 32-byte Folded Reload +; AVX-NEXT: # ymm2 = ymm1[3,0],mem[3,0],ymm1[7,4],mem[7,4] +; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm2[2,0],ymm0[2,3],ymm2[6,4],ymm0[6,7] ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload -; AVX-NEXT: vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm10 # 32-byte Folded Reload -; AVX-NEXT: # ymm10 = ymm1[2],mem[2],ymm1[3],mem[3],ymm1[6],mem[6],ymm1[7],mem[7] +; AVX-NEXT: vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm2 # 32-byte Folded Reload +; AVX-NEXT: # ymm2 = ymm1[2],mem[2],ymm1[3],mem[3],ymm1[6],mem[6],ymm1[7],mem[7] ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload -; AVX-NEXT: vshufps $51, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm14 # 32-byte Folded Reload -; AVX-NEXT: # ymm14 = ymm1[3,0],mem[3,0],ymm1[7,4],mem[7,4] -; AVX-NEXT: vextractf128 $1, %ymm10, %xmm10 -; AVX-NEXT: vshufps {{.*#+}} ymm14 = ymm14[2,0,2,3,6,4,6,7] -; AVX-NEXT: vextractf128 $1, %ymm14, %xmm14 -; AVX-NEXT: vblendps {{.*#+}} xmm10 = xmm14[0,1],xmm10[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm10 = ymm10[0,1,2,3],ymm0[4,5,6,7] +; AVX-NEXT: vshufps $51, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm15 # 32-byte Folded Reload +; AVX-NEXT: # ymm15 = ymm1[3,0],mem[3,0],ymm1[7,4],mem[7,4] +; AVX-NEXT: vextractf128 $1, %ymm2, %xmm2 +; AVX-NEXT: vextractf128 $1, %ymm15, %xmm15 +; AVX-NEXT: vshufps {{.*#+}} xmm2 = xmm15[2,0],xmm2[2,3] +; AVX-NEXT: vblendps {{.*#+}} ymm6 = ymm2[0,1,2,3],ymm0[4,5,6,7] ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload ; AVX-NEXT: vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload ; AVX-NEXT: # ymm0 = ymm0[2],mem[2],ymm0[3],mem[3],ymm0[6],mem[6],ymm0[7],mem[7] ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload -; AVX-NEXT: vshufps $51, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm14 # 32-byte Folded Reload -; AVX-NEXT: # ymm14 = ymm1[3,0],mem[3,0],ymm1[7,4],mem[7,4] -; AVX-NEXT: vshufps {{.*#+}} ymm1 = ymm14[2,0],ymm0[2,3],ymm14[6,4],ymm0[6,7] -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload -; AVX-NEXT: vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm14 # 32-byte Folded Reload -; AVX-NEXT: # ymm14 = ymm0[2],mem[2],ymm0[3],mem[3],ymm0[6],mem[6],ymm0[7],mem[7] -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload -; AVX-NEXT: vshufps $51, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload -; AVX-NEXT: # ymm0 = ymm0[3,0],mem[3,0],ymm0[7,4],mem[7,4] -; AVX-NEXT: vextractf128 $1, %ymm14, %xmm14 -; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm0[2,0,2,3,6,4,6,7] -; AVX-NEXT: vextractf128 $1, %ymm0, %xmm0 -; AVX-NEXT: vblendps {{.*#+}} xmm0 = xmm0[0,1],xmm14[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm14 = ymm0[0,1,2,3],ymm1[4,5,6,7] +; AVX-NEXT: vshufps $51, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm2 # 32-byte Folded Reload +; AVX-NEXT: # ymm2 = ymm1[3,0],mem[3,0],ymm1[7,4],mem[7,4] +; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm2[2,0],ymm0[2,3],ymm2[6,4],ymm0[6,7] +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload +; AVX-NEXT: vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm2 # 32-byte Folded Reload +; AVX-NEXT: # ymm2 = ymm1[2],mem[2],ymm1[3],mem[3],ymm1[6],mem[6],ymm1[7],mem[7] +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload +; AVX-NEXT: vshufps $51, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm15 # 32-byte Folded Reload +; AVX-NEXT: # ymm15 = ymm1[3,0],mem[3,0],ymm1[7,4],mem[7,4] +; AVX-NEXT: vextractf128 $1, %ymm2, %xmm2 +; AVX-NEXT: vextractf128 $1, %ymm15, %xmm15 +; AVX-NEXT: vshufps {{.*#+}} xmm2 = xmm15[2,0],xmm2[2,3] +; AVX-NEXT: vblendps {{.*#+}} ymm15 = ymm2[0,1,2,3],ymm0[4,5,6,7] ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload ; AVX-NEXT: vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload ; AVX-NEXT: # ymm0 = ymm0[2],mem[2],ymm0[3],mem[3],ymm0[6],mem[6],ymm0[7],mem[7] ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload -; AVX-NEXT: vshufps $51, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm1 # 32-byte Folded Reload -; AVX-NEXT: # ymm1 = ymm1[3,0],mem[3,0],ymm1[7,4],mem[7,4] -; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm1[2,0],ymm0[2,3],ymm1[6,4],ymm0[6,7] +; AVX-NEXT: vshufps $51, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm2 # 32-byte Folded Reload +; AVX-NEXT: # ymm2 = ymm1[3,0],mem[3,0],ymm1[7,4],mem[7,4] +; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm2[2,0],ymm0[2,3],ymm2[6,4],ymm0[6,7] ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload -; AVX-NEXT: vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm1 # 32-byte Folded Reload -; AVX-NEXT: # ymm1 = ymm1[2],mem[2],ymm1[3],mem[3],ymm1[6],mem[6],ymm1[7],mem[7] -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload -; AVX-NEXT: vshufps $51, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm5 # 32-byte Folded Reload -; AVX-NEXT: # ymm5 = ymm2[3,0],mem[3,0],ymm2[7,4],mem[7,4] -; AVX-NEXT: vextractf128 $1, %ymm1, %xmm1 -; AVX-NEXT: vshufps {{.*#+}} ymm5 = ymm5[2,0,2,3,6,4,6,7] -; AVX-NEXT: vextractf128 $1, %ymm5, %xmm5 -; AVX-NEXT: vblendps {{.*#+}} xmm1 = xmm5[0,1],xmm1[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm5 = ymm1[0,1,2,3],ymm0[4,5,6,7] +; AVX-NEXT: vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm2 # 32-byte Folded Reload +; AVX-NEXT: # ymm2 = ymm1[2],mem[2],ymm1[3],mem[3],ymm1[6],mem[6],ymm1[7],mem[7] +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload +; AVX-NEXT: vshufps $51, (%rsp), %ymm1, %ymm7 # 32-byte Folded Reload +; AVX-NEXT: # ymm7 = ymm1[3,0],mem[3,0],ymm1[7,4],mem[7,4] +; AVX-NEXT: vextractf128 $1, %ymm2, %xmm2 +; AVX-NEXT: vextractf128 $1, %ymm7, %xmm7 +; AVX-NEXT: vshufps {{.*#+}} xmm2 = xmm7[2,0],xmm2[2,3] +; AVX-NEXT: vblendps {{.*#+}} ymm7 = ymm2[0,1,2,3],ymm0[4,5,6,7] ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload -; AVX-NEXT: vunpckhps (%rsp), %ymm0, %ymm0 # 32-byte Folded Reload +; AVX-NEXT: vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload ; AVX-NEXT: # ymm0 = ymm0[2],mem[2],ymm0[3],mem[3],ymm0[6],mem[6],ymm0[7],mem[7] ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload +; AVX-NEXT: vshufps $51, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm2 # 32-byte Folded Reload +; AVX-NEXT: # ymm2 = ymm1[3,0],mem[3,0],ymm1[7,4],mem[7,4] +; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm2[2,0],ymm0[2,3],ymm2[6,4],ymm0[6,7] +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload +; AVX-NEXT: vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm2 # 32-byte Folded Reload +; AVX-NEXT: # ymm2 = ymm1[2],mem[2],ymm1[3],mem[3],ymm1[6],mem[6],ymm1[7],mem[7] +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload ; AVX-NEXT: vshufps $51, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm1 # 32-byte Folded Reload ; AVX-NEXT: # ymm1 = ymm1[3,0],mem[3,0],ymm1[7,4],mem[7,4] -; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm1[2,0],ymm0[2,3],ymm1[6,4],ymm0[6,7] -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload -; AVX-NEXT: vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm1 # 32-byte Folded Reload -; AVX-NEXT: # ymm1 = ymm1[2],mem[2],ymm1[3],mem[3],ymm1[6],mem[6],ymm1[7],mem[7] -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload -; AVX-NEXT: vshufps $51, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm2 # 32-byte Folded Reload -; AVX-NEXT: # ymm2 = ymm2[3,0],mem[3,0],ymm2[7,4],mem[7,4] -; AVX-NEXT: vextractf128 $1, %ymm1, %xmm1 -; AVX-NEXT: vshufps {{.*#+}} ymm2 = ymm2[2,0,2,3,6,4,6,7] ; AVX-NEXT: vextractf128 $1, %ymm2, %xmm2 -; AVX-NEXT: vblendps {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3] +; AVX-NEXT: vextractf128 $1, %ymm1, %xmm1 +; AVX-NEXT: vshufps {{.*#+}} xmm1 = xmm1[2,0],xmm2[2,3] ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7] ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload ; AVX-NEXT: vmovaps %ymm1, 192(%rsi) @@ -11661,14 +11593,14 @@ define void @load_i32_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload ; AVX-NEXT: vmovaps %ymm1, (%rax) ; AVX-NEXT: movq {{[0-9]+}}(%rsp), %rax -; AVX-NEXT: vmovaps %ymm5, 224(%rax) -; AVX-NEXT: vmovaps %ymm14, 192(%rax) -; AVX-NEXT: vmovaps %ymm10, 160(%rax) -; AVX-NEXT: vmovaps %ymm4, 128(%rax) +; AVX-NEXT: vmovaps %ymm7, 224(%rax) +; AVX-NEXT: vmovaps %ymm15, 192(%rax) +; AVX-NEXT: vmovaps %ymm6, 160(%rax) +; AVX-NEXT: vmovaps %ymm5, 128(%rax) ; AVX-NEXT: vmovaps %ymm3, 96(%rax) -; AVX-NEXT: vmovaps %ymm12, 64(%rax) ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm1, 32(%rax) +; AVX-NEXT: vmovaps %ymm1, 64(%rax) +; AVX-NEXT: vmovaps %ymm4, 32(%rax) ; AVX-NEXT: vmovaps %ymm0, (%rax) ; AVX-NEXT: addq $3720, %rsp # imm = 0xE88 ; AVX-NEXT: vzeroupper diff --git a/llvm/test/CodeGen/X86/vector-interleaved-store-i32-stride-6.ll b/llvm/test/CodeGen/X86/vector-interleaved-store-i32-stride-6.ll index c511c131193f..9d53325ed7c5 100644 --- a/llvm/test/CodeGen/X86/vector-interleaved-store-i32-stride-6.ll +++ b/llvm/test/CodeGen/X86/vector-interleaved-store-i32-stride-6.ll @@ -779,8 +779,8 @@ define void @store_i32_stride6_vf8(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX-NEXT: vunpcklps {{.*#+}} ymm0 = ymm7[0],ymm8[0],ymm7[1],ymm8[1],ymm7[4],ymm8[4],ymm7[5],ymm8[5] ; AVX-NEXT: vperm2f128 {{.*#+}} ymm0 = ymm0[2,3,2,3] ; AVX-NEXT: vunpcklpd {{.*#+}} ymm4 = ymm3[0],ymm2[0],ymm3[2],ymm2[2] -; AVX-NEXT: vshufps {{.*#+}} ymm4 = ymm4[0,1,2,0,4,5,6,4] ; AVX-NEXT: vextractf128 $1, %ymm4, %xmm4 +; AVX-NEXT: vshufps {{.*#+}} xmm4 = xmm4[0,1,2,0] ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm4[2,3],ymm0[4,5,6,7] ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5],ymm0[6,7] ; AVX-NEXT: vbroadcastss 16(%r9), %ymm4 @@ -1515,8 +1515,8 @@ define void @store_i32_stride6_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX-NEXT: vunpcklps {{.*#+}} ymm0 = ymm5[0],ymm13[0],ymm5[1],ymm13[1],ymm5[4],ymm13[4],ymm5[5],ymm13[5] ; AVX-NEXT: vperm2f128 {{.*#+}} ymm0 = ymm0[2,3,2,3] ; AVX-NEXT: vunpcklpd {{.*#+}} ymm1 = ymm9[0],ymm7[0],ymm9[2],ymm7[2] -; AVX-NEXT: vshufps {{.*#+}} ymm1 = ymm1[0,1,2,0,4,5,6,4] ; AVX-NEXT: vextractf128 $1, %ymm1, %xmm1 +; AVX-NEXT: vshufps {{.*#+}} xmm1 = xmm1[0,1,2,0] ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5,6,7] ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm11[4,5],ymm0[6,7] ; AVX-NEXT: vbroadcastss 48(%r9), %ymm1 @@ -1542,8 +1542,8 @@ define void @store_i32_stride6_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX-NEXT: vmovaps (%rdx), %ymm11 ; AVX-NEXT: vmovaps (%rcx), %ymm12 ; AVX-NEXT: vunpcklpd {{.*#+}} ymm10 = ymm12[0],ymm11[0],ymm12[2],ymm11[2] -; AVX-NEXT: vshufps {{.*#+}} ymm10 = ymm10[0,1,2,0,4,5,6,4] ; AVX-NEXT: vextractf128 $1, %ymm10, %xmm10 +; AVX-NEXT: vshufps {{.*#+}} xmm10 = xmm10[0,1,2,0] ; AVX-NEXT: vblendps {{.*#+}} ymm10 = ymm14[0,1],ymm10[2,3],ymm14[4,5,6,7] ; AVX-NEXT: vmovaps (%r8), %ymm14 ; AVX-NEXT: vmovups %ymm14, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill @@ -3268,8 +3268,8 @@ define void @store_i32_stride6_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX-NEXT: vunpcklps {{.*#+}} ymm0 = ymm12[0],ymm8[0],ymm12[1],ymm8[1],ymm12[4],ymm8[4],ymm12[5],ymm8[5] ; AVX-NEXT: vperm2f128 {{.*#+}} ymm0 = ymm0[2,3,2,3] ; AVX-NEXT: vunpcklpd {{.*#+}} ymm1 = ymm6[0],ymm4[0],ymm6[2],ymm4[2] -; AVX-NEXT: vshufps {{.*#+}} ymm1 = ymm1[0,1,2,0,4,5,6,4] ; AVX-NEXT: vextractf128 $1, %ymm1, %xmm1 +; AVX-NEXT: vshufps {{.*#+}} xmm1 = xmm1[0,1,2,0] ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5,6,7] ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm5[4,5],ymm0[6,7] ; AVX-NEXT: vbroadcastss 16(%r9), %ymm1 @@ -3301,8 +3301,8 @@ define void @store_i32_stride6_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX-NEXT: vmovaps 32(%rcx), %ymm13 ; AVX-NEXT: vunpcklpd {{.*#+}} ymm1 = ymm13[0],ymm8[0],ymm13[2],ymm8[2] ; AVX-NEXT: vmovups %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vshufps {{.*#+}} ymm1 = ymm1[0,1,2,0,4,5,6,4] ; AVX-NEXT: vextractf128 $1, %ymm1, %xmm1 +; AVX-NEXT: vshufps {{.*#+}} xmm1 = xmm1[0,1,2,0] ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5,6,7] ; AVX-NEXT: vmovaps 32(%r8), %ymm1 ; AVX-NEXT: vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill @@ -3339,8 +3339,8 @@ define void @store_i32_stride6_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX-NEXT: vmovaps 64(%rcx), %ymm1 ; AVX-NEXT: vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX-NEXT: vunpcklpd {{.*#+}} ymm1 = ymm1[0],ymm2[0],ymm1[2],ymm2[2] -; AVX-NEXT: vshufps {{.*#+}} ymm1 = ymm1[0,1,2,0,4,5,6,4] ; AVX-NEXT: vextractf128 $1, %ymm1, %xmm1 +; AVX-NEXT: vshufps {{.*#+}} xmm1 = xmm1[0,1,2,0] ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5,6,7] ; AVX-NEXT: vmovaps 64(%r8), %ymm1 ; AVX-NEXT: vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill @@ -3374,8 +3374,8 @@ define void @store_i32_stride6_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX-NEXT: vmovaps 96(%rcx), %ymm2 ; AVX-NEXT: vunpcklpd {{.*#+}} ymm15 = ymm2[0],ymm10[0],ymm2[2],ymm10[2] ; AVX-NEXT: vmovups %ymm10, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vshufps {{.*#+}} ymm15 = ymm15[0,1,2,0,4,5,6,4] ; AVX-NEXT: vextractf128 $1, %ymm15, %xmm15 +; AVX-NEXT: vshufps {{.*#+}} xmm15 = xmm15[0,1,2,0] ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1],ymm15[2,3],ymm1[4,5,6,7] ; AVX-NEXT: vmovaps 96(%r8), %ymm1 ; AVX-NEXT: vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill @@ -6613,8 +6613,8 @@ define void @store_i32_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX-NEXT: vperm2f128 {{.*#+}} ymm0 = ymm0[2,3,2,3] ; AVX-NEXT: vunpcklpd {{.*#+}} ymm1 = ymm15[0],ymm4[0],ymm15[2],ymm4[2] ; AVX-NEXT: vmovups %ymm15, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vshufps {{.*#+}} ymm1 = ymm1[0,1,2,0,4,5,6,4] ; AVX-NEXT: vextractf128 $1, %ymm1, %xmm1 +; AVX-NEXT: vshufps {{.*#+}} xmm1 = xmm1[0,1,2,0] ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5,6,7] ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm6[4,5],ymm0[6,7] ; AVX-NEXT: vbroadcastss 16(%r9), %ymm1 @@ -6647,8 +6647,8 @@ define void @store_i32_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX-NEXT: vmovaps 32(%rcx), %ymm1 ; AVX-NEXT: vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX-NEXT: vunpcklpd {{.*#+}} ymm1 = ymm1[0],ymm2[0],ymm1[2],ymm2[2] -; AVX-NEXT: vshufps {{.*#+}} ymm1 = ymm1[0,1,2,0,4,5,6,4] ; AVX-NEXT: vextractf128 $1, %ymm1, %xmm1 +; AVX-NEXT: vshufps {{.*#+}} xmm1 = xmm1[0,1,2,0] ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5,6,7] ; AVX-NEXT: vmovaps 32(%r8), %ymm1 ; AVX-NEXT: vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill @@ -6687,8 +6687,8 @@ define void @store_i32_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX-NEXT: vmovaps 64(%rcx), %ymm1 ; AVX-NEXT: vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX-NEXT: vunpcklpd {{.*#+}} ymm1 = ymm1[0],ymm2[0],ymm1[2],ymm2[2] -; AVX-NEXT: vshufps {{.*#+}} ymm1 = ymm1[0,1,2,0,4,5,6,4] ; AVX-NEXT: vextractf128 $1, %ymm1, %xmm1 +; AVX-NEXT: vshufps {{.*#+}} xmm1 = xmm1[0,1,2,0] ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5,6,7] ; AVX-NEXT: vmovaps 64(%r8), %ymm1 ; AVX-NEXT: vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill @@ -6727,8 +6727,8 @@ define void @store_i32_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX-NEXT: vmovaps 96(%rcx), %ymm1 ; AVX-NEXT: vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX-NEXT: vunpcklpd {{.*#+}} ymm1 = ymm1[0],ymm2[0],ymm1[2],ymm2[2] -; AVX-NEXT: vshufps {{.*#+}} ymm1 = ymm1[0,1,2,0,4,5,6,4] ; AVX-NEXT: vextractf128 $1, %ymm1, %xmm1 +; AVX-NEXT: vshufps {{.*#+}} xmm1 = xmm1[0,1,2,0] ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5,6,7] ; AVX-NEXT: vmovaps 96(%r8), %ymm1 ; AVX-NEXT: vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill @@ -6766,8 +6766,8 @@ define void @store_i32_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX-NEXT: vmovaps 128(%rcx), %ymm2 ; AVX-NEXT: vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX-NEXT: vunpcklpd {{.*#+}} ymm1 = ymm2[0],ymm1[0],ymm2[2],ymm1[2] -; AVX-NEXT: vshufps {{.*#+}} ymm1 = ymm1[0,1,2,0,4,5,6,4] ; AVX-NEXT: vextractf128 $1, %ymm1, %xmm1 +; AVX-NEXT: vshufps {{.*#+}} xmm1 = xmm1[0,1,2,0] ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5,6,7] ; AVX-NEXT: vmovaps 128(%r8), %ymm1 ; AVX-NEXT: vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill @@ -6804,8 +6804,8 @@ define void @store_i32_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX-NEXT: vmovaps 160(%rcx), %ymm8 ; AVX-NEXT: vunpcklpd {{.*#+}} ymm1 = ymm8[0],ymm1[0],ymm8[2],ymm1[2] ; AVX-NEXT: vmovups %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vshufps {{.*#+}} ymm1 = ymm1[0,1,2,0,4,5,6,4] ; AVX-NEXT: vextractf128 $1, %ymm1, %xmm1 +; AVX-NEXT: vshufps {{.*#+}} xmm1 = xmm1[0,1,2,0] ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5,6,7] ; AVX-NEXT: vmovaps 160(%r8), %ymm1 ; AVX-NEXT: vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill @@ -6842,8 +6842,8 @@ define void @store_i32_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX-NEXT: vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX-NEXT: vunpcklpd {{.*#+}} ymm1 = ymm1[0],ymm4[0],ymm1[2],ymm4[2] ; AVX-NEXT: vmovups %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vshufps {{.*#+}} ymm1 = ymm1[0,1,2,0,4,5,6,4] ; AVX-NEXT: vextractf128 $1, %ymm1, %xmm1 +; AVX-NEXT: vshufps {{.*#+}} xmm1 = xmm1[0,1,2,0] ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5,6,7] ; AVX-NEXT: vmovaps 192(%r8), %ymm1 ; AVX-NEXT: vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill @@ -6880,8 +6880,8 @@ define void @store_i32_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX-NEXT: vunpcklpd {{.*#+}} ymm13 = ymm1[0],ymm5[0],ymm1[2],ymm5[2] ; AVX-NEXT: vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX-NEXT: vmovups %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vshufps {{.*#+}} ymm13 = ymm13[0,1,2,0,4,5,6,4] ; AVX-NEXT: vextractf128 $1, %ymm13, %xmm13 +; AVX-NEXT: vshufps {{.*#+}} xmm13 = xmm13[0,1,2,0] ; AVX-NEXT: vblendps {{.*#+}} ymm13 = ymm14[0,1],ymm13[2,3],ymm14[4,5,6,7] ; AVX-NEXT: vmovaps 224(%r8), %ymm10 ; AVX-NEXT: vmovups %ymm10, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill diff --git a/llvm/test/CodeGen/X86/vector-interleaved-store-i32-stride-8.ll b/llvm/test/CodeGen/X86/vector-interleaved-store-i32-stride-8.ll index bd6c527e06b1..8d74c175f4bd 100644 --- a/llvm/test/CodeGen/X86/vector-interleaved-store-i32-stride-8.ll +++ b/llvm/test/CodeGen/X86/vector-interleaved-store-i32-stride-8.ll @@ -982,117 +982,130 @@ define void @store_i32_stride8_vf8(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; ; AVX-LABEL: store_i32_stride8_vf8: ; AVX: # %bb.0: +; AVX-NEXT: subq $120, %rsp ; AVX-NEXT: movq {{[0-9]+}}(%rsp), %rax ; AVX-NEXT: movq {{[0-9]+}}(%rsp), %r10 -; AVX-NEXT: vmovaps (%rdi), %ymm3 -; AVX-NEXT: vmovaps (%rsi), %ymm5 -; AVX-NEXT: vmovaps (%rdx), %ymm4 -; AVX-NEXT: vmovaps (%rcx), %ymm6 -; AVX-NEXT: vmovaps (%r8), %ymm7 -; AVX-NEXT: vmovaps (%r9), %ymm8 -; AVX-NEXT: vmovaps (%r10), %ymm9 -; AVX-NEXT: vmovaps (%rax), %ymm10 -; AVX-NEXT: vunpcklps {{.*#+}} ymm0 = ymm9[0],ymm10[0],ymm9[1],ymm10[1],ymm9[4],ymm10[4],ymm9[5],ymm10[5] -; AVX-NEXT: vshufps {{.*#+}} ymm1 = ymm8[1,0],ymm7[1,0],ymm8[5,4],ymm7[5,4] -; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm1[2,0],ymm0[2,3],ymm1[6,4],ymm0[6,7] -; AVX-NEXT: vunpcklps {{.*#+}} ymm1 = ymm4[0],ymm6[0],ymm4[1],ymm6[1],ymm4[4],ymm6[4],ymm4[5],ymm6[5] -; AVX-NEXT: vextractf128 $1, %ymm1, %xmm1 -; AVX-NEXT: vshufps {{.*#+}} ymm2 = ymm5[1,0],ymm3[1,0],ymm5[5,4],ymm3[5,4] -; AVX-NEXT: vshufps {{.*#+}} ymm2 = ymm2[2,0,2,3,6,4,6,7] -; AVX-NEXT: vextractf128 $1, %ymm2, %xmm2 -; AVX-NEXT: vblendps {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7] -; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vunpckhps {{.*#+}} ymm1 = ymm9[2],ymm10[2],ymm9[3],ymm10[3],ymm9[6],ymm10[6],ymm9[7],ymm10[7] -; AVX-NEXT: vshufps {{.*#+}} ymm2 = ymm8[3,0],ymm7[3,0],ymm8[7,4],ymm7[7,4] -; AVX-NEXT: vshufps {{.*#+}} ymm1 = ymm2[2,0],ymm1[2,3],ymm2[6,4],ymm1[6,7] -; AVX-NEXT: vunpckhps {{.*#+}} ymm2 = ymm4[2],ymm6[2],ymm4[3],ymm6[3],ymm4[6],ymm6[6],ymm4[7],ymm6[7] -; AVX-NEXT: vextractf128 $1, %ymm2, %xmm2 -; AVX-NEXT: vshufps {{.*#+}} ymm11 = ymm5[3,0],ymm3[3,0],ymm5[7,4],ymm3[7,4] -; AVX-NEXT: vshufps {{.*#+}} ymm11 = ymm11[2,0,2,3,6,4,6,7] -; AVX-NEXT: vextractf128 $1, %ymm11, %xmm11 -; AVX-NEXT: vblendps {{.*#+}} xmm2 = xmm11[0,1],xmm2[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm1[4,5,6,7] -; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vunpckhpd {{.*#+}} ymm2 = ymm10[1],ymm9[1],ymm10[3],ymm9[3] -; AVX-NEXT: vunpckhps {{.*#+}} ymm11 = ymm7[2],ymm8[2],ymm7[3],ymm8[3],ymm7[6],ymm8[6],ymm7[7],ymm8[7] -; AVX-NEXT: vshufps {{.*#+}} ymm2 = ymm11[0,1],ymm2[2,0],ymm11[4,5],ymm2[6,4] -; AVX-NEXT: vunpckhps {{.*#+}} ymm11 = ymm3[2],ymm5[2],ymm3[3],ymm5[3],ymm3[6],ymm5[6],ymm3[7],ymm5[7] -; AVX-NEXT: vextractf128 $1, %ymm11, %xmm11 -; AVX-NEXT: vunpckhpd {{.*#+}} ymm12 = ymm6[1],ymm4[1],ymm6[3],ymm4[3] -; AVX-NEXT: vshufps {{.*#+}} ymm12 = ymm12[0,1,2,0,4,5,6,4] -; AVX-NEXT: vextractf128 $1, %ymm12, %xmm12 -; AVX-NEXT: vblendps {{.*#+}} xmm11 = xmm11[0,1],xmm12[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm11[0,1,2,3],ymm2[4,5,6,7] -; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vunpcklpd {{.*#+}} ymm9 = ymm10[0],ymm9[0],ymm10[2],ymm9[2] -; AVX-NEXT: vunpcklps {{.*#+}} ymm7 = ymm7[0],ymm8[0],ymm7[1],ymm8[1],ymm7[4],ymm8[4],ymm7[5],ymm8[5] -; AVX-NEXT: vshufps {{.*#+}} ymm7 = ymm7[0,1],ymm9[2,0],ymm7[4,5],ymm9[6,4] -; AVX-NEXT: vunpcklps {{.*#+}} ymm3 = ymm3[0],ymm5[0],ymm3[1],ymm5[1],ymm3[4],ymm5[4],ymm3[5],ymm5[5] -; AVX-NEXT: vextractf128 $1, %ymm3, %xmm3 -; AVX-NEXT: vunpcklpd {{.*#+}} ymm4 = ymm6[0],ymm4[0],ymm6[2],ymm4[2] -; AVX-NEXT: vshufps {{.*#+}} ymm4 = ymm4[0,1,2,0,4,5,6,4] -; AVX-NEXT: vextractf128 $1, %ymm4, %xmm4 -; AVX-NEXT: vblendps {{.*#+}} xmm3 = xmm3[0,1],xmm4[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm3 = ymm3[0,1,2,3],ymm7[4,5,6,7] +; AVX-NEXT: vmovaps (%r8), %ymm9 ; AVX-NEXT: vmovaps (%rcx), %xmm4 -; AVX-NEXT: vmovaps (%rdx), %xmm5 -; AVX-NEXT: vunpcklps {{.*#+}} xmm9 = xmm5[0],xmm4[0],xmm5[1],xmm4[1] -; AVX-NEXT: vmovaps (%rsi), %xmm10 -; AVX-NEXT: vmovaps (%rdi), %xmm11 +; AVX-NEXT: vmovaps (%rdx), %xmm3 +; AVX-NEXT: vunpcklps {{.*#+}} xmm2 = xmm3[0],xmm4[0],xmm3[1],xmm4[1] +; AVX-NEXT: vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; AVX-NEXT: vmovaps %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; AVX-NEXT: vmovaps %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; AVX-NEXT: vmovaps (%rsi), %xmm11 +; AVX-NEXT: vmovaps (%rdi), %xmm12 +; AVX-NEXT: vshufps {{.*#+}} xmm0 = xmm12[1,1,1,1] +; AVX-NEXT: vblendps {{.*#+}} xmm0 = xmm0[0],xmm11[1],xmm0[2,3] +; AVX-NEXT: vblendps {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3] ; AVX-NEXT: vmovaps (%rax), %xmm6 -; AVX-NEXT: vmovaps (%r10), %xmm7 -; AVX-NEXT: vunpcklps {{.*#+}} xmm12 = xmm7[0],xmm6[0],xmm7[1],xmm6[1] -; AVX-NEXT: vinsertf128 $1, %xmm12, %ymm0, %ymm8 -; AVX-NEXT: vmovaps (%r9), %xmm13 -; AVX-NEXT: vmovaps (%r8), %xmm14 -; AVX-NEXT: vshufps {{.*#+}} xmm15 = xmm14[1,1,1,1] -; AVX-NEXT: vblendps {{.*#+}} xmm15 = xmm15[0],xmm13[1],xmm15[2,3] -; AVX-NEXT: vinsertf128 $1, %xmm15, %ymm0, %ymm15 -; AVX-NEXT: vblendps {{.*#+}} ymm8 = ymm15[0,1,2,3,4,5],ymm8[6,7] -; AVX-NEXT: vshufps {{.*#+}} xmm15 = xmm11[1,1,1,1] -; AVX-NEXT: vblendps {{.*#+}} xmm15 = xmm15[0],xmm10[1],xmm15[2,3] -; AVX-NEXT: vblendps {{.*#+}} xmm15 = xmm15[0,1],xmm9[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm8 = ymm15[0,1,2,3],ymm8[4,5,6,7] -; AVX-NEXT: vunpckhps {{.*#+}} xmm15 = xmm7[2],xmm6[2],xmm7[3],xmm6[3] -; AVX-NEXT: vinsertf128 $1, %xmm15, %ymm0, %ymm15 -; AVX-NEXT: vunpckhps {{.*#+}} xmm2 = xmm14[2],xmm13[2],xmm14[3],xmm13[3] -; AVX-NEXT: vshufps {{.*#+}} xmm1 = xmm2[2,3,2,3] +; AVX-NEXT: vmovaps (%r10), %xmm5 +; AVX-NEXT: vunpcklps {{.*#+}} xmm13 = xmm5[0],xmm6[0],xmm5[1],xmm6[1] +; AVX-NEXT: vmovaps %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; AVX-NEXT: vmovaps %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; AVX-NEXT: vinsertf128 $1, %xmm13, %ymm0, %ymm1 +; AVX-NEXT: vmovaps (%r9), %xmm14 +; AVX-NEXT: vmovaps (%r8), %xmm15 +; AVX-NEXT: vshufps {{.*#+}} xmm2 = xmm15[1,1,1,1] +; AVX-NEXT: vblendps {{.*#+}} xmm2 = xmm2[0],xmm14[1],xmm2[2,3] +; AVX-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm2 +; AVX-NEXT: vblendps {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5],ymm1[6,7] +; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7] +; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vunpckhps {{.*#+}} xmm0 = xmm3[2],xmm4[2],xmm3[3],xmm4[3] +; AVX-NEXT: vunpckhps {{.*#+}} xmm1 = xmm5[2],xmm6[2],xmm5[3],xmm6[3] ; AVX-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm1 -; AVX-NEXT: vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5],ymm15[6,7] -; AVX-NEXT: vunpckhps {{.*#+}} xmm15 = xmm5[2],xmm4[2],xmm5[3],xmm4[3] -; AVX-NEXT: vunpckhps {{.*#+}} xmm0 = xmm11[2],xmm10[2],xmm11[3],xmm10[3] -; AVX-NEXT: vunpckhpd {{.*#+}} xmm15 = xmm0[1],xmm15[1] -; AVX-NEXT: vblendps {{.*#+}} ymm1 = ymm15[0,1,2,3],ymm1[4,5,6,7] -; AVX-NEXT: vunpcklps {{.*#+}} xmm10 = xmm11[0],xmm10[0],xmm11[1],xmm10[1] -; AVX-NEXT: vmovlhps {{.*#+}} xmm9 = xmm10[0],xmm9[0] -; AVX-NEXT: vunpcklps {{.*#+}} xmm10 = xmm14[0],xmm13[0],xmm14[1],xmm13[1] +; AVX-NEXT: vunpckhps {{.*#+}} xmm2 = xmm15[2],xmm14[2],xmm15[3],xmm14[3] +; AVX-NEXT: vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; AVX-NEXT: vshufps {{.*#+}} xmm2 = xmm2[2,3,2,3] +; AVX-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm2 +; AVX-NEXT: vblendps {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5],ymm1[6,7] +; AVX-NEXT: vunpckhps {{.*#+}} xmm2 = xmm12[2],xmm11[2],xmm12[3],xmm11[3] +; AVX-NEXT: vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; AVX-NEXT: vunpckhpd {{.*#+}} xmm0 = xmm2[1],xmm0[1] +; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7] +; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vmovaps (%r9), %ymm6 +; AVX-NEXT: vmovaps (%r10), %ymm5 +; AVX-NEXT: vmovaps (%rax), %ymm4 +; AVX-NEXT: vunpcklps {{.*#+}} ymm0 = ymm5[0],ymm4[0],ymm5[1],ymm4[1],ymm5[4],ymm4[4],ymm5[5],ymm4[5] +; AVX-NEXT: vshufps {{.*#+}} ymm1 = ymm6[1,0],ymm9[1,0],ymm6[5,4],ymm9[5,4] +; AVX-NEXT: vshufps {{.*#+}} ymm8 = ymm1[2,0],ymm0[2,3],ymm1[6,4],ymm0[6,7] +; AVX-NEXT: vmovaps (%rdx), %ymm3 +; AVX-NEXT: vmovaps (%rcx), %ymm2 +; AVX-NEXT: vunpcklps {{.*#+}} ymm0 = ymm3[0],ymm2[0],ymm3[1],ymm2[1],ymm3[4],ymm2[4],ymm3[5],ymm2[5] +; AVX-NEXT: vextractf128 $1, %ymm0, %xmm7 +; AVX-NEXT: vmovaps (%rdi), %ymm1 +; AVX-NEXT: vmovaps (%rsi), %ymm0 +; AVX-NEXT: vshufps {{.*#+}} ymm10 = ymm0[1,0],ymm1[1,0],ymm0[5,4],ymm1[5,4] +; AVX-NEXT: vextractf128 $1, %ymm10, %xmm10 +; AVX-NEXT: vshufps {{.*#+}} xmm7 = xmm10[2,0],xmm7[2,3] +; AVX-NEXT: vblendps {{.*#+}} ymm7 = ymm7[0,1,2,3],ymm8[4,5,6,7] +; AVX-NEXT: vmovups %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vunpckhps {{.*#+}} ymm7 = ymm5[2],ymm4[2],ymm5[3],ymm4[3],ymm5[6],ymm4[6],ymm5[7],ymm4[7] +; AVX-NEXT: vshufps {{.*#+}} ymm8 = ymm6[3,0],ymm9[3,0],ymm6[7,4],ymm9[7,4] +; AVX-NEXT: vshufps {{.*#+}} ymm7 = ymm8[2,0],ymm7[2,3],ymm8[6,4],ymm7[6,7] +; AVX-NEXT: vunpckhps {{.*#+}} ymm8 = ymm3[2],ymm2[2],ymm3[3],ymm2[3],ymm3[6],ymm2[6],ymm3[7],ymm2[7] +; AVX-NEXT: vextractf128 $1, %ymm8, %xmm8 +; AVX-NEXT: vshufps {{.*#+}} ymm10 = ymm0[3,0],ymm1[3,0],ymm0[7,4],ymm1[7,4] +; AVX-NEXT: vextractf128 $1, %ymm10, %xmm10 +; AVX-NEXT: vshufps {{.*#+}} xmm8 = xmm10[2,0],xmm8[2,3] +; AVX-NEXT: vblendps {{.*#+}} ymm7 = ymm8[0,1,2,3],ymm7[4,5,6,7] +; AVX-NEXT: vmovups %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vunpcklps {{.*#+}} xmm7 = xmm12[0],xmm11[0],xmm12[1],xmm11[1] +; AVX-NEXT: vunpcklpd {{[-0-9]+}}(%r{{[sb]}}p), %xmm7, %xmm7 # 16-byte Folded Reload +; AVX-NEXT: # xmm7 = xmm7[0],mem[0] +; AVX-NEXT: vunpcklps {{.*#+}} xmm8 = xmm15[0],xmm14[0],xmm15[1],xmm14[1] +; AVX-NEXT: vinsertf128 $1, %xmm8, %ymm0, %ymm8 +; AVX-NEXT: vshufps {{.*#+}} xmm10 = xmm13[0,1,0,1] ; AVX-NEXT: vinsertf128 $1, %xmm10, %ymm0, %ymm10 -; AVX-NEXT: vshufps {{.*#+}} xmm11 = xmm12[0,1,0,1] -; AVX-NEXT: vinsertf128 $1, %xmm11, %ymm0, %ymm11 -; AVX-NEXT: vblendps {{.*#+}} ymm10 = ymm10[0,1,2,3,4,5],ymm11[6,7] -; AVX-NEXT: vblendps {{.*#+}} ymm9 = ymm9[0,1,2,3],ymm10[4,5,6,7] -; AVX-NEXT: vshufps {{.*#+}} xmm4 = xmm4[2,2,2,2] -; AVX-NEXT: vblendps {{.*#+}} xmm4 = xmm5[0,1,2],xmm4[3] -; AVX-NEXT: vblendps {{.*#+}} xmm0 = xmm0[0,1],xmm4[2,3] -; AVX-NEXT: vshufps {{.*#+}} xmm4 = xmm6[2,2,2,2] -; AVX-NEXT: vblendps {{.*#+}} xmm4 = xmm7[0,1,2],xmm4[3] +; AVX-NEXT: vblendps {{.*#+}} ymm8 = ymm8[0,1,2,3,4,5],ymm10[6,7] +; AVX-NEXT: vblendps {{.*#+}} ymm7 = ymm7[0,1,2,3],ymm8[4,5,6,7] +; AVX-NEXT: vunpckhpd {{.*#+}} ymm8 = ymm4[1],ymm5[1],ymm4[3],ymm5[3] +; AVX-NEXT: vunpckhps {{.*#+}} ymm10 = ymm9[2],ymm6[2],ymm9[3],ymm6[3],ymm9[6],ymm6[6],ymm9[7],ymm6[7] +; AVX-NEXT: vshufps {{.*#+}} ymm8 = ymm10[0,1],ymm8[2,0],ymm10[4,5],ymm8[6,4] +; AVX-NEXT: vunpckhpd {{.*#+}} ymm10 = ymm2[1],ymm3[1],ymm2[3],ymm3[3] +; AVX-NEXT: vextractf128 $1, %ymm10, %xmm10 +; AVX-NEXT: vunpckhps {{.*#+}} ymm11 = ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[6],ymm0[6],ymm1[7],ymm0[7] +; AVX-NEXT: vextractf128 $1, %ymm11, %xmm11 +; AVX-NEXT: vshufps {{.*#+}} xmm10 = xmm11[0,1],xmm10[2,0] +; AVX-NEXT: vblendps {{.*#+}} ymm8 = ymm10[0,1,2,3],ymm8[4,5,6,7] +; AVX-NEXT: vunpcklpd {{.*#+}} ymm4 = ymm4[0],ymm5[0],ymm4[2],ymm5[2] +; AVX-NEXT: vunpcklps {{.*#+}} ymm5 = ymm9[0],ymm6[0],ymm9[1],ymm6[1],ymm9[4],ymm6[4],ymm9[5],ymm6[5] +; AVX-NEXT: vshufps {{.*#+}} ymm4 = ymm5[0,1],ymm4[2,0],ymm5[4,5],ymm4[6,4] +; AVX-NEXT: vunpcklpd {{.*#+}} ymm2 = ymm2[0],ymm3[0],ymm2[2],ymm3[2] +; AVX-NEXT: vunpcklps {{.*#+}} ymm0 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[4],ymm0[4],ymm1[5],ymm0[5] +; AVX-NEXT: vextractf128 $1, %ymm2, %xmm1 +; AVX-NEXT: vextractf128 $1, %ymm0, %xmm0 +; AVX-NEXT: vshufps {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,0] +; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm4[4,5,6,7] +; AVX-NEXT: vpermilps $170, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload +; AVX-NEXT: # xmm1 = mem[2,2,2,2] +; AVX-NEXT: vblendps $7, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm1 # 16-byte Folded Reload +; AVX-NEXT: # xmm1 = mem[0,1,2],xmm1[3] +; AVX-NEXT: vblendps $3, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm1 # 16-byte Folded Reload +; AVX-NEXT: # xmm1 = mem[0,1],xmm1[2,3] +; AVX-NEXT: vpermilps $170, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload +; AVX-NEXT: # xmm2 = mem[2,2,2,2] +; AVX-NEXT: vblendps $7, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2, %xmm2 # 16-byte Folded Reload +; AVX-NEXT: # xmm2 = mem[0,1,2],xmm2[3] +; AVX-NEXT: vinsertf128 $1, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm3 # 16-byte Folded Reload ; AVX-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm2 -; AVX-NEXT: vinsertf128 $1, %xmm4, %ymm0, %ymm4 -; AVX-NEXT: vblendps {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm4[6,7] -; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm2[4,5,6,7] +; AVX-NEXT: vblendps {{.*#+}} ymm2 = ymm3[0,1,2,3,4,5],ymm2[6,7] +; AVX-NEXT: vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7] ; AVX-NEXT: movq {{[0-9]+}}(%rsp), %rax -; AVX-NEXT: vmovaps %ymm0, 64(%rax) -; AVX-NEXT: vmovaps %ymm9, (%rax) -; AVX-NEXT: vmovaps %ymm1, 96(%rax) -; AVX-NEXT: vmovaps %ymm8, 32(%rax) -; AVX-NEXT: vmovaps %ymm3, 128(%rax) -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm0, 192(%rax) +; AVX-NEXT: vmovaps %ymm1, 64(%rax) +; AVX-NEXT: vmovaps %ymm0, 128(%rax) +; AVX-NEXT: vmovaps %ymm8, 192(%rax) +; AVX-NEXT: vmovaps %ymm7, (%rax) ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload ; AVX-NEXT: vmovaps %ymm0, 224(%rax) ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload ; AVX-NEXT: vmovaps %ymm0, 160(%rax) +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm0, 96(%rax) +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm0, 32(%rax) +; AVX-NEXT: addq $120, %rsp ; AVX-NEXT: vzeroupper ; AVX-NEXT: retq ; @@ -2041,231 +2054,221 @@ define void @store_i32_stride8_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX-NEXT: subq $264, %rsp # imm = 0x108 ; AVX-NEXT: movq {{[0-9]+}}(%rsp), %rax ; AVX-NEXT: movq {{[0-9]+}}(%rsp), %r10 -; AVX-NEXT: vmovaps (%rdi), %ymm7 -; AVX-NEXT: vmovaps 32(%rdi), %ymm0 -; AVX-NEXT: vmovaps (%rsi), %ymm8 -; AVX-NEXT: vmovaps 32(%rsi), %ymm1 -; AVX-NEXT: vmovaps (%rdx), %ymm9 -; AVX-NEXT: vmovaps 32(%rdx), %ymm3 -; AVX-NEXT: vmovaps 32(%rcx), %ymm4 -; AVX-NEXT: vmovaps 32(%r8), %ymm5 -; AVX-NEXT: vmovaps 32(%r9), %ymm6 -; AVX-NEXT: vmovaps 32(%r10), %ymm11 -; AVX-NEXT: vmovaps 32(%rax), %ymm12 -; AVX-NEXT: vunpckhpd {{.*#+}} ymm2 = ymm12[1],ymm11[1],ymm12[3],ymm11[3] -; AVX-NEXT: vunpckhps {{.*#+}} ymm10 = ymm5[2],ymm6[2],ymm5[3],ymm6[3],ymm5[6],ymm6[6],ymm5[7],ymm6[7] -; AVX-NEXT: vshufps {{.*#+}} ymm2 = ymm10[0,1],ymm2[2,0],ymm10[4,5],ymm2[6,4] -; AVX-NEXT: vunpckhps {{.*#+}} ymm10 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7] -; AVX-NEXT: vextractf128 $1, %ymm10, %xmm10 -; AVX-NEXT: vunpckhpd {{.*#+}} ymm13 = ymm4[1],ymm3[1],ymm4[3],ymm3[3] -; AVX-NEXT: vshufps {{.*#+}} ymm13 = ymm13[0,1,2,0,4,5,6,4] -; AVX-NEXT: vextractf128 $1, %ymm13, %xmm13 -; AVX-NEXT: vblendps {{.*#+}} xmm10 = xmm10[0,1],xmm13[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm2 = ymm10[0,1,2,3],ymm2[4,5,6,7] -; AVX-NEXT: vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vunpckhps {{.*#+}} ymm2 = ymm11[2],ymm12[2],ymm11[3],ymm12[3],ymm11[6],ymm12[6],ymm11[7],ymm12[7] -; AVX-NEXT: vshufps {{.*#+}} ymm10 = ymm6[3,0],ymm5[3,0],ymm6[7,4],ymm5[7,4] -; AVX-NEXT: vshufps {{.*#+}} ymm2 = ymm10[2,0],ymm2[2,3],ymm10[6,4],ymm2[6,7] -; AVX-NEXT: vunpckhps {{.*#+}} ymm10 = ymm3[2],ymm4[2],ymm3[3],ymm4[3],ymm3[6],ymm4[6],ymm3[7],ymm4[7] -; AVX-NEXT: vextractf128 $1, %ymm10, %xmm10 -; AVX-NEXT: vshufps {{.*#+}} ymm13 = ymm1[3,0],ymm0[3,0],ymm1[7,4],ymm0[7,4] -; AVX-NEXT: vshufps {{.*#+}} ymm13 = ymm13[2,0,2,3,6,4,6,7] -; AVX-NEXT: vextractf128 $1, %ymm13, %xmm13 -; AVX-NEXT: vblendps {{.*#+}} xmm10 = xmm13[0,1],xmm10[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm2 = ymm10[0,1,2,3],ymm2[4,5,6,7] -; AVX-NEXT: vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vunpcklpd {{.*#+}} ymm2 = ymm12[0],ymm11[0],ymm12[2],ymm11[2] -; AVX-NEXT: vunpcklps {{.*#+}} ymm10 = ymm5[0],ymm6[0],ymm5[1],ymm6[1],ymm5[4],ymm6[4],ymm5[5],ymm6[5] -; AVX-NEXT: vshufps {{.*#+}} ymm10 = ymm10[0,1],ymm2[2,0],ymm10[4,5],ymm2[6,4] -; AVX-NEXT: vunpcklps {{.*#+}} ymm2 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5] -; AVX-NEXT: vextractf128 $1, %ymm2, %xmm2 -; AVX-NEXT: vunpcklpd {{.*#+}} ymm13 = ymm4[0],ymm3[0],ymm4[2],ymm3[2] -; AVX-NEXT: vshufps {{.*#+}} ymm13 = ymm13[0,1,2,0,4,5,6,4] -; AVX-NEXT: vextractf128 $1, %ymm13, %xmm13 -; AVX-NEXT: vblendps {{.*#+}} xmm13 = xmm2[0,1],xmm13[2,3] -; AVX-NEXT: vmovaps (%rcx), %ymm2 -; AVX-NEXT: vblendps {{.*#+}} ymm10 = ymm13[0,1,2,3],ymm10[4,5,6,7] -; AVX-NEXT: vmovups %ymm10, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vmovaps (%r8), %ymm10 -; AVX-NEXT: vunpcklps {{.*#+}} ymm13 = ymm11[0],ymm12[0],ymm11[1],ymm12[1],ymm11[4],ymm12[4],ymm11[5],ymm12[5] -; AVX-NEXT: vmovaps (%r9), %ymm11 -; AVX-NEXT: vshufps {{.*#+}} ymm5 = ymm6[1,0],ymm5[1,0],ymm6[5,4],ymm5[5,4] -; AVX-NEXT: vmovaps (%r10), %ymm12 -; AVX-NEXT: vshufps {{.*#+}} ymm6 = ymm5[2,0],ymm13[2,3],ymm5[6,4],ymm13[6,7] -; AVX-NEXT: vmovaps (%rax), %ymm5 -; AVX-NEXT: vunpcklps {{.*#+}} ymm3 = ymm3[0],ymm4[0],ymm3[1],ymm4[1],ymm3[4],ymm4[4],ymm3[5],ymm4[5] -; AVX-NEXT: vextractf128 $1, %ymm3, %xmm3 -; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm1[1,0],ymm0[1,0],ymm1[5,4],ymm0[5,4] -; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm0[2,0,2,3,6,4,6,7] -; AVX-NEXT: vextractf128 $1, %ymm0, %xmm0 -; AVX-NEXT: vblendps {{.*#+}} xmm0 = xmm0[0,1],xmm3[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm6[4,5,6,7] -; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vunpckhpd {{.*#+}} ymm0 = ymm5[1],ymm12[1],ymm5[3],ymm12[3] -; AVX-NEXT: vunpckhps {{.*#+}} ymm1 = ymm10[2],ymm11[2],ymm10[3],ymm11[3],ymm10[6],ymm11[6],ymm10[7],ymm11[7] -; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,0],ymm1[4,5],ymm0[6,4] -; AVX-NEXT: vunpckhps {{.*#+}} ymm1 = ymm7[2],ymm8[2],ymm7[3],ymm8[3],ymm7[6],ymm8[6],ymm7[7],ymm8[7] -; AVX-NEXT: vextractf128 $1, %ymm1, %xmm1 -; AVX-NEXT: vunpckhpd {{.*#+}} ymm3 = ymm2[1],ymm9[1],ymm2[3],ymm9[3] -; AVX-NEXT: vshufps {{.*#+}} ymm3 = ymm3[0,1,2,0,4,5,6,4] -; AVX-NEXT: vextractf128 $1, %ymm3, %xmm3 -; AVX-NEXT: vblendps {{.*#+}} xmm1 = xmm1[0,1],xmm3[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7] -; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vunpckhps {{.*#+}} ymm0 = ymm12[2],ymm5[2],ymm12[3],ymm5[3],ymm12[6],ymm5[6],ymm12[7],ymm5[7] -; AVX-NEXT: vshufps {{.*#+}} ymm1 = ymm11[3,0],ymm10[3,0],ymm11[7,4],ymm10[7,4] -; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm1[2,0],ymm0[2,3],ymm1[6,4],ymm0[6,7] -; AVX-NEXT: vunpckhps {{.*#+}} ymm1 = ymm9[2],ymm2[2],ymm9[3],ymm2[3],ymm9[6],ymm2[6],ymm9[7],ymm2[7] -; AVX-NEXT: vextractf128 $1, %ymm1, %xmm1 -; AVX-NEXT: vshufps {{.*#+}} ymm3 = ymm8[3,0],ymm7[3,0],ymm8[7,4],ymm7[7,4] -; AVX-NEXT: vshufps {{.*#+}} ymm3 = ymm3[2,0,2,3,6,4,6,7] -; AVX-NEXT: vextractf128 $1, %ymm3, %xmm3 -; AVX-NEXT: vblendps {{.*#+}} xmm1 = xmm3[0,1],xmm1[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7] -; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vunpcklpd {{.*#+}} ymm0 = ymm5[0],ymm12[0],ymm5[2],ymm12[2] -; AVX-NEXT: vunpcklps {{.*#+}} ymm1 = ymm10[0],ymm11[0],ymm10[1],ymm11[1],ymm10[4],ymm11[4],ymm10[5],ymm11[5] -; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,0],ymm1[4,5],ymm0[6,4] -; AVX-NEXT: vunpcklps {{.*#+}} ymm1 = ymm7[0],ymm8[0],ymm7[1],ymm8[1],ymm7[4],ymm8[4],ymm7[5],ymm8[5] -; AVX-NEXT: vextractf128 $1, %ymm1, %xmm1 -; AVX-NEXT: vunpcklpd {{.*#+}} ymm3 = ymm2[0],ymm9[0],ymm2[2],ymm9[2] -; AVX-NEXT: vshufps {{.*#+}} ymm3 = ymm3[0,1,2,0,4,5,6,4] -; AVX-NEXT: vextractf128 $1, %ymm3, %xmm3 -; AVX-NEXT: vblendps {{.*#+}} xmm1 = xmm1[0,1],xmm3[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7] -; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vunpcklps {{.*#+}} ymm0 = ymm12[0],ymm5[0],ymm12[1],ymm5[1],ymm12[4],ymm5[4],ymm12[5],ymm5[5] -; AVX-NEXT: vshufps {{.*#+}} ymm1 = ymm11[1,0],ymm10[1,0],ymm11[5,4],ymm10[5,4] -; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm1[2,0],ymm0[2,3],ymm1[6,4],ymm0[6,7] -; AVX-NEXT: vunpcklps {{.*#+}} ymm1 = ymm9[0],ymm2[0],ymm9[1],ymm2[1],ymm9[4],ymm2[4],ymm9[5],ymm2[5] -; AVX-NEXT: vextractf128 $1, %ymm1, %xmm1 -; AVX-NEXT: vshufps {{.*#+}} ymm2 = ymm8[1,0],ymm7[1,0],ymm8[5,4],ymm7[5,4] -; AVX-NEXT: vshufps {{.*#+}} ymm2 = ymm2[2,0,2,3,6,4,6,7] -; AVX-NEXT: vextractf128 $1, %ymm2, %xmm2 -; AVX-NEXT: vblendps {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7] -; AVX-NEXT: vmovups %ymm0, (%rsp) # 32-byte Spill -; AVX-NEXT: vmovaps (%rsi), %xmm4 -; AVX-NEXT: vmovaps %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; AVX-NEXT: vmovaps (%rax), %xmm1 +; AVX-NEXT: vmovaps 32(%r8), %ymm0 +; AVX-NEXT: vmovaps 32(%r9), %ymm5 +; AVX-NEXT: vmovaps 32(%r10), %ymm13 +; AVX-NEXT: vmovaps 32(%rax), %ymm14 +; AVX-NEXT: vmovaps (%rcx), %xmm2 +; AVX-NEXT: vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; AVX-NEXT: vmovaps (%rdx), %xmm1 ; AVX-NEXT: vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; AVX-NEXT: vmovaps (%r10), %xmm0 -; AVX-NEXT: vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; AVX-NEXT: vunpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] -; AVX-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm1 +; AVX-NEXT: vunpcklps {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1] +; AVX-NEXT: vmovaps (%rsi), %xmm6 +; AVX-NEXT: vmovaps %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; AVX-NEXT: vmovaps (%rdi), %xmm7 +; AVX-NEXT: vmovaps %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; AVX-NEXT: vmovaps (%rax), %xmm3 +; AVX-NEXT: vmovaps %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; AVX-NEXT: vmovaps (%r10), %xmm2 +; AVX-NEXT: vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; AVX-NEXT: vunpcklps {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1] +; AVX-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm3 ; AVX-NEXT: vmovaps (%r9), %xmm15 -; AVX-NEXT: vmovaps (%r8), %xmm13 -; AVX-NEXT: vshufps {{.*#+}} xmm2 = xmm13[1,1,1,1] -; AVX-NEXT: vblendps {{.*#+}} xmm2 = xmm2[0],xmm15[1],xmm2[2,3] +; AVX-NEXT: vmovaps (%r8), %xmm11 +; AVX-NEXT: vshufps {{.*#+}} xmm4 = xmm11[1,1,1,1] +; AVX-NEXT: vblendps {{.*#+}} xmm4 = xmm4[0],xmm15[1],xmm4[2,3] +; AVX-NEXT: vinsertf128 $1, %xmm4, %ymm0, %ymm4 +; AVX-NEXT: vblendps {{.*#+}} ymm3 = ymm4[0,1,2,3,4,5],ymm3[6,7] +; AVX-NEXT: vshufps {{.*#+}} xmm4 = xmm7[1,1,1,1] +; AVX-NEXT: vblendps {{.*#+}} xmm4 = xmm4[0],xmm6[1],xmm4[2,3] +; AVX-NEXT: vblendps {{.*#+}} xmm4 = xmm4[0,1],xmm1[2,3] +; AVX-NEXT: vblendps {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7] +; AVX-NEXT: vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vunpcklps {{.*#+}} xmm3 = xmm7[0],xmm6[0],xmm7[1],xmm6[1] +; AVX-NEXT: vmovlhps {{.*#+}} xmm1 = xmm3[0],xmm1[0] +; AVX-NEXT: vunpcklps {{.*#+}} xmm3 = xmm11[0],xmm15[0],xmm11[1],xmm15[1] +; AVX-NEXT: vinsertf128 $1, %xmm3, %ymm0, %ymm3 +; AVX-NEXT: vshufps {{.*#+}} xmm2 = xmm2[0,1,0,1] ; AVX-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm2 -; AVX-NEXT: vblendps {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5],ymm1[6,7] -; AVX-NEXT: vmovaps (%rdi), %xmm11 -; AVX-NEXT: vshufps {{.*#+}} xmm2 = xmm11[1,1,1,1] -; AVX-NEXT: vblendps {{.*#+}} xmm2 = xmm2[0],xmm4[1],xmm2[2,3] -; AVX-NEXT: vmovaps (%rcx), %xmm3 -; AVX-NEXT: vmovaps %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; AVX-NEXT: vmovaps (%rdx), %xmm14 -; AVX-NEXT: vunpcklps {{.*#+}} xmm3 = xmm14[0],xmm3[0],xmm14[1],xmm3[1] -; AVX-NEXT: vblendps {{.*#+}} xmm2 = xmm2[0,1],xmm3[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7] +; AVX-NEXT: vblendps {{.*#+}} ymm2 = ymm3[0,1,2,3,4,5],ymm2[6,7] +; AVX-NEXT: vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7] ; AVX-NEXT: vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vunpcklps {{.*#+}} xmm1 = xmm11[0],xmm4[0],xmm11[1],xmm4[1] -; AVX-NEXT: vmovlhps {{.*#+}} xmm1 = xmm1[0],xmm3[0] -; AVX-NEXT: vunpcklps {{.*#+}} xmm2 = xmm13[0],xmm15[0],xmm13[1],xmm15[1] -; AVX-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm2 -; AVX-NEXT: vshufps {{.*#+}} xmm0 = xmm0[0,1,0,1] -; AVX-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0 -; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm2[0,1,2,3,4,5],ymm0[6,7] +; AVX-NEXT: vunpckhpd {{.*#+}} ymm1 = ymm14[1],ymm13[1],ymm14[3],ymm13[3] +; AVX-NEXT: vunpckhps {{.*#+}} ymm2 = ymm0[2],ymm5[2],ymm0[3],ymm5[3],ymm0[6],ymm5[6],ymm0[7],ymm5[7] +; AVX-NEXT: vshufps {{.*#+}} ymm3 = ymm2[0,1],ymm1[2,0],ymm2[4,5],ymm1[6,4] +; AVX-NEXT: vmovaps 32(%rdx), %ymm1 +; AVX-NEXT: vmovaps 32(%rcx), %ymm2 +; AVX-NEXT: vunpckhpd {{.*#+}} ymm4 = ymm2[1],ymm1[1],ymm2[3],ymm1[3] +; AVX-NEXT: vextractf128 $1, %ymm4, %xmm4 +; AVX-NEXT: vmovaps 32(%rdi), %ymm6 +; AVX-NEXT: vmovaps 32(%rsi), %ymm7 +; AVX-NEXT: vunpckhps {{.*#+}} ymm8 = ymm6[2],ymm7[2],ymm6[3],ymm7[3],ymm6[6],ymm7[6],ymm6[7],ymm7[7] +; AVX-NEXT: vextractf128 $1, %ymm8, %xmm8 +; AVX-NEXT: vshufps {{.*#+}} xmm4 = xmm8[0,1],xmm4[2,0] +; AVX-NEXT: vblendps {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7] +; AVX-NEXT: vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vunpckhps {{.*#+}} ymm3 = ymm13[2],ymm14[2],ymm13[3],ymm14[3],ymm13[6],ymm14[6],ymm13[7],ymm14[7] +; AVX-NEXT: vshufps {{.*#+}} ymm4 = ymm5[3,0],ymm0[3,0],ymm5[7,4],ymm0[7,4] +; AVX-NEXT: vshufps {{.*#+}} ymm3 = ymm4[2,0],ymm3[2,3],ymm4[6,4],ymm3[6,7] +; AVX-NEXT: vunpckhps {{.*#+}} ymm4 = ymm1[2],ymm2[2],ymm1[3],ymm2[3],ymm1[6],ymm2[6],ymm1[7],ymm2[7] +; AVX-NEXT: vextractf128 $1, %ymm4, %xmm4 +; AVX-NEXT: vshufps {{.*#+}} ymm8 = ymm7[3,0],ymm6[3,0],ymm7[7,4],ymm6[7,4] +; AVX-NEXT: vextractf128 $1, %ymm8, %xmm8 +; AVX-NEXT: vshufps {{.*#+}} xmm4 = xmm8[2,0],xmm4[2,3] +; AVX-NEXT: vblendps {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7] +; AVX-NEXT: vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vunpcklpd {{.*#+}} ymm3 = ymm14[0],ymm13[0],ymm14[2],ymm13[2] +; AVX-NEXT: vunpcklps {{.*#+}} ymm4 = ymm0[0],ymm5[0],ymm0[1],ymm5[1],ymm0[4],ymm5[4],ymm0[5],ymm5[5] +; AVX-NEXT: vshufps {{.*#+}} ymm3 = ymm4[0,1],ymm3[2,0],ymm4[4,5],ymm3[6,4] +; AVX-NEXT: vunpcklpd {{.*#+}} ymm4 = ymm2[0],ymm1[0],ymm2[2],ymm1[2] +; AVX-NEXT: vextractf128 $1, %ymm4, %xmm4 +; AVX-NEXT: vunpcklps {{.*#+}} ymm8 = ymm6[0],ymm7[0],ymm6[1],ymm7[1],ymm6[4],ymm7[4],ymm6[5],ymm7[5] +; AVX-NEXT: vextractf128 $1, %ymm8, %xmm8 +; AVX-NEXT: vshufps {{.*#+}} xmm4 = xmm8[0,1],xmm4[2,0] +; AVX-NEXT: vblendps {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7] +; AVX-NEXT: vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vunpcklps {{.*#+}} ymm3 = ymm13[0],ymm14[0],ymm13[1],ymm14[1],ymm13[4],ymm14[4],ymm13[5],ymm14[5] +; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm5[1,0],ymm0[1,0],ymm5[5,4],ymm0[5,4] +; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm0[2,0],ymm3[2,3],ymm0[6,4],ymm3[6,7] +; AVX-NEXT: vunpcklps {{.*#+}} ymm1 = ymm1[0],ymm2[0],ymm1[1],ymm2[1],ymm1[4],ymm2[4],ymm1[5],ymm2[5] +; AVX-NEXT: vshufps {{.*#+}} ymm2 = ymm7[1,0],ymm6[1,0],ymm7[5,4],ymm6[5,4] +; AVX-NEXT: vextractf128 $1, %ymm1, %xmm1 +; AVX-NEXT: vextractf128 $1, %ymm2, %xmm2 +; AVX-NEXT: vshufps {{.*#+}} xmm1 = xmm2[2,0],xmm1[2,3] +; AVX-NEXT: vmovaps 32(%rax), %xmm13 ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7] ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vmovaps 32(%rax), %xmm9 -; AVX-NEXT: vmovaps 32(%r10), %xmm7 -; AVX-NEXT: vshufps {{.*#+}} xmm0 = xmm9[2,2,2,2] -; AVX-NEXT: vblendps {{.*#+}} xmm0 = xmm7[0,1,2],xmm0[3] -; AVX-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm6 -; AVX-NEXT: vmovaps 32(%r9), %xmm8 -; AVX-NEXT: vmovaps 32(%r8), %xmm5 -; AVX-NEXT: vunpckhps {{.*#+}} xmm4 = xmm5[2],xmm8[2],xmm5[3],xmm8[3] -; AVX-NEXT: vinsertf128 $1, %xmm4, %ymm0, %ymm10 -; AVX-NEXT: vblendps {{.*#+}} ymm10 = ymm10[0,1,2,3,4,5],ymm6[6,7] +; AVX-NEXT: vmovaps 32(%r10), %xmm1 +; AVX-NEXT: vshufps {{.*#+}} xmm0 = xmm13[2,2,2,2] +; AVX-NEXT: vblendps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[3] +; AVX-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm3 +; AVX-NEXT: vmovaps 32(%r9), %xmm2 +; AVX-NEXT: vmovaps 32(%r8), %xmm0 +; AVX-NEXT: vunpckhps {{.*#+}} xmm4 = xmm0[2],xmm2[2],xmm0[3],xmm2[3] +; AVX-NEXT: vinsertf128 $1, %xmm4, %ymm0, %ymm6 +; AVX-NEXT: vblendps {{.*#+}} ymm3 = ymm6[0,1,2,3,4,5],ymm3[6,7] ; AVX-NEXT: vmovaps 32(%rcx), %xmm6 -; AVX-NEXT: vmovaps 32(%rdx), %xmm1 -; AVX-NEXT: vshufps {{.*#+}} xmm12 = xmm6[2,2,2,2] -; AVX-NEXT: vblendps {{.*#+}} xmm12 = xmm1[0,1,2],xmm12[3] -; AVX-NEXT: vmovaps 32(%rsi), %xmm3 -; AVX-NEXT: vmovaps 32(%rdi), %xmm2 -; AVX-NEXT: vunpckhps {{.*#+}} xmm0 = xmm2[2],xmm3[2],xmm2[3],xmm3[3] -; AVX-NEXT: vblendps {{.*#+}} xmm12 = xmm0[0,1],xmm12[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm12 = ymm12[0,1,2,3],ymm10[4,5,6,7] -; AVX-NEXT: vunpckhps {{.*#+}} xmm10 = xmm1[2],xmm6[2],xmm1[3],xmm6[3] -; AVX-NEXT: vunpckhpd {{.*#+}} xmm0 = xmm0[1],xmm10[1] -; AVX-NEXT: vunpckhps {{.*#+}} xmm10 = xmm7[2],xmm9[2],xmm7[3],xmm9[3] -; AVX-NEXT: vinsertf128 $1, %xmm10, %ymm0, %ymm10 +; AVX-NEXT: vmovaps 32(%rdx), %xmm7 +; AVX-NEXT: vshufps {{.*#+}} xmm8 = xmm6[2,2,2,2] +; AVX-NEXT: vblendps {{.*#+}} xmm8 = xmm7[0,1,2],xmm8[3] +; AVX-NEXT: vmovaps 32(%rsi), %xmm12 +; AVX-NEXT: vmovaps 32(%rdi), %xmm5 +; AVX-NEXT: vunpckhps {{.*#+}} xmm14 = xmm5[2],xmm12[2],xmm5[3],xmm12[3] +; AVX-NEXT: vblendps {{.*#+}} xmm8 = xmm14[0,1],xmm8[2,3] +; AVX-NEXT: vblendps {{.*#+}} ymm3 = ymm8[0,1,2,3],ymm3[4,5,6,7] +; AVX-NEXT: vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vunpckhps {{.*#+}} xmm3 = xmm7[2],xmm6[2],xmm7[3],xmm6[3] +; AVX-NEXT: vunpckhpd {{.*#+}} xmm3 = xmm14[1],xmm3[1] +; AVX-NEXT: vunpckhps {{.*#+}} xmm8 = xmm1[2],xmm13[2],xmm1[3],xmm13[3] +; AVX-NEXT: vinsertf128 $1, %xmm8, %ymm0, %ymm8 ; AVX-NEXT: vshufps {{.*#+}} xmm4 = xmm4[2,3,2,3] ; AVX-NEXT: vinsertf128 $1, %xmm4, %ymm0, %ymm4 -; AVX-NEXT: vblendps {{.*#+}} ymm4 = ymm4[0,1,2,3,4,5],ymm10[6,7] -; AVX-NEXT: vblendps {{.*#+}} ymm10 = ymm0[0,1,2,3],ymm4[4,5,6,7] -; AVX-NEXT: vunpcklps {{.*#+}} xmm1 = xmm1[0],xmm6[0],xmm1[1],xmm6[1] -; AVX-NEXT: vunpcklps {{.*#+}} xmm4 = xmm7[0],xmm9[0],xmm7[1],xmm9[1] -; AVX-NEXT: vunpcklps {{.*#+}} xmm0 = xmm5[0],xmm8[0],xmm5[1],xmm8[1] -; AVX-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0 -; AVX-NEXT: vshufps {{.*#+}} xmm6 = xmm4[0,1,0,1] +; AVX-NEXT: vblendps {{.*#+}} ymm4 = ymm4[0,1,2,3,4,5],ymm8[6,7] +; AVX-NEXT: vblendps {{.*#+}} ymm3 = ymm3[0,1,2,3],ymm4[4,5,6,7] +; AVX-NEXT: vmovups %ymm3, (%rsp) # 32-byte Spill +; AVX-NEXT: vunpcklps {{.*#+}} xmm3 = xmm7[0],xmm6[0],xmm7[1],xmm6[1] +; AVX-NEXT: vunpcklps {{.*#+}} xmm1 = xmm1[0],xmm13[0],xmm1[1],xmm13[1] +; AVX-NEXT: vunpcklps {{.*#+}} xmm4 = xmm0[0],xmm2[0],xmm0[1],xmm2[1] +; AVX-NEXT: vinsertf128 $1, %xmm4, %ymm0, %ymm4 +; AVX-NEXT: vshufps {{.*#+}} xmm6 = xmm1[0,1,0,1] ; AVX-NEXT: vinsertf128 $1, %xmm6, %ymm0, %ymm6 -; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm6[6,7] -; AVX-NEXT: vunpcklps {{.*#+}} xmm6 = xmm2[0],xmm3[0],xmm2[1],xmm3[1] -; AVX-NEXT: vmovlhps {{.*#+}} xmm6 = xmm6[0],xmm1[0] -; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm6[0,1,2,3],ymm0[4,5,6,7] -; AVX-NEXT: vshufps {{.*#+}} xmm2 = xmm2[1,1,1,1] -; AVX-NEXT: vblendps {{.*#+}} xmm2 = xmm2[0],xmm3[1],xmm2[2,3] -; AVX-NEXT: vblendps {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3] -; AVX-NEXT: vshufps {{.*#+}} xmm2 = xmm5[1,1,1,1] -; AVX-NEXT: vblendps {{.*#+}} xmm2 = xmm2[0],xmm8[1],xmm2[2,3] -; AVX-NEXT: vinsertf128 $1, %xmm4, %ymm0, %ymm3 -; AVX-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm2 -; AVX-NEXT: vblendps {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm3[6,7] -; AVX-NEXT: vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7] -; AVX-NEXT: vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %xmm11, %xmm2 # 16-byte Folded Reload -; AVX-NEXT: # xmm2 = xmm11[2],mem[2],xmm11[3],mem[3] -; AVX-NEXT: vunpckhps {{.*#+}} xmm3 = xmm13[2],xmm15[2],xmm13[3],xmm15[3] -; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload -; AVX-NEXT: vshufps {{.*#+}} xmm4 = xmm7[2,2,2,2] +; AVX-NEXT: vblendps {{.*#+}} ymm4 = ymm4[0,1,2,3,4,5],ymm6[6,7] +; AVX-NEXT: vunpcklps {{.*#+}} xmm6 = xmm5[0],xmm12[0],xmm5[1],xmm12[1] +; AVX-NEXT: vmovlhps {{.*#+}} xmm6 = xmm6[0],xmm3[0] +; AVX-NEXT: vblendps {{.*#+}} ymm4 = ymm6[0,1,2,3],ymm4[4,5,6,7] +; AVX-NEXT: vmovups %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vshufps {{.*#+}} xmm4 = xmm5[1,1,1,1] +; AVX-NEXT: vblendps {{.*#+}} xmm4 = xmm4[0],xmm12[1],xmm4[2,3] +; AVX-NEXT: vblendps {{.*#+}} xmm3 = xmm4[0,1],xmm3[2,3] +; AVX-NEXT: vshufps {{.*#+}} xmm0 = xmm0[1,1,1,1] +; AVX-NEXT: vblendps {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2,3] +; AVX-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm1 +; AVX-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0 +; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7] +; AVX-NEXT: vblendps {{.*#+}} ymm12 = ymm3[0,1,2,3],ymm0[4,5,6,7] +; AVX-NEXT: vmovaps (%r8), %ymm8 +; AVX-NEXT: vmovaps (%r9), %ymm6 +; AVX-NEXT: vmovaps (%r10), %ymm5 +; AVX-NEXT: vmovaps (%rax), %ymm4 +; AVX-NEXT: vunpckhpd {{.*#+}} ymm0 = ymm4[1],ymm5[1],ymm4[3],ymm5[3] +; AVX-NEXT: vunpckhps {{.*#+}} ymm1 = ymm8[2],ymm6[2],ymm8[3],ymm6[3],ymm8[6],ymm6[6],ymm8[7],ymm6[7] +; AVX-NEXT: vshufps {{.*#+}} ymm9 = ymm1[0,1],ymm0[2,0],ymm1[4,5],ymm0[6,4] +; AVX-NEXT: vmovaps (%rdx), %ymm3 +; AVX-NEXT: vmovaps (%rcx), %ymm2 +; AVX-NEXT: vunpckhpd {{.*#+}} ymm7 = ymm2[1],ymm3[1],ymm2[3],ymm3[3] +; AVX-NEXT: vextractf128 $1, %ymm7, %xmm7 +; AVX-NEXT: vmovaps (%rdi), %ymm1 +; AVX-NEXT: vmovaps (%rsi), %ymm0 +; AVX-NEXT: vunpckhps {{.*#+}} ymm10 = ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[6],ymm0[6],ymm1[7],ymm0[7] +; AVX-NEXT: vextractf128 $1, %ymm10, %xmm10 +; AVX-NEXT: vshufps {{.*#+}} xmm7 = xmm10[0,1],xmm7[2,0] +; AVX-NEXT: vblendps {{.*#+}} ymm10 = ymm7[0,1,2,3],ymm9[4,5,6,7] +; AVX-NEXT: vunpckhps {{.*#+}} ymm7 = ymm5[2],ymm4[2],ymm5[3],ymm4[3],ymm5[6],ymm4[6],ymm5[7],ymm4[7] +; AVX-NEXT: vshufps {{.*#+}} ymm9 = ymm6[3,0],ymm8[3,0],ymm6[7,4],ymm8[7,4] +; AVX-NEXT: vshufps {{.*#+}} ymm7 = ymm9[2,0],ymm7[2,3],ymm9[6,4],ymm7[6,7] +; AVX-NEXT: vunpckhps {{.*#+}} ymm9 = ymm3[2],ymm2[2],ymm3[3],ymm2[3],ymm3[6],ymm2[6],ymm3[7],ymm2[7] +; AVX-NEXT: vextractf128 $1, %ymm9, %xmm9 +; AVX-NEXT: vshufps {{.*#+}} ymm14 = ymm0[3,0],ymm1[3,0],ymm0[7,4],ymm1[7,4] +; AVX-NEXT: vextractf128 $1, %ymm14, %xmm14 +; AVX-NEXT: vshufps {{.*#+}} xmm9 = xmm14[2,0],xmm9[2,3] +; AVX-NEXT: vblendps {{.*#+}} ymm9 = ymm9[0,1,2,3],ymm7[4,5,6,7] +; AVX-NEXT: vunpcklpd {{.*#+}} ymm7 = ymm4[0],ymm5[0],ymm4[2],ymm5[2] +; AVX-NEXT: vunpcklps {{.*#+}} ymm14 = ymm8[0],ymm6[0],ymm8[1],ymm6[1],ymm8[4],ymm6[4],ymm8[5],ymm6[5] +; AVX-NEXT: vshufps {{.*#+}} ymm7 = ymm14[0,1],ymm7[2,0],ymm14[4,5],ymm7[6,4] +; AVX-NEXT: vunpcklpd {{.*#+}} ymm14 = ymm2[0],ymm3[0],ymm2[2],ymm3[2] +; AVX-NEXT: vextractf128 $1, %ymm14, %xmm14 +; AVX-NEXT: vunpcklps {{.*#+}} ymm13 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[4],ymm0[4],ymm1[5],ymm0[5] +; AVX-NEXT: vextractf128 $1, %ymm13, %xmm13 +; AVX-NEXT: vshufps {{.*#+}} xmm13 = xmm13[0,1],xmm14[2,0] +; AVX-NEXT: vblendps {{.*#+}} ymm7 = ymm13[0,1,2,3],ymm7[4,5,6,7] +; AVX-NEXT: vunpcklps {{.*#+}} ymm4 = ymm5[0],ymm4[0],ymm5[1],ymm4[1],ymm5[4],ymm4[4],ymm5[5],ymm4[5] +; AVX-NEXT: vshufps {{.*#+}} ymm5 = ymm6[1,0],ymm8[1,0],ymm6[5,4],ymm8[5,4] +; AVX-NEXT: vshufps {{.*#+}} ymm4 = ymm5[2,0],ymm4[2,3],ymm5[6,4],ymm4[6,7] +; AVX-NEXT: vunpcklps {{.*#+}} ymm2 = ymm3[0],ymm2[0],ymm3[1],ymm2[1],ymm3[4],ymm2[4],ymm3[5],ymm2[5] +; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm0[1,0],ymm1[1,0],ymm0[5,4],ymm1[5,4] +; AVX-NEXT: vextractf128 $1, %ymm2, %xmm1 +; AVX-NEXT: vextractf128 $1, %ymm0, %xmm0 +; AVX-NEXT: vshufps {{.*#+}} xmm0 = xmm0[2,0],xmm1[2,3] +; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm4[4,5,6,7] +; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload +; AVX-NEXT: vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm1 # 16-byte Folded Reload +; AVX-NEXT: # xmm1 = xmm1[2],mem[2],xmm1[3],mem[3] +; AVX-NEXT: vunpckhps {{.*#+}} xmm2 = xmm11[2],xmm15[2],xmm11[3],xmm15[3] +; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload +; AVX-NEXT: vshufps {{.*#+}} xmm3 = xmm11[2,2,2,2] +; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload +; AVX-NEXT: vblendps {{.*#+}} xmm3 = xmm8[0,1,2],xmm3[3] +; AVX-NEXT: vinsertf128 $1, %xmm3, %ymm0, %ymm3 +; AVX-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm4 +; AVX-NEXT: vblendps {{.*#+}} ymm3 = ymm4[0,1,2,3,4,5],ymm3[6,7] +; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload +; AVX-NEXT: vshufps {{.*#+}} xmm4 = xmm5[2,2,2,2] ; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload ; AVX-NEXT: vblendps {{.*#+}} xmm4 = xmm6[0,1,2],xmm4[3] +; AVX-NEXT: vblendps {{.*#+}} xmm4 = xmm1[0,1],xmm4[2,3] +; AVX-NEXT: vblendps {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7] +; AVX-NEXT: vunpckhps {{.*#+}} xmm4 = xmm6[2],xmm5[2],xmm6[3],xmm5[3] +; AVX-NEXT: vunpckhpd {{.*#+}} xmm1 = xmm1[1],xmm4[1] +; AVX-NEXT: vunpckhps {{.*#+}} xmm4 = xmm8[2],xmm11[2],xmm8[3],xmm11[3] ; AVX-NEXT: vinsertf128 $1, %xmm4, %ymm0, %ymm4 -; AVX-NEXT: vinsertf128 $1, %xmm3, %ymm0, %ymm5 -; AVX-NEXT: vblendps {{.*#+}} ymm4 = ymm5[0,1,2,3,4,5],ymm4[6,7] -; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload -; AVX-NEXT: vshufps {{.*#+}} xmm5 = xmm8[2,2,2,2] -; AVX-NEXT: vblendps {{.*#+}} xmm5 = xmm14[0,1,2],xmm5[3] -; AVX-NEXT: vblendps {{.*#+}} xmm5 = xmm2[0,1],xmm5[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm4 = ymm5[0,1,2,3],ymm4[4,5,6,7] -; AVX-NEXT: vunpckhps {{.*#+}} xmm5 = xmm14[2],xmm8[2],xmm14[3],xmm8[3] -; AVX-NEXT: vunpckhpd {{.*#+}} xmm2 = xmm2[1],xmm5[1] -; AVX-NEXT: vunpckhps {{.*#+}} xmm5 = xmm6[2],xmm7[2],xmm6[3],xmm7[3] -; AVX-NEXT: vinsertf128 $1, %xmm5, %ymm0, %ymm5 -; AVX-NEXT: vshufps {{.*#+}} xmm3 = xmm3[2,3,2,3] -; AVX-NEXT: vinsertf128 $1, %xmm3, %ymm0, %ymm3 -; AVX-NEXT: vblendps {{.*#+}} ymm3 = ymm3[0,1,2,3,4,5],ymm5[6,7] -; AVX-NEXT: vblendps {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm3[4,5,6,7] +; AVX-NEXT: vshufps {{.*#+}} xmm2 = xmm2[2,3,2,3] +; AVX-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm2 +; AVX-NEXT: vblendps {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm4[6,7] +; AVX-NEXT: vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7] ; AVX-NEXT: movq {{[0-9]+}}(%rsp), %rax -; AVX-NEXT: vmovaps %ymm2, 96(%rax) -; AVX-NEXT: vmovaps %ymm4, 64(%rax) -; AVX-NEXT: vmovaps %ymm1, 288(%rax) -; AVX-NEXT: vmovaps %ymm0, 256(%rax) -; AVX-NEXT: vmovaps %ymm10, 352(%rax) -; AVX-NEXT: vmovaps %ymm12, 320(%rax) -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm0, (%rax) -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm0, 32(%rax) -; AVX-NEXT: vmovups (%rsp), %ymm0 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm1, 96(%rax) +; AVX-NEXT: vmovaps %ymm3, 64(%rax) ; AVX-NEXT: vmovaps %ymm0, 160(%rax) +; AVX-NEXT: vmovaps %ymm7, 128(%rax) +; AVX-NEXT: vmovaps %ymm9, 224(%rax) +; AVX-NEXT: vmovaps %ymm10, 192(%rax) +; AVX-NEXT: vmovaps %ymm12, 288(%rax) ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm0, 128(%rax) -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm0, 224(%rax) +; AVX-NEXT: vmovaps %ymm0, 256(%rax) +; AVX-NEXT: vmovups (%rsp), %ymm0 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm0, 352(%rax) ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm0, 192(%rax) +; AVX-NEXT: vmovaps %ymm0, 320(%rax) ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload ; AVX-NEXT: vmovaps %ymm0, 416(%rax) ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload @@ -2274,6 +2277,10 @@ define void @store_i32_stride8_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX-NEXT: vmovaps %ymm0, 480(%rax) ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload ; AVX-NEXT: vmovaps %ymm0, 448(%rax) +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm0, (%rax) +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm0, 32(%rax) ; AVX-NEXT: addq $264, %rsp # imm = 0x108 ; AVX-NEXT: vzeroupper ; AVX-NEXT: retq @@ -4493,282 +4500,220 @@ define void @store_i32_stride8_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; ; AVX-LABEL: store_i32_stride8_vf32: ; AVX: # %bb.0: -; AVX-NEXT: subq $648, %rsp # imm = 0x288 +; AVX-NEXT: subq $680, %rsp # imm = 0x2A8 ; AVX-NEXT: movq {{[0-9]+}}(%rsp), %rax ; AVX-NEXT: movq {{[0-9]+}}(%rsp), %r10 -; AVX-NEXT: vmovaps (%rdi), %ymm3 -; AVX-NEXT: vmovaps 32(%rdi), %ymm0 -; AVX-NEXT: vmovaps (%rsi), %ymm4 -; AVX-NEXT: vmovaps 32(%rsi), %ymm1 -; AVX-NEXT: vmovaps 32(%rdx), %ymm2 -; AVX-NEXT: vmovaps (%rdx), %ymm5 -; AVX-NEXT: vmovaps (%rcx), %ymm7 -; AVX-NEXT: vmovaps (%r8), %ymm8 -; AVX-NEXT: vmovaps (%r9), %ymm11 -; AVX-NEXT: vmovaps (%r10), %ymm10 -; AVX-NEXT: vmovaps (%rax), %ymm12 -; AVX-NEXT: vunpcklpd {{.*#+}} ymm6 = ymm12[0],ymm10[0],ymm12[2],ymm10[2] -; AVX-NEXT: vunpcklps {{.*#+}} ymm9 = ymm8[0],ymm11[0],ymm8[1],ymm11[1],ymm8[4],ymm11[4],ymm8[5],ymm11[5] -; AVX-NEXT: vshufps {{.*#+}} ymm6 = ymm9[0,1],ymm6[2,0],ymm9[4,5],ymm6[6,4] -; AVX-NEXT: vunpcklps {{.*#+}} ymm9 = ymm3[0],ymm4[0],ymm3[1],ymm4[1],ymm3[4],ymm4[4],ymm3[5],ymm4[5] -; AVX-NEXT: vextractf128 $1, %ymm9, %xmm9 -; AVX-NEXT: vunpcklpd {{.*#+}} ymm13 = ymm7[0],ymm5[0],ymm7[2],ymm5[2] -; AVX-NEXT: vshufps {{.*#+}} ymm13 = ymm13[0,1,2,0,4,5,6,4] -; AVX-NEXT: vextractf128 $1, %ymm13, %xmm13 -; AVX-NEXT: vblendps {{.*#+}} xmm9 = xmm9[0,1],xmm13[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm6 = ymm9[0,1,2,3],ymm6[4,5,6,7] +; AVX-NEXT: vmovaps (%rdi), %ymm0 +; AVX-NEXT: vmovaps (%rsi), %ymm1 +; AVX-NEXT: vmovaps (%rdx), %ymm2 +; AVX-NEXT: vmovaps (%rcx), %ymm3 +; AVX-NEXT: vmovaps (%rcx), %xmm5 +; AVX-NEXT: vmovaps (%rdx), %xmm7 +; AVX-NEXT: vunpcklps {{.*#+}} xmm4 = xmm7[0],xmm5[0],xmm7[1],xmm5[1] +; AVX-NEXT: vmovaps (%rsi), %xmm9 +; AVX-NEXT: vmovaps (%rdi), %xmm12 +; AVX-NEXT: vmovaps (%r9), %xmm11 +; AVX-NEXT: vmovaps (%r8), %xmm13 +; AVX-NEXT: vunpcklps {{.*#+}} xmm6 = xmm13[0],xmm11[0],xmm13[1],xmm11[1] +; AVX-NEXT: vinsertf128 $1, %xmm6, %ymm0, %ymm6 +; AVX-NEXT: vmovaps (%rax), %xmm8 +; AVX-NEXT: vmovaps (%r10), %xmm10 +; AVX-NEXT: vunpcklps {{.*#+}} xmm14 = xmm10[0],xmm8[0],xmm10[1],xmm8[1] +; AVX-NEXT: vshufps {{.*#+}} xmm15 = xmm14[0,1,0,1] +; AVX-NEXT: vinsertf128 $1, %xmm15, %ymm0, %ymm15 +; AVX-NEXT: vblendps {{.*#+}} ymm6 = ymm6[0,1,2,3,4,5],ymm15[6,7] +; AVX-NEXT: vunpcklps {{.*#+}} xmm15 = xmm12[0],xmm9[0],xmm12[1],xmm9[1] +; AVX-NEXT: vmovlhps {{.*#+}} xmm15 = xmm15[0],xmm4[0] +; AVX-NEXT: vblendps {{.*#+}} ymm6 = ymm15[0,1,2,3],ymm6[4,5,6,7] ; AVX-NEXT: vmovups %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vunpcklps {{.*#+}} ymm6 = ymm10[0],ymm12[0],ymm10[1],ymm12[1],ymm10[4],ymm12[4],ymm10[5],ymm12[5] -; AVX-NEXT: vshufps {{.*#+}} ymm9 = ymm11[1,0],ymm8[1,0],ymm11[5,4],ymm8[5,4] -; AVX-NEXT: vshufps {{.*#+}} ymm6 = ymm9[2,0],ymm6[2,3],ymm9[6,4],ymm6[6,7] -; AVX-NEXT: vunpcklps {{.*#+}} ymm9 = ymm5[0],ymm7[0],ymm5[1],ymm7[1],ymm5[4],ymm7[4],ymm5[5],ymm7[5] -; AVX-NEXT: vextractf128 $1, %ymm9, %xmm9 -; AVX-NEXT: vshufps {{.*#+}} ymm13 = ymm4[1,0],ymm3[1,0],ymm4[5,4],ymm3[5,4] -; AVX-NEXT: vshufps {{.*#+}} ymm13 = ymm13[2,0,2,3,6,4,6,7] -; AVX-NEXT: vextractf128 $1, %ymm13, %xmm13 -; AVX-NEXT: vblendps {{.*#+}} xmm9 = xmm13[0,1],xmm9[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm6 = ymm9[0,1,2,3],ymm6[4,5,6,7] +; AVX-NEXT: vshufps {{.*#+}} xmm6 = xmm12[1,1,1,1] +; AVX-NEXT: vblendps {{.*#+}} xmm6 = xmm6[0],xmm9[1],xmm6[2,3] +; AVX-NEXT: vblendps {{.*#+}} xmm6 = xmm6[0,1],xmm4[2,3] +; AVX-NEXT: vinsertf128 $1, %xmm14, %ymm0, %ymm4 +; AVX-NEXT: vshufps {{.*#+}} xmm14 = xmm13[1,1,1,1] +; AVX-NEXT: vblendps {{.*#+}} xmm14 = xmm14[0],xmm11[1],xmm14[2,3] +; AVX-NEXT: vinsertf128 $1, %xmm14, %ymm0, %ymm14 +; AVX-NEXT: vblendps {{.*#+}} ymm14 = ymm14[0,1,2,3,4,5],ymm4[6,7] +; AVX-NEXT: vmovaps (%r8), %ymm4 +; AVX-NEXT: vblendps {{.*#+}} ymm6 = ymm6[0,1,2,3],ymm14[4,5,6,7] ; AVX-NEXT: vmovups %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vunpckhpd {{.*#+}} ymm6 = ymm12[1],ymm10[1],ymm12[3],ymm10[3] -; AVX-NEXT: vunpckhps {{.*#+}} ymm9 = ymm8[2],ymm11[2],ymm8[3],ymm11[3],ymm8[6],ymm11[6],ymm8[7],ymm11[7] -; AVX-NEXT: vshufps {{.*#+}} ymm9 = ymm9[0,1],ymm6[2,0],ymm9[4,5],ymm6[6,4] -; AVX-NEXT: vunpckhps {{.*#+}} ymm6 = ymm3[2],ymm4[2],ymm3[3],ymm4[3],ymm3[6],ymm4[6],ymm3[7],ymm4[7] -; AVX-NEXT: vextractf128 $1, %ymm6, %xmm6 -; AVX-NEXT: vunpckhpd {{.*#+}} ymm13 = ymm7[1],ymm5[1],ymm7[3],ymm5[3] -; AVX-NEXT: vshufps {{.*#+}} ymm13 = ymm13[0,1,2,0,4,5,6,4] -; AVX-NEXT: vextractf128 $1, %ymm13, %xmm13 -; AVX-NEXT: vblendps {{.*#+}} xmm13 = xmm6[0,1],xmm13[2,3] -; AVX-NEXT: vmovaps 32(%rcx), %ymm6 +; AVX-NEXT: vmovaps (%r9), %ymm6 +; AVX-NEXT: vunpckhps {{.*#+}} xmm12 = xmm12[2],xmm9[2],xmm12[3],xmm9[3] +; AVX-NEXT: vunpckhps {{.*#+}} xmm11 = xmm13[2],xmm11[2],xmm13[3],xmm11[3] +; AVX-NEXT: vinsertf128 $1, %xmm11, %ymm0, %ymm9 +; AVX-NEXT: vshufps {{.*#+}} xmm13 = xmm8[2,2,2,2] +; AVX-NEXT: vblendps {{.*#+}} xmm13 = xmm10[0,1,2],xmm13[3] +; AVX-NEXT: vinsertf128 $1, %xmm13, %ymm0, %ymm13 +; AVX-NEXT: vblendps {{.*#+}} ymm9 = ymm9[0,1,2,3,4,5],ymm13[6,7] +; AVX-NEXT: vshufps {{.*#+}} xmm13 = xmm5[2,2,2,2] +; AVX-NEXT: vblendps {{.*#+}} xmm13 = xmm7[0,1,2],xmm13[3] +; AVX-NEXT: vblendps {{.*#+}} xmm13 = xmm12[0,1],xmm13[2,3] ; AVX-NEXT: vblendps {{.*#+}} ymm9 = ymm13[0,1,2,3],ymm9[4,5,6,7] ; AVX-NEXT: vmovups %ymm9, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vmovaps 32(%r8), %ymm9 -; AVX-NEXT: vunpckhps {{.*#+}} ymm12 = ymm10[2],ymm12[2],ymm10[3],ymm12[3],ymm10[6],ymm12[6],ymm10[7],ymm12[7] -; AVX-NEXT: vmovaps 32(%r9), %ymm10 -; AVX-NEXT: vshufps {{.*#+}} ymm11 = ymm11[3,0],ymm8[3,0],ymm11[7,4],ymm8[7,4] -; AVX-NEXT: vmovaps 32(%r10), %ymm8 -; AVX-NEXT: vshufps {{.*#+}} ymm12 = ymm11[2,0],ymm12[2,3],ymm11[6,4],ymm12[6,7] -; AVX-NEXT: vmovaps 32(%rax), %ymm11 -; AVX-NEXT: vunpckhps {{.*#+}} ymm5 = ymm5[2],ymm7[2],ymm5[3],ymm7[3],ymm5[6],ymm7[6],ymm5[7],ymm7[7] -; AVX-NEXT: vextractf128 $1, %ymm5, %xmm5 -; AVX-NEXT: vshufps {{.*#+}} ymm3 = ymm4[3,0],ymm3[3,0],ymm4[7,4],ymm3[7,4] -; AVX-NEXT: vshufps {{.*#+}} ymm3 = ymm3[2,0,2,3,6,4,6,7] -; AVX-NEXT: vextractf128 $1, %ymm3, %xmm3 -; AVX-NEXT: vblendps {{.*#+}} xmm3 = xmm3[0,1],xmm5[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm3 = ymm3[0,1,2,3],ymm12[4,5,6,7] -; AVX-NEXT: vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vunpcklpd {{.*#+}} ymm3 = ymm11[0],ymm8[0],ymm11[2],ymm8[2] -; AVX-NEXT: vunpcklps {{.*#+}} ymm4 = ymm9[0],ymm10[0],ymm9[1],ymm10[1],ymm9[4],ymm10[4],ymm9[5],ymm10[5] -; AVX-NEXT: vshufps {{.*#+}} ymm3 = ymm4[0,1],ymm3[2,0],ymm4[4,5],ymm3[6,4] -; AVX-NEXT: vunpcklps {{.*#+}} ymm4 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5] -; AVX-NEXT: vextractf128 $1, %ymm4, %xmm4 -; AVX-NEXT: vunpcklpd {{.*#+}} ymm5 = ymm6[0],ymm2[0],ymm6[2],ymm2[2] -; AVX-NEXT: vshufps {{.*#+}} ymm5 = ymm5[0,1,2,0,4,5,6,4] -; AVX-NEXT: vextractf128 $1, %ymm5, %xmm5 -; AVX-NEXT: vblendps {{.*#+}} xmm4 = xmm4[0,1],xmm5[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7] -; AVX-NEXT: vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vunpcklps {{.*#+}} ymm3 = ymm8[0],ymm11[0],ymm8[1],ymm11[1],ymm8[4],ymm11[4],ymm8[5],ymm11[5] -; AVX-NEXT: vshufps {{.*#+}} ymm4 = ymm10[1,0],ymm9[1,0],ymm10[5,4],ymm9[5,4] -; AVX-NEXT: vshufps {{.*#+}} ymm3 = ymm4[2,0],ymm3[2,3],ymm4[6,4],ymm3[6,7] -; AVX-NEXT: vunpcklps {{.*#+}} ymm4 = ymm2[0],ymm6[0],ymm2[1],ymm6[1],ymm2[4],ymm6[4],ymm2[5],ymm6[5] -; AVX-NEXT: vextractf128 $1, %ymm4, %xmm4 -; AVX-NEXT: vshufps {{.*#+}} ymm5 = ymm1[1,0],ymm0[1,0],ymm1[5,4],ymm0[5,4] -; AVX-NEXT: vshufps {{.*#+}} ymm5 = ymm5[2,0,2,3,6,4,6,7] -; AVX-NEXT: vextractf128 $1, %ymm5, %xmm5 -; AVX-NEXT: vblendps {{.*#+}} xmm4 = xmm5[0,1],xmm4[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7] -; AVX-NEXT: vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vunpckhpd {{.*#+}} ymm3 = ymm11[1],ymm8[1],ymm11[3],ymm8[3] -; AVX-NEXT: vunpckhps {{.*#+}} ymm4 = ymm9[2],ymm10[2],ymm9[3],ymm10[3],ymm9[6],ymm10[6],ymm9[7],ymm10[7] -; AVX-NEXT: vshufps {{.*#+}} ymm3 = ymm4[0,1],ymm3[2,0],ymm4[4,5],ymm3[6,4] -; AVX-NEXT: vunpckhps {{.*#+}} ymm4 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7] -; AVX-NEXT: vextractf128 $1, %ymm4, %xmm4 -; AVX-NEXT: vunpckhpd {{.*#+}} ymm5 = ymm6[1],ymm2[1],ymm6[3],ymm2[3] -; AVX-NEXT: vshufps {{.*#+}} ymm5 = ymm5[0,1,2,0,4,5,6,4] -; AVX-NEXT: vextractf128 $1, %ymm5, %xmm5 -; AVX-NEXT: vblendps {{.*#+}} xmm4 = xmm4[0,1],xmm5[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7] -; AVX-NEXT: vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vmovaps 64(%rdi), %ymm3 -; AVX-NEXT: vunpckhps {{.*#+}} ymm7 = ymm8[2],ymm11[2],ymm8[3],ymm11[3],ymm8[6],ymm11[6],ymm8[7],ymm11[7] -; AVX-NEXT: vmovaps 64(%rsi), %ymm4 -; AVX-NEXT: vshufps {{.*#+}} ymm8 = ymm10[3,0],ymm9[3,0],ymm10[7,4],ymm9[7,4] -; AVX-NEXT: vmovaps 64(%r8), %ymm5 -; AVX-NEXT: vshufps {{.*#+}} ymm8 = ymm8[2,0],ymm7[2,3],ymm8[6,4],ymm7[6,7] -; AVX-NEXT: vmovaps 64(%r9), %ymm7 -; AVX-NEXT: vunpckhps {{.*#+}} ymm6 = ymm2[2],ymm6[2],ymm2[3],ymm6[3],ymm2[6],ymm6[6],ymm2[7],ymm6[7] -; AVX-NEXT: vmovaps 64(%r10), %ymm2 -; AVX-NEXT: vshufps {{.*#+}} ymm1 = ymm1[3,0],ymm0[3,0],ymm1[7,4],ymm0[7,4] -; AVX-NEXT: vmovaps 64(%rax), %ymm0 -; AVX-NEXT: vextractf128 $1, %ymm6, %xmm6 -; AVX-NEXT: vshufps {{.*#+}} ymm1 = ymm1[2,0,2,3,6,4,6,7] -; AVX-NEXT: vextractf128 $1, %ymm1, %xmm1 -; AVX-NEXT: vblendps {{.*#+}} xmm1 = xmm1[0,1],xmm6[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm8[4,5,6,7] -; AVX-NEXT: vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vunpcklpd {{.*#+}} ymm1 = ymm0[0],ymm2[0],ymm0[2],ymm2[2] -; AVX-NEXT: vunpcklps {{.*#+}} ymm6 = ymm5[0],ymm7[0],ymm5[1],ymm7[1],ymm5[4],ymm7[4],ymm5[5],ymm7[5] -; AVX-NEXT: vshufps {{.*#+}} ymm6 = ymm6[0,1],ymm1[2,0],ymm6[4,5],ymm1[6,4] -; AVX-NEXT: vunpcklps {{.*#+}} ymm1 = ymm3[0],ymm4[0],ymm3[1],ymm4[1],ymm3[4],ymm4[4],ymm3[5],ymm4[5] -; AVX-NEXT: vextractf128 $1, %ymm1, %xmm8 -; AVX-NEXT: vmovaps 64(%rdx), %ymm1 -; AVX-NEXT: vmovaps 64(%rcx), %ymm9 -; AVX-NEXT: vunpcklpd {{.*#+}} ymm10 = ymm9[0],ymm1[0],ymm9[2],ymm1[2] -; AVX-NEXT: vshufps {{.*#+}} ymm10 = ymm10[0,1,2,0,4,5,6,4] -; AVX-NEXT: vextractf128 $1, %ymm10, %xmm10 -; AVX-NEXT: vblendps {{.*#+}} xmm8 = xmm8[0,1],xmm10[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm6 = ymm8[0,1,2,3],ymm6[4,5,6,7] -; AVX-NEXT: vmovups %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vunpcklps {{.*#+}} ymm6 = ymm2[0],ymm0[0],ymm2[1],ymm0[1],ymm2[4],ymm0[4],ymm2[5],ymm0[5] -; AVX-NEXT: vshufps {{.*#+}} ymm8 = ymm7[1,0],ymm5[1,0],ymm7[5,4],ymm5[5,4] -; AVX-NEXT: vshufps {{.*#+}} ymm6 = ymm8[2,0],ymm6[2,3],ymm8[6,4],ymm6[6,7] -; AVX-NEXT: vunpcklps {{.*#+}} ymm8 = ymm1[0],ymm9[0],ymm1[1],ymm9[1],ymm1[4],ymm9[4],ymm1[5],ymm9[5] -; AVX-NEXT: vextractf128 $1, %ymm8, %xmm8 -; AVX-NEXT: vshufps {{.*#+}} ymm10 = ymm4[1,0],ymm3[1,0],ymm4[5,4],ymm3[5,4] -; AVX-NEXT: vshufps {{.*#+}} ymm10 = ymm10[2,0,2,3,6,4,6,7] -; AVX-NEXT: vextractf128 $1, %ymm10, %xmm10 -; AVX-NEXT: vblendps {{.*#+}} xmm8 = xmm10[0,1],xmm8[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm6 = ymm8[0,1,2,3],ymm6[4,5,6,7] -; AVX-NEXT: vmovups %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vunpckhpd {{.*#+}} ymm6 = ymm0[1],ymm2[1],ymm0[3],ymm2[3] -; AVX-NEXT: vunpckhps {{.*#+}} ymm8 = ymm5[2],ymm7[2],ymm5[3],ymm7[3],ymm5[6],ymm7[6],ymm5[7],ymm7[7] -; AVX-NEXT: vshufps {{.*#+}} ymm6 = ymm8[0,1],ymm6[2,0],ymm8[4,5],ymm6[6,4] -; AVX-NEXT: vunpckhps {{.*#+}} ymm8 = ymm3[2],ymm4[2],ymm3[3],ymm4[3],ymm3[6],ymm4[6],ymm3[7],ymm4[7] +; AVX-NEXT: vmovaps (%r10), %ymm9 +; AVX-NEXT: vunpckhps {{.*#+}} xmm7 = xmm7[2],xmm5[2],xmm7[3],xmm5[3] +; AVX-NEXT: vmovaps (%rax), %ymm5 +; AVX-NEXT: vunpckhpd {{.*#+}} xmm7 = xmm12[1],xmm7[1] +; AVX-NEXT: vunpckhps {{.*#+}} xmm8 = xmm10[2],xmm8[2],xmm10[3],xmm8[3] +; AVX-NEXT: vinsertf128 $1, %xmm8, %ymm0, %ymm8 +; AVX-NEXT: vshufps {{.*#+}} xmm10 = xmm11[2,3,2,3] +; AVX-NEXT: vinsertf128 $1, %xmm10, %ymm0, %ymm10 +; AVX-NEXT: vblendps {{.*#+}} ymm8 = ymm10[0,1,2,3,4,5],ymm8[6,7] +; AVX-NEXT: vblendps {{.*#+}} ymm7 = ymm7[0,1,2,3],ymm8[4,5,6,7] +; AVX-NEXT: vmovups %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vunpcklpd {{.*#+}} ymm7 = ymm5[0],ymm9[0],ymm5[2],ymm9[2] +; AVX-NEXT: vunpcklps {{.*#+}} ymm8 = ymm4[0],ymm6[0],ymm4[1],ymm6[1],ymm4[4],ymm6[4],ymm4[5],ymm6[5] +; AVX-NEXT: vshufps {{.*#+}} ymm7 = ymm8[0,1],ymm7[2,0],ymm8[4,5],ymm7[6,4] +; AVX-NEXT: vunpcklpd {{.*#+}} ymm8 = ymm3[0],ymm2[0],ymm3[2],ymm2[2] ; AVX-NEXT: vextractf128 $1, %ymm8, %xmm8 -; AVX-NEXT: vunpckhpd {{.*#+}} ymm10 = ymm9[1],ymm1[1],ymm9[3],ymm1[3] -; AVX-NEXT: vshufps {{.*#+}} ymm10 = ymm10[0,1,2,0,4,5,6,4] -; AVX-NEXT: vextractf128 $1, %ymm10, %xmm10 -; AVX-NEXT: vblendps {{.*#+}} xmm8 = xmm8[0,1],xmm10[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm6 = ymm8[0,1,2,3],ymm6[4,5,6,7] -; AVX-NEXT: vmovups %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vunpckhps {{.*#+}} ymm0 = ymm2[2],ymm0[2],ymm2[3],ymm0[3],ymm2[6],ymm0[6],ymm2[7],ymm0[7] -; AVX-NEXT: vshufps {{.*#+}} ymm2 = ymm7[3,0],ymm5[3,0],ymm7[7,4],ymm5[7,4] -; AVX-NEXT: vshufps {{.*#+}} ymm5 = ymm2[2,0],ymm0[2,3],ymm2[6,4],ymm0[6,7] -; AVX-NEXT: vunpckhps {{.*#+}} ymm2 = ymm1[2],ymm9[2],ymm1[3],ymm9[3],ymm1[6],ymm9[6],ymm1[7],ymm9[7] -; AVX-NEXT: vmovaps 96(%r8), %ymm0 -; AVX-NEXT: vshufps {{.*#+}} ymm3 = ymm4[3,0],ymm3[3,0],ymm4[7,4],ymm3[7,4] -; AVX-NEXT: vmovaps 96(%r9), %ymm1 -; AVX-NEXT: vextractf128 $1, %ymm2, %xmm2 -; AVX-NEXT: vshufps {{.*#+}} ymm3 = ymm3[2,0,2,3,6,4,6,7] -; AVX-NEXT: vextractf128 $1, %ymm3, %xmm3 -; AVX-NEXT: vblendps {{.*#+}} xmm3 = xmm3[0,1],xmm2[2,3] -; AVX-NEXT: vmovaps 96(%r10), %ymm2 -; AVX-NEXT: vblendps {{.*#+}} ymm3 = ymm3[0,1,2,3],ymm5[4,5,6,7] -; AVX-NEXT: vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vmovaps 96(%rax), %ymm5 -; AVX-NEXT: vunpcklpd {{.*#+}} ymm3 = ymm5[0],ymm2[0],ymm5[2],ymm2[2] -; AVX-NEXT: vunpcklps {{.*#+}} ymm4 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5] -; AVX-NEXT: vshufps {{.*#+}} ymm7 = ymm4[0,1],ymm3[2,0],ymm4[4,5],ymm3[6,4] -; AVX-NEXT: vmovaps 96(%rdi), %ymm3 -; AVX-NEXT: vmovaps 96(%rsi), %ymm4 -; AVX-NEXT: vunpcklps {{.*#+}} ymm6 = ymm3[0],ymm4[0],ymm3[1],ymm4[1],ymm3[4],ymm4[4],ymm3[5],ymm4[5] -; AVX-NEXT: vextractf128 $1, %ymm6, %xmm8 -; AVX-NEXT: vmovaps 96(%rdx), %ymm6 -; AVX-NEXT: vmovaps 96(%rcx), %ymm9 -; AVX-NEXT: vunpcklpd {{.*#+}} ymm10 = ymm9[0],ymm6[0],ymm9[2],ymm6[2] -; AVX-NEXT: vshufps {{.*#+}} ymm10 = ymm10[0,1,2,0,4,5,6,4] +; AVX-NEXT: vunpcklps {{.*#+}} ymm10 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5] ; AVX-NEXT: vextractf128 $1, %ymm10, %xmm10 -; AVX-NEXT: vblendps {{.*#+}} xmm8 = xmm8[0,1],xmm10[2,3] +; AVX-NEXT: vshufps {{.*#+}} xmm8 = xmm10[0,1],xmm8[2,0] ; AVX-NEXT: vblendps {{.*#+}} ymm7 = ymm8[0,1,2,3],ymm7[4,5,6,7] ; AVX-NEXT: vmovups %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vunpcklps {{.*#+}} ymm7 = ymm2[0],ymm5[0],ymm2[1],ymm5[1],ymm2[4],ymm5[4],ymm2[5],ymm5[5] -; AVX-NEXT: vshufps {{.*#+}} ymm8 = ymm1[1,0],ymm0[1,0],ymm1[5,4],ymm0[5,4] +; AVX-NEXT: vunpcklps {{.*#+}} ymm7 = ymm9[0],ymm5[0],ymm9[1],ymm5[1],ymm9[4],ymm5[4],ymm9[5],ymm5[5] +; AVX-NEXT: vshufps {{.*#+}} ymm8 = ymm6[1,0],ymm4[1,0],ymm6[5,4],ymm4[5,4] ; AVX-NEXT: vshufps {{.*#+}} ymm7 = ymm8[2,0],ymm7[2,3],ymm8[6,4],ymm7[6,7] -; AVX-NEXT: vunpcklps {{.*#+}} ymm8 = ymm6[0],ymm9[0],ymm6[1],ymm9[1],ymm6[4],ymm9[4],ymm6[5],ymm9[5] +; AVX-NEXT: vunpcklps {{.*#+}} ymm8 = ymm2[0],ymm3[0],ymm2[1],ymm3[1],ymm2[4],ymm3[4],ymm2[5],ymm3[5] ; AVX-NEXT: vextractf128 $1, %ymm8, %xmm8 -; AVX-NEXT: vshufps {{.*#+}} ymm10 = ymm4[1,0],ymm3[1,0],ymm4[5,4],ymm3[5,4] -; AVX-NEXT: vshufps {{.*#+}} ymm10 = ymm10[2,0,2,3,6,4,6,7] +; AVX-NEXT: vshufps {{.*#+}} ymm10 = ymm1[1,0],ymm0[1,0],ymm1[5,4],ymm0[5,4] ; AVX-NEXT: vextractf128 $1, %ymm10, %xmm10 -; AVX-NEXT: vblendps {{.*#+}} xmm8 = xmm10[0,1],xmm8[2,3] +; AVX-NEXT: vshufps {{.*#+}} xmm8 = xmm10[2,0],xmm8[2,3] ; AVX-NEXT: vblendps {{.*#+}} ymm7 = ymm8[0,1,2,3],ymm7[4,5,6,7] ; AVX-NEXT: vmovups %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vunpckhpd {{.*#+}} ymm7 = ymm5[1],ymm2[1],ymm5[3],ymm2[3] -; AVX-NEXT: vunpckhps {{.*#+}} ymm8 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7] +; AVX-NEXT: vunpckhpd {{.*#+}} ymm7 = ymm5[1],ymm9[1],ymm5[3],ymm9[3] +; AVX-NEXT: vunpckhps {{.*#+}} ymm8 = ymm4[2],ymm6[2],ymm4[3],ymm6[3],ymm4[6],ymm6[6],ymm4[7],ymm6[7] ; AVX-NEXT: vshufps {{.*#+}} ymm7 = ymm8[0,1],ymm7[2,0],ymm8[4,5],ymm7[6,4] -; AVX-NEXT: vunpckhps {{.*#+}} ymm8 = ymm3[2],ymm4[2],ymm3[3],ymm4[3],ymm3[6],ymm4[6],ymm3[7],ymm4[7] +; AVX-NEXT: vunpckhpd {{.*#+}} ymm8 = ymm3[1],ymm2[1],ymm3[3],ymm2[3] ; AVX-NEXT: vextractf128 $1, %ymm8, %xmm8 -; AVX-NEXT: vunpckhpd {{.*#+}} ymm10 = ymm9[1],ymm6[1],ymm9[3],ymm6[3] -; AVX-NEXT: vshufps {{.*#+}} ymm10 = ymm10[0,1,2,0,4,5,6,4] +; AVX-NEXT: vunpckhps {{.*#+}} ymm10 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7] ; AVX-NEXT: vextractf128 $1, %ymm10, %xmm10 -; AVX-NEXT: vblendps {{.*#+}} xmm8 = xmm8[0,1],xmm10[2,3] +; AVX-NEXT: vshufps {{.*#+}} xmm8 = xmm10[0,1],xmm8[2,0] ; AVX-NEXT: vblendps {{.*#+}} ymm7 = ymm8[0,1,2,3],ymm7[4,5,6,7] ; AVX-NEXT: vmovups %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vunpckhps {{.*#+}} ymm2 = ymm2[2],ymm5[2],ymm2[3],ymm5[3],ymm2[6],ymm5[6],ymm2[7],ymm5[7] -; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm1[3,0],ymm0[3,0],ymm1[7,4],ymm0[7,4] -; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm0[2,0],ymm2[2,3],ymm0[6,4],ymm2[6,7] -; AVX-NEXT: vunpckhps {{.*#+}} ymm1 = ymm6[2],ymm9[2],ymm6[3],ymm9[3],ymm6[6],ymm9[6],ymm6[7],ymm9[7] -; AVX-NEXT: vshufps {{.*#+}} ymm2 = ymm4[3,0],ymm3[3,0],ymm4[7,4],ymm3[7,4] -; AVX-NEXT: vextractf128 $1, %ymm1, %xmm1 -; AVX-NEXT: vshufps {{.*#+}} ymm2 = ymm2[2,0,2,3,6,4,6,7] +; AVX-NEXT: vmovaps 32(%rcx), %xmm7 +; AVX-NEXT: vunpckhps {{.*#+}} ymm8 = ymm9[2],ymm5[2],ymm9[3],ymm5[3],ymm9[6],ymm5[6],ymm9[7],ymm5[7] +; AVX-NEXT: vmovaps 32(%rax), %xmm5 +; AVX-NEXT: vshufps {{.*#+}} ymm6 = ymm6[3,0],ymm4[3,0],ymm6[7,4],ymm4[7,4] +; AVX-NEXT: vmovaps 32(%r10), %xmm4 +; AVX-NEXT: vshufps {{.*#+}} ymm6 = ymm6[2,0],ymm8[2,3],ymm6[6,4],ymm8[6,7] +; AVX-NEXT: vunpckhps {{.*#+}} ymm2 = ymm2[2],ymm3[2],ymm2[3],ymm3[3],ymm2[6],ymm3[6],ymm2[7],ymm3[7] ; AVX-NEXT: vextractf128 $1, %ymm2, %xmm2 -; AVX-NEXT: vblendps {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7] +; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm1[3,0],ymm0[3,0],ymm1[7,4],ymm0[7,4] +; AVX-NEXT: vextractf128 $1, %ymm0, %xmm0 +; AVX-NEXT: vshufps {{.*#+}} xmm0 = xmm0[2,0],xmm2[2,3] +; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm6[4,5,6,7] ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vmovaps (%r9), %xmm2 -; AVX-NEXT: vmovaps (%r8), %xmm3 -; AVX-NEXT: vunpcklps {{.*#+}} xmm0 = xmm3[0],xmm2[0],xmm3[1],xmm2[1] -; AVX-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm4 -; AVX-NEXT: vmovaps (%rax), %xmm0 -; AVX-NEXT: vmovaps (%r10), %xmm1 -; AVX-NEXT: vunpcklps {{.*#+}} xmm6 = xmm1[0],xmm0[0],xmm1[1],xmm0[1] -; AVX-NEXT: vshufps {{.*#+}} xmm5 = xmm6[0,1,0,1] -; AVX-NEXT: vinsertf128 $1, %xmm5, %ymm0, %ymm5 -; AVX-NEXT: vblendps {{.*#+}} ymm7 = ymm4[0,1,2,3,4,5],ymm5[6,7] -; AVX-NEXT: vmovaps (%rcx), %xmm4 -; AVX-NEXT: vmovaps (%rdx), %xmm5 -; AVX-NEXT: vunpcklps {{.*#+}} xmm8 = xmm5[0],xmm4[0],xmm5[1],xmm4[1] -; AVX-NEXT: vmovaps (%rsi), %xmm9 -; AVX-NEXT: vmovaps (%rdi), %xmm10 +; AVX-NEXT: vmovaps 32(%r9), %xmm1 +; AVX-NEXT: vmovaps 32(%r8), %xmm2 +; AVX-NEXT: vunpcklps {{.*#+}} xmm0 = xmm2[0],xmm1[0],xmm2[1],xmm1[1] +; AVX-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0 +; AVX-NEXT: vunpcklps {{.*#+}} xmm3 = xmm4[0],xmm5[0],xmm4[1],xmm5[1] +; AVX-NEXT: vshufps {{.*#+}} xmm6 = xmm3[0,1,0,1] +; AVX-NEXT: vinsertf128 $1, %xmm6, %ymm0, %ymm6 +; AVX-NEXT: vblendps {{.*#+}} ymm6 = ymm0[0,1,2,3,4,5],ymm6[6,7] +; AVX-NEXT: vmovaps 32(%rdx), %xmm0 +; AVX-NEXT: vunpcklps {{.*#+}} xmm8 = xmm0[0],xmm7[0],xmm0[1],xmm7[1] +; AVX-NEXT: vmovaps 32(%rsi), %xmm9 +; AVX-NEXT: vmovaps 32(%rdi), %xmm10 ; AVX-NEXT: vunpcklps {{.*#+}} xmm11 = xmm10[0],xmm9[0],xmm10[1],xmm9[1] ; AVX-NEXT: vmovlhps {{.*#+}} xmm11 = xmm11[0],xmm8[0] -; AVX-NEXT: vblendps {{.*#+}} ymm7 = ymm11[0,1,2,3],ymm7[4,5,6,7] -; AVX-NEXT: vmovups %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vshufps {{.*#+}} xmm7 = xmm10[1,1,1,1] -; AVX-NEXT: vblendps {{.*#+}} xmm7 = xmm7[0],xmm9[1],xmm7[2,3] -; AVX-NEXT: vblendps {{.*#+}} xmm7 = xmm7[0,1],xmm8[2,3] -; AVX-NEXT: vinsertf128 $1, %xmm6, %ymm0, %ymm6 -; AVX-NEXT: vshufps {{.*#+}} xmm8 = xmm3[1,1,1,1] -; AVX-NEXT: vblendps {{.*#+}} xmm8 = xmm8[0],xmm2[1],xmm8[2,3] -; AVX-NEXT: vinsertf128 $1, %xmm8, %ymm0, %ymm8 -; AVX-NEXT: vblendps {{.*#+}} ymm6 = ymm8[0,1,2,3,4,5],ymm6[6,7] -; AVX-NEXT: vblendps {{.*#+}} ymm6 = ymm7[0,1,2,3],ymm6[4,5,6,7] +; AVX-NEXT: vblendps {{.*#+}} ymm6 = ymm11[0,1,2,3],ymm6[4,5,6,7] ; AVX-NEXT: vmovups %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vunpckhps {{.*#+}} xmm6 = xmm10[2],xmm9[2],xmm10[3],xmm9[3] -; AVX-NEXT: vunpckhps {{.*#+}} xmm2 = xmm3[2],xmm2[2],xmm3[3],xmm2[3] -; AVX-NEXT: vshufps {{.*#+}} xmm3 = xmm0[2,2,2,2] -; AVX-NEXT: vblendps {{.*#+}} xmm3 = xmm1[0,1,2],xmm3[3] +; AVX-NEXT: vshufps {{.*#+}} xmm6 = xmm10[1,1,1,1] +; AVX-NEXT: vblendps {{.*#+}} xmm6 = xmm6[0],xmm9[1],xmm6[2,3] +; AVX-NEXT: vblendps {{.*#+}} xmm6 = xmm6[0,1],xmm8[2,3] ; AVX-NEXT: vinsertf128 $1, %xmm3, %ymm0, %ymm3 -; AVX-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm7 -; AVX-NEXT: vblendps {{.*#+}} ymm3 = ymm7[0,1,2,3,4,5],ymm3[6,7] -; AVX-NEXT: vshufps {{.*#+}} xmm7 = xmm4[2,2,2,2] -; AVX-NEXT: vblendps {{.*#+}} xmm7 = xmm5[0,1,2],xmm7[3] -; AVX-NEXT: vblendps {{.*#+}} xmm7 = xmm6[0,1],xmm7[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm3 = ymm7[0,1,2,3],ymm3[4,5,6,7] +; AVX-NEXT: vshufps {{.*#+}} xmm8 = xmm2[1,1,1,1] +; AVX-NEXT: vblendps {{.*#+}} xmm8 = xmm8[0],xmm1[1],xmm8[2,3] +; AVX-NEXT: vinsertf128 $1, %xmm8, %ymm0, %ymm8 +; AVX-NEXT: vblendps {{.*#+}} ymm3 = ymm8[0,1,2,3,4,5],ymm3[6,7] +; AVX-NEXT: vblendps {{.*#+}} ymm3 = ymm6[0,1,2,3],ymm3[4,5,6,7] ; AVX-NEXT: vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vunpckhps {{.*#+}} xmm3 = xmm5[2],xmm4[2],xmm5[3],xmm4[3] -; AVX-NEXT: vunpckhpd {{.*#+}} xmm3 = xmm6[1],xmm3[1] -; AVX-NEXT: vunpckhps {{.*#+}} xmm0 = xmm1[2],xmm0[2],xmm1[3],xmm0[3] -; AVX-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0 -; AVX-NEXT: vshufps {{.*#+}} xmm1 = xmm2[2,3,2,3] +; AVX-NEXT: vunpckhps {{.*#+}} xmm3 = xmm10[2],xmm9[2],xmm10[3],xmm9[3] +; AVX-NEXT: vunpckhps {{.*#+}} xmm2 = xmm2[2],xmm1[2],xmm2[3],xmm1[3] +; AVX-NEXT: vshufps {{.*#+}} xmm1 = xmm5[2,2,2,2] +; AVX-NEXT: vblendps {{.*#+}} xmm1 = xmm4[0,1,2],xmm1[3] ; AVX-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm1 -; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7] -; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm3[0,1,2,3],ymm0[4,5,6,7] -; AVX-NEXT: vmovups %ymm0, (%rsp) # 32-byte Spill -; AVX-NEXT: vmovaps 32(%r9), %xmm4 -; AVX-NEXT: vmovaps 32(%r8), %xmm5 +; AVX-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm6 +; AVX-NEXT: vblendps {{.*#+}} ymm1 = ymm6[0,1,2,3,4,5],ymm1[6,7] +; AVX-NEXT: vshufps {{.*#+}} xmm6 = xmm7[2,2,2,2] +; AVX-NEXT: vblendps {{.*#+}} xmm6 = xmm0[0,1,2],xmm6[3] +; AVX-NEXT: vblendps {{.*#+}} xmm6 = xmm3[0,1],xmm6[2,3] +; AVX-NEXT: vblendps {{.*#+}} ymm1 = ymm6[0,1,2,3],ymm1[4,5,6,7] +; AVX-NEXT: vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vunpckhps {{.*#+}} xmm0 = xmm0[2],xmm7[2],xmm0[3],xmm7[3] +; AVX-NEXT: vunpckhpd {{.*#+}} xmm3 = xmm3[1],xmm0[1] +; AVX-NEXT: vmovaps 32(%r8), %ymm0 +; AVX-NEXT: vunpckhps {{.*#+}} xmm4 = xmm4[2],xmm5[2],xmm4[3],xmm5[3] +; AVX-NEXT: vmovaps 32(%r9), %ymm1 +; AVX-NEXT: vinsertf128 $1, %xmm4, %ymm0, %ymm4 +; AVX-NEXT: vshufps {{.*#+}} xmm2 = xmm2[2,3,2,3] +; AVX-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm2 +; AVX-NEXT: vblendps {{.*#+}} ymm4 = ymm2[0,1,2,3,4,5],ymm4[6,7] +; AVX-NEXT: vmovaps 32(%r10), %ymm2 +; AVX-NEXT: vblendps {{.*#+}} ymm3 = ymm3[0,1,2,3],ymm4[4,5,6,7] +; AVX-NEXT: vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vmovaps 32(%rax), %ymm3 +; AVX-NEXT: vunpcklpd {{.*#+}} ymm4 = ymm3[0],ymm2[0],ymm3[2],ymm2[2] +; AVX-NEXT: vunpcklps {{.*#+}} ymm5 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5] +; AVX-NEXT: vshufps {{.*#+}} ymm6 = ymm5[0,1],ymm4[2,0],ymm5[4,5],ymm4[6,4] +; AVX-NEXT: vmovaps 32(%rdx), %ymm4 +; AVX-NEXT: vmovaps 32(%rcx), %ymm5 +; AVX-NEXT: vunpcklpd {{.*#+}} ymm7 = ymm5[0],ymm4[0],ymm5[2],ymm4[2] +; AVX-NEXT: vextractf128 $1, %ymm7, %xmm7 +; AVX-NEXT: vmovaps 32(%rdi), %ymm8 +; AVX-NEXT: vmovaps 32(%rsi), %ymm9 +; AVX-NEXT: vunpcklps {{.*#+}} ymm10 = ymm8[0],ymm9[0],ymm8[1],ymm9[1],ymm8[4],ymm9[4],ymm8[5],ymm9[5] +; AVX-NEXT: vextractf128 $1, %ymm10, %xmm10 +; AVX-NEXT: vshufps {{.*#+}} xmm7 = xmm10[0,1],xmm7[2,0] +; AVX-NEXT: vblendps {{.*#+}} ymm6 = ymm7[0,1,2,3],ymm6[4,5,6,7] +; AVX-NEXT: vmovups %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vunpcklps {{.*#+}} ymm6 = ymm2[0],ymm3[0],ymm2[1],ymm3[1],ymm2[4],ymm3[4],ymm2[5],ymm3[5] +; AVX-NEXT: vshufps {{.*#+}} ymm7 = ymm1[1,0],ymm0[1,0],ymm1[5,4],ymm0[5,4] +; AVX-NEXT: vshufps {{.*#+}} ymm6 = ymm7[2,0],ymm6[2,3],ymm7[6,4],ymm6[6,7] +; AVX-NEXT: vunpcklps {{.*#+}} ymm7 = ymm4[0],ymm5[0],ymm4[1],ymm5[1],ymm4[4],ymm5[4],ymm4[5],ymm5[5] +; AVX-NEXT: vextractf128 $1, %ymm7, %xmm7 +; AVX-NEXT: vshufps {{.*#+}} ymm10 = ymm9[1,0],ymm8[1,0],ymm9[5,4],ymm8[5,4] +; AVX-NEXT: vextractf128 $1, %ymm10, %xmm10 +; AVX-NEXT: vshufps {{.*#+}} xmm7 = xmm10[2,0],xmm7[2,3] +; AVX-NEXT: vblendps {{.*#+}} ymm6 = ymm7[0,1,2,3],ymm6[4,5,6,7] +; AVX-NEXT: vmovups %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vunpckhpd {{.*#+}} ymm6 = ymm3[1],ymm2[1],ymm3[3],ymm2[3] +; AVX-NEXT: vunpckhps {{.*#+}} ymm7 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7] +; AVX-NEXT: vshufps {{.*#+}} ymm6 = ymm7[0,1],ymm6[2,0],ymm7[4,5],ymm6[6,4] +; AVX-NEXT: vunpckhpd {{.*#+}} ymm7 = ymm5[1],ymm4[1],ymm5[3],ymm4[3] +; AVX-NEXT: vextractf128 $1, %ymm7, %xmm7 +; AVX-NEXT: vunpckhps {{.*#+}} ymm10 = ymm8[2],ymm9[2],ymm8[3],ymm9[3],ymm8[6],ymm9[6],ymm8[7],ymm9[7] +; AVX-NEXT: vextractf128 $1, %ymm10, %xmm10 +; AVX-NEXT: vshufps {{.*#+}} xmm7 = xmm10[0,1],xmm7[2,0] +; AVX-NEXT: vblendps {{.*#+}} ymm6 = ymm7[0,1,2,3],ymm6[4,5,6,7] +; AVX-NEXT: vmovups %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vunpckhps {{.*#+}} ymm2 = ymm2[2],ymm3[2],ymm2[3],ymm3[3],ymm2[6],ymm3[6],ymm2[7],ymm3[7] +; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm1[3,0],ymm0[3,0],ymm1[7,4],ymm0[7,4] +; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm0[2,0],ymm2[2,3],ymm0[6,4],ymm2[6,7] +; AVX-NEXT: vunpckhps {{.*#+}} ymm1 = ymm4[2],ymm5[2],ymm4[3],ymm5[3],ymm4[6],ymm5[6],ymm4[7],ymm5[7] +; AVX-NEXT: vshufps {{.*#+}} ymm2 = ymm9[3,0],ymm8[3,0],ymm9[7,4],ymm8[7,4] +; AVX-NEXT: vextractf128 $1, %ymm1, %xmm1 +; AVX-NEXT: vextractf128 $1, %ymm2, %xmm2 +; AVX-NEXT: vshufps {{.*#+}} xmm1 = xmm2[2,0],xmm1[2,3] +; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7] +; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vmovaps 64(%r9), %xmm4 +; AVX-NEXT: vmovaps 64(%r8), %xmm5 ; AVX-NEXT: vunpcklps {{.*#+}} xmm0 = xmm5[0],xmm4[0],xmm5[1],xmm4[1] ; AVX-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm2 -; AVX-NEXT: vmovaps 32(%rax), %xmm0 -; AVX-NEXT: vmovaps 32(%r10), %xmm1 +; AVX-NEXT: vmovaps 64(%rax), %xmm0 +; AVX-NEXT: vmovaps 64(%r10), %xmm1 ; AVX-NEXT: vunpcklps {{.*#+}} xmm6 = xmm1[0],xmm0[0],xmm1[1],xmm0[1] ; AVX-NEXT: vshufps {{.*#+}} xmm3 = xmm6[0,1,0,1] ; AVX-NEXT: vinsertf128 $1, %xmm3, %ymm0, %ymm3 ; AVX-NEXT: vblendps {{.*#+}} ymm7 = ymm2[0,1,2,3,4,5],ymm3[6,7] -; AVX-NEXT: vmovaps 32(%rcx), %xmm2 -; AVX-NEXT: vmovaps 32(%rdx), %xmm3 +; AVX-NEXT: vmovaps 64(%rcx), %xmm2 +; AVX-NEXT: vmovaps 64(%rdx), %xmm3 ; AVX-NEXT: vunpcklps {{.*#+}} xmm8 = xmm3[0],xmm2[0],xmm3[1],xmm2[1] -; AVX-NEXT: vmovaps 32(%rsi), %xmm9 -; AVX-NEXT: vmovaps 32(%rdi), %xmm10 +; AVX-NEXT: vmovaps 64(%rsi), %xmm9 +; AVX-NEXT: vmovaps 64(%rdi), %xmm10 ; AVX-NEXT: vunpcklps {{.*#+}} xmm11 = xmm10[0],xmm9[0],xmm10[1],xmm9[1] ; AVX-NEXT: vmovlhps {{.*#+}} xmm11 = xmm11[0],xmm8[0] ; AVX-NEXT: vblendps {{.*#+}} ymm7 = ymm11[0,1,2,3],ymm7[4,5,6,7] @@ -4804,32 +4749,81 @@ define void @store_i32_stride8_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7] ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm0[4,5,6,7] ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vmovaps 64(%r9), %xmm4 -; AVX-NEXT: vmovaps 64(%r8), %xmm5 +; AVX-NEXT: vmovaps 64(%r8), %ymm0 +; AVX-NEXT: vmovaps 64(%r9), %ymm1 +; AVX-NEXT: vmovaps 64(%r10), %ymm2 +; AVX-NEXT: vmovaps 64(%rax), %ymm3 +; AVX-NEXT: vunpcklpd {{.*#+}} ymm4 = ymm3[0],ymm2[0],ymm3[2],ymm2[2] +; AVX-NEXT: vunpcklps {{.*#+}} ymm5 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5] +; AVX-NEXT: vshufps {{.*#+}} ymm6 = ymm5[0,1],ymm4[2,0],ymm5[4,5],ymm4[6,4] +; AVX-NEXT: vmovaps 64(%rdx), %ymm4 +; AVX-NEXT: vmovaps 64(%rcx), %ymm5 +; AVX-NEXT: vunpcklpd {{.*#+}} ymm7 = ymm5[0],ymm4[0],ymm5[2],ymm4[2] +; AVX-NEXT: vextractf128 $1, %ymm7, %xmm7 +; AVX-NEXT: vmovaps 64(%rdi), %ymm8 +; AVX-NEXT: vmovaps 64(%rsi), %ymm9 +; AVX-NEXT: vunpcklps {{.*#+}} ymm10 = ymm8[0],ymm9[0],ymm8[1],ymm9[1],ymm8[4],ymm9[4],ymm8[5],ymm9[5] +; AVX-NEXT: vextractf128 $1, %ymm10, %xmm10 +; AVX-NEXT: vshufps {{.*#+}} xmm7 = xmm10[0,1],xmm7[2,0] +; AVX-NEXT: vblendps {{.*#+}} ymm6 = ymm7[0,1,2,3],ymm6[4,5,6,7] +; AVX-NEXT: vmovups %ymm6, (%rsp) # 32-byte Spill +; AVX-NEXT: vunpcklps {{.*#+}} ymm6 = ymm2[0],ymm3[0],ymm2[1],ymm3[1],ymm2[4],ymm3[4],ymm2[5],ymm3[5] +; AVX-NEXT: vshufps {{.*#+}} ymm7 = ymm1[1,0],ymm0[1,0],ymm1[5,4],ymm0[5,4] +; AVX-NEXT: vshufps {{.*#+}} ymm6 = ymm7[2,0],ymm6[2,3],ymm7[6,4],ymm6[6,7] +; AVX-NEXT: vunpcklps {{.*#+}} ymm7 = ymm4[0],ymm5[0],ymm4[1],ymm5[1],ymm4[4],ymm5[4],ymm4[5],ymm5[5] +; AVX-NEXT: vextractf128 $1, %ymm7, %xmm7 +; AVX-NEXT: vshufps {{.*#+}} ymm10 = ymm9[1,0],ymm8[1,0],ymm9[5,4],ymm8[5,4] +; AVX-NEXT: vextractf128 $1, %ymm10, %xmm10 +; AVX-NEXT: vshufps {{.*#+}} xmm7 = xmm10[2,0],xmm7[2,3] +; AVX-NEXT: vblendps {{.*#+}} ymm6 = ymm7[0,1,2,3],ymm6[4,5,6,7] +; AVX-NEXT: vmovups %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vunpckhpd {{.*#+}} ymm6 = ymm3[1],ymm2[1],ymm3[3],ymm2[3] +; AVX-NEXT: vunpckhps {{.*#+}} ymm7 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7] +; AVX-NEXT: vshufps {{.*#+}} ymm6 = ymm7[0,1],ymm6[2,0],ymm7[4,5],ymm6[6,4] +; AVX-NEXT: vunpckhpd {{.*#+}} ymm7 = ymm5[1],ymm4[1],ymm5[3],ymm4[3] +; AVX-NEXT: vextractf128 $1, %ymm7, %xmm7 +; AVX-NEXT: vunpckhps {{.*#+}} ymm10 = ymm8[2],ymm9[2],ymm8[3],ymm9[3],ymm8[6],ymm9[6],ymm8[7],ymm9[7] +; AVX-NEXT: vextractf128 $1, %ymm10, %xmm10 +; AVX-NEXT: vshufps {{.*#+}} xmm7 = xmm10[0,1],xmm7[2,0] +; AVX-NEXT: vblendps {{.*#+}} ymm6 = ymm7[0,1,2,3],ymm6[4,5,6,7] +; AVX-NEXT: vmovups %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vunpckhps {{.*#+}} ymm2 = ymm2[2],ymm3[2],ymm2[3],ymm3[3],ymm2[6],ymm3[6],ymm2[7],ymm3[7] +; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm1[3,0],ymm0[3,0],ymm1[7,4],ymm0[7,4] +; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm0[2,0],ymm2[2,3],ymm0[6,4],ymm2[6,7] +; AVX-NEXT: vunpckhps {{.*#+}} ymm1 = ymm4[2],ymm5[2],ymm4[3],ymm5[3],ymm4[6],ymm5[6],ymm4[7],ymm5[7] +; AVX-NEXT: vshufps {{.*#+}} ymm2 = ymm9[3,0],ymm8[3,0],ymm9[7,4],ymm8[7,4] +; AVX-NEXT: vextractf128 $1, %ymm1, %xmm1 +; AVX-NEXT: vextractf128 $1, %ymm2, %xmm2 +; AVX-NEXT: vshufps {{.*#+}} xmm1 = xmm2[2,0],xmm1[2,3] +; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7] +; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vmovaps 96(%r9), %xmm4 +; AVX-NEXT: vmovaps 96(%r8), %xmm5 ; AVX-NEXT: vunpcklps {{.*#+}} xmm0 = xmm5[0],xmm4[0],xmm5[1],xmm4[1] ; AVX-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm2 -; AVX-NEXT: vmovaps 64(%rax), %xmm9 -; AVX-NEXT: vmovaps 64(%r10), %xmm1 +; AVX-NEXT: vmovaps 96(%rax), %xmm9 +; AVX-NEXT: vmovaps 96(%r10), %xmm1 ; AVX-NEXT: vunpcklps {{.*#+}} xmm6 = xmm1[0],xmm9[0],xmm1[1],xmm9[1] ; AVX-NEXT: vshufps {{.*#+}} xmm3 = xmm6[0,1,0,1] ; AVX-NEXT: vinsertf128 $1, %xmm3, %ymm0, %ymm3 ; AVX-NEXT: vblendps {{.*#+}} ymm10 = ymm2[0,1,2,3,4,5],ymm3[6,7] -; AVX-NEXT: vmovaps 64(%rcx), %xmm2 -; AVX-NEXT: vmovaps 64(%rdx), %xmm3 +; AVX-NEXT: vmovaps 96(%rcx), %xmm2 +; AVX-NEXT: vmovaps 96(%rdx), %xmm3 ; AVX-NEXT: vunpcklps {{.*#+}} xmm8 = xmm3[0],xmm2[0],xmm3[1],xmm2[1] -; AVX-NEXT: vmovaps 64(%rsi), %xmm0 -; AVX-NEXT: vmovaps 64(%rdi), %xmm7 -; AVX-NEXT: vunpcklps {{.*#+}} xmm15 = xmm7[0],xmm0[0],xmm7[1],xmm0[1] -; AVX-NEXT: vmovlhps {{.*#+}} xmm15 = xmm15[0],xmm8[0] -; AVX-NEXT: vblendps {{.*#+}} ymm13 = ymm15[0,1,2,3],ymm10[4,5,6,7] -; AVX-NEXT: vshufps {{.*#+}} xmm15 = xmm7[1,1,1,1] -; AVX-NEXT: vblendps {{.*#+}} xmm15 = xmm15[0],xmm0[1],xmm15[2,3] -; AVX-NEXT: vblendps {{.*#+}} xmm8 = xmm15[0,1],xmm8[2,3] +; AVX-NEXT: vmovaps 96(%rsi), %xmm0 +; AVX-NEXT: vmovaps 96(%rdi), %xmm7 +; AVX-NEXT: vunpcklps {{.*#+}} xmm11 = xmm7[0],xmm0[0],xmm7[1],xmm0[1] +; AVX-NEXT: vmovlhps {{.*#+}} xmm11 = xmm11[0],xmm8[0] +; AVX-NEXT: vblendps {{.*#+}} ymm10 = ymm11[0,1,2,3],ymm10[4,5,6,7] +; AVX-NEXT: vmovups %ymm10, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vshufps {{.*#+}} xmm10 = xmm7[1,1,1,1] +; AVX-NEXT: vblendps {{.*#+}} xmm10 = xmm10[0],xmm0[1],xmm10[2,3] +; AVX-NEXT: vblendps {{.*#+}} xmm8 = xmm10[0,1],xmm8[2,3] ; AVX-NEXT: vinsertf128 $1, %xmm6, %ymm0, %ymm6 -; AVX-NEXT: vshufps {{.*#+}} xmm15 = xmm5[1,1,1,1] -; AVX-NEXT: vblendps {{.*#+}} xmm15 = xmm15[0],xmm4[1],xmm15[2,3] -; AVX-NEXT: vinsertf128 $1, %xmm15, %ymm0, %ymm15 -; AVX-NEXT: vblendps {{.*#+}} ymm6 = ymm15[0,1,2,3,4,5],ymm6[6,7] +; AVX-NEXT: vshufps {{.*#+}} xmm10 = xmm5[1,1,1,1] +; AVX-NEXT: vblendps {{.*#+}} xmm10 = xmm10[0],xmm4[1],xmm10[2,3] +; AVX-NEXT: vinsertf128 $1, %xmm10, %ymm0, %ymm10 +; AVX-NEXT: vblendps {{.*#+}} ymm6 = ymm10[0,1,2,3,4,5],ymm6[6,7] ; AVX-NEXT: vblendps {{.*#+}} ymm12 = ymm8[0,1,2,3],ymm6[4,5,6,7] ; AVX-NEXT: vunpckhps {{.*#+}} xmm0 = xmm7[2],xmm0[2],xmm7[3],xmm0[3] ; AVX-NEXT: vunpckhps {{.*#+}} xmm4 = xmm5[2],xmm4[2],xmm5[3],xmm4[3] @@ -4850,94 +4844,77 @@ define void @store_i32_stride8_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm2 ; AVX-NEXT: vblendps {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5],ymm1[6,7] ; AVX-NEXT: vblendps {{.*#+}} ymm10 = ymm0[0,1,2,3],ymm1[4,5,6,7] -; AVX-NEXT: vmovaps 96(%r9), %xmm3 -; AVX-NEXT: vmovaps 96(%r8), %xmm9 -; AVX-NEXT: vunpcklps {{.*#+}} xmm0 = xmm9[0],xmm3[0],xmm9[1],xmm3[1] -; AVX-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0 -; AVX-NEXT: vmovaps 96(%rax), %xmm7 -; AVX-NEXT: vmovaps 96(%r10), %xmm6 -; AVX-NEXT: vunpcklps {{.*#+}} xmm15 = xmm6[0],xmm7[0],xmm6[1],xmm7[1] -; AVX-NEXT: vshufps {{.*#+}} xmm2 = xmm15[0,1,0,1] -; AVX-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm2 -; AVX-NEXT: vblendps {{.*#+}} ymm8 = ymm0[0,1,2,3,4,5],ymm2[6,7] -; AVX-NEXT: vmovaps 96(%rcx), %xmm5 -; AVX-NEXT: vmovaps 96(%rdx), %xmm4 -; AVX-NEXT: vunpcklps {{.*#+}} xmm2 = xmm4[0],xmm5[0],xmm4[1],xmm5[1] -; AVX-NEXT: vmovaps 96(%rsi), %xmm1 -; AVX-NEXT: vmovaps 96(%rdi), %xmm0 -; AVX-NEXT: vunpcklps {{.*#+}} xmm14 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] -; AVX-NEXT: vmovlhps {{.*#+}} xmm14 = xmm14[0],xmm2[0] +; AVX-NEXT: vmovaps 96(%r8), %ymm7 +; AVX-NEXT: vmovaps 96(%r9), %ymm6 +; AVX-NEXT: vmovaps 96(%r10), %ymm5 +; AVX-NEXT: vmovaps 96(%rax), %ymm4 +; AVX-NEXT: vunpcklpd {{.*#+}} ymm0 = ymm4[0],ymm5[0],ymm4[2],ymm5[2] +; AVX-NEXT: vunpcklps {{.*#+}} ymm1 = ymm7[0],ymm6[0],ymm7[1],ymm6[1],ymm7[4],ymm6[4],ymm7[5],ymm6[5] +; AVX-NEXT: vshufps {{.*#+}} ymm8 = ymm1[0,1],ymm0[2,0],ymm1[4,5],ymm0[6,4] +; AVX-NEXT: vmovaps 96(%rdx), %ymm3 +; AVX-NEXT: vmovaps 96(%rcx), %ymm2 +; AVX-NEXT: vunpcklpd {{.*#+}} ymm9 = ymm2[0],ymm3[0],ymm2[2],ymm3[2] +; AVX-NEXT: vextractf128 $1, %ymm9, %xmm9 +; AVX-NEXT: vmovaps 96(%rdi), %ymm1 +; AVX-NEXT: vmovaps 96(%rsi), %ymm0 +; AVX-NEXT: vunpcklps {{.*#+}} ymm15 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[4],ymm0[4],ymm1[5],ymm0[5] +; AVX-NEXT: vextractf128 $1, %ymm15, %xmm15 +; AVX-NEXT: vshufps {{.*#+}} xmm9 = xmm15[0,1],xmm9[2,0] +; AVX-NEXT: vblendps {{.*#+}} ymm9 = ymm9[0,1,2,3],ymm8[4,5,6,7] +; AVX-NEXT: vunpcklps {{.*#+}} ymm8 = ymm5[0],ymm4[0],ymm5[1],ymm4[1],ymm5[4],ymm4[4],ymm5[5],ymm4[5] +; AVX-NEXT: vshufps {{.*#+}} ymm15 = ymm6[1,0],ymm7[1,0],ymm6[5,4],ymm7[5,4] +; AVX-NEXT: vshufps {{.*#+}} ymm8 = ymm15[2,0],ymm8[2,3],ymm15[6,4],ymm8[6,7] +; AVX-NEXT: vunpcklps {{.*#+}} ymm15 = ymm3[0],ymm2[0],ymm3[1],ymm2[1],ymm3[4],ymm2[4],ymm3[5],ymm2[5] +; AVX-NEXT: vextractf128 $1, %ymm15, %xmm15 +; AVX-NEXT: vshufps {{.*#+}} ymm14 = ymm0[1,0],ymm1[1,0],ymm0[5,4],ymm1[5,4] +; AVX-NEXT: vextractf128 $1, %ymm14, %xmm14 +; AVX-NEXT: vshufps {{.*#+}} xmm14 = xmm14[2,0],xmm15[2,3] ; AVX-NEXT: vblendps {{.*#+}} ymm8 = ymm14[0,1,2,3],ymm8[4,5,6,7] -; AVX-NEXT: vshufps {{.*#+}} xmm14 = xmm0[1,1,1,1] -; AVX-NEXT: vblendps {{.*#+}} xmm14 = xmm14[0],xmm1[1],xmm14[2,3] -; AVX-NEXT: vblendps {{.*#+}} xmm2 = xmm14[0,1],xmm2[2,3] -; AVX-NEXT: vinsertf128 $1, %xmm15, %ymm0, %ymm14 -; AVX-NEXT: vshufps {{.*#+}} xmm15 = xmm9[1,1,1,1] -; AVX-NEXT: vblendps {{.*#+}} xmm15 = xmm15[0],xmm3[1],xmm15[2,3] -; AVX-NEXT: vinsertf128 $1, %xmm15, %ymm0, %ymm15 -; AVX-NEXT: vblendps {{.*#+}} ymm14 = ymm15[0,1,2,3,4,5],ymm14[6,7] -; AVX-NEXT: vblendps {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm14[4,5,6,7] -; AVX-NEXT: vunpckhps {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3] -; AVX-NEXT: vunpckhps {{.*#+}} xmm1 = xmm9[2],xmm3[2],xmm9[3],xmm3[3] -; AVX-NEXT: vshufps {{.*#+}} xmm3 = xmm7[2,2,2,2] -; AVX-NEXT: vblendps {{.*#+}} xmm3 = xmm6[0,1,2],xmm3[3] -; AVX-NEXT: vinsertf128 $1, %xmm3, %ymm0, %ymm3 -; AVX-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm9 -; AVX-NEXT: vblendps {{.*#+}} ymm3 = ymm9[0,1,2,3,4,5],ymm3[6,7] -; AVX-NEXT: vshufps {{.*#+}} xmm9 = xmm5[2,2,2,2] -; AVX-NEXT: vblendps {{.*#+}} xmm9 = xmm4[0,1,2],xmm9[3] -; AVX-NEXT: vblendps {{.*#+}} xmm9 = xmm0[0,1],xmm9[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm3 = ymm9[0,1,2,3],ymm3[4,5,6,7] -; AVX-NEXT: vunpckhps {{.*#+}} xmm4 = xmm4[2],xmm5[2],xmm4[3],xmm5[3] -; AVX-NEXT: vunpckhpd {{.*#+}} xmm0 = xmm0[1],xmm4[1] -; AVX-NEXT: vunpckhps {{.*#+}} xmm4 = xmm6[2],xmm7[2],xmm6[3],xmm7[3] -; AVX-NEXT: vinsertf128 $1, %xmm4, %ymm0, %ymm4 -; AVX-NEXT: vshufps {{.*#+}} xmm1 = xmm1[2,3,2,3] -; AVX-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm1 -; AVX-NEXT: vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5],ymm4[6,7] -; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7] +; AVX-NEXT: vunpckhpd {{.*#+}} ymm14 = ymm4[1],ymm5[1],ymm4[3],ymm5[3] +; AVX-NEXT: vunpckhps {{.*#+}} ymm15 = ymm7[2],ymm6[2],ymm7[3],ymm6[3],ymm7[6],ymm6[6],ymm7[7],ymm6[7] +; AVX-NEXT: vshufps {{.*#+}} ymm14 = ymm15[0,1],ymm14[2,0],ymm15[4,5],ymm14[6,4] +; AVX-NEXT: vunpckhpd {{.*#+}} ymm15 = ymm2[1],ymm3[1],ymm2[3],ymm3[3] +; AVX-NEXT: vextractf128 $1, %ymm15, %xmm15 +; AVX-NEXT: vunpckhps {{.*#+}} ymm13 = ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[6],ymm0[6],ymm1[7],ymm0[7] +; AVX-NEXT: vextractf128 $1, %ymm13, %xmm13 +; AVX-NEXT: vshufps {{.*#+}} xmm13 = xmm13[0,1],xmm15[2,0] +; AVX-NEXT: vblendps {{.*#+}} ymm13 = ymm13[0,1,2,3],ymm14[4,5,6,7] +; AVX-NEXT: vunpckhps {{.*#+}} ymm4 = ymm5[2],ymm4[2],ymm5[3],ymm4[3],ymm5[6],ymm4[6],ymm5[7],ymm4[7] +; AVX-NEXT: vshufps {{.*#+}} ymm5 = ymm6[3,0],ymm7[3,0],ymm6[7,4],ymm7[7,4] +; AVX-NEXT: vshufps {{.*#+}} ymm4 = ymm5[2,0],ymm4[2,3],ymm5[6,4],ymm4[6,7] +; AVX-NEXT: vunpckhps {{.*#+}} ymm2 = ymm3[2],ymm2[2],ymm3[3],ymm2[3],ymm3[6],ymm2[6],ymm3[7],ymm2[7] +; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm0[3,0],ymm1[3,0],ymm0[7,4],ymm1[7,4] +; AVX-NEXT: vextractf128 $1, %ymm2, %xmm1 +; AVX-NEXT: vextractf128 $1, %ymm0, %xmm0 +; AVX-NEXT: vshufps {{.*#+}} xmm0 = xmm0[2,0],xmm1[2,3] +; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm4[4,5,6,7] ; AVX-NEXT: movq {{[0-9]+}}(%rsp), %rax -; AVX-NEXT: vmovaps %ymm0, 864(%rax) -; AVX-NEXT: vmovaps %ymm3, 832(%rax) -; AVX-NEXT: vmovaps %ymm2, 800(%rax) -; AVX-NEXT: vmovaps %ymm8, 768(%rax) -; AVX-NEXT: vmovaps %ymm10, 608(%rax) -; AVX-NEXT: vmovaps %ymm11, 576(%rax) -; AVX-NEXT: vmovaps %ymm12, 544(%rax) -; AVX-NEXT: vmovaps %ymm13, 512(%rax) -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm0, 352(%rax) -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm0, 320(%rax) -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm0, 288(%rax) -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm0, 256(%rax) -; AVX-NEXT: vmovups (%rsp), %ymm0 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm0, 96(%rax) -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm0, 64(%rax) -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm0, 32(%rax) -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm0, (%rax) -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload ; AVX-NEXT: vmovaps %ymm0, 992(%rax) +; AVX-NEXT: vmovaps %ymm13, 960(%rax) +; AVX-NEXT: vmovaps %ymm8, 928(%rax) +; AVX-NEXT: vmovaps %ymm9, 896(%rax) +; AVX-NEXT: vmovaps %ymm10, 864(%rax) +; AVX-NEXT: vmovaps %ymm11, 832(%rax) +; AVX-NEXT: vmovaps %ymm12, 800(%rax) ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm0, 960(%rax) -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm0, 928(%rax) -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm0, 896(%rax) +; AVX-NEXT: vmovaps %ymm0, 768(%rax) ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload ; AVX-NEXT: vmovaps %ymm0, 736(%rax) ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload ; AVX-NEXT: vmovaps %ymm0, 704(%rax) ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload ; AVX-NEXT: vmovaps %ymm0, 672(%rax) -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload +; AVX-NEXT: vmovups (%rsp), %ymm0 # 32-byte Reload ; AVX-NEXT: vmovaps %ymm0, 640(%rax) ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm0, 608(%rax) +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm0, 576(%rax) +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm0, 544(%rax) +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm0, 512(%rax) +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload ; AVX-NEXT: vmovaps %ymm0, 480(%rax) ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload ; AVX-NEXT: vmovaps %ymm0, 448(%rax) @@ -4946,6 +4923,14 @@ define void @store_i32_stride8_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload ; AVX-NEXT: vmovaps %ymm0, 384(%rax) ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm0, 352(%rax) +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm0, 320(%rax) +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm0, 288(%rax) +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm0, 256(%rax) +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload ; AVX-NEXT: vmovaps %ymm0, 224(%rax) ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload ; AVX-NEXT: vmovaps %ymm0, 192(%rax) @@ -4953,7 +4938,15 @@ define void @store_i32_stride8_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX-NEXT: vmovaps %ymm0, 160(%rax) ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload ; AVX-NEXT: vmovaps %ymm0, 128(%rax) -; AVX-NEXT: addq $648, %rsp # imm = 0x288 +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm0, 96(%rax) +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm0, 64(%rax) +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm0, 32(%rax) +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm0, (%rax) +; AVX-NEXT: addq $680, %rsp # imm = 0x2A8 ; AVX-NEXT: vzeroupper ; AVX-NEXT: retq ; @@ -9726,490 +9719,220 @@ define void @store_i32_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; ; AVX-LABEL: store_i32_stride8_vf64: ; AVX: # %bb.0: -; AVX-NEXT: subq $1672, %rsp # imm = 0x688 +; AVX-NEXT: subq $1704, %rsp # imm = 0x6A8 ; AVX-NEXT: movq {{[0-9]+}}(%rsp), %rax ; AVX-NEXT: movq {{[0-9]+}}(%rsp), %r10 -; AVX-NEXT: vmovaps (%rdi), %ymm3 -; AVX-NEXT: vmovaps 32(%rdi), %ymm0 -; AVX-NEXT: vmovaps (%rsi), %ymm4 -; AVX-NEXT: vmovaps 32(%rsi), %ymm1 -; AVX-NEXT: vmovaps 32(%rdx), %ymm2 -; AVX-NEXT: vmovaps (%rdx), %ymm5 -; AVX-NEXT: vmovaps (%rcx), %ymm6 -; AVX-NEXT: vmovaps (%r8), %ymm7 -; AVX-NEXT: vmovaps (%r9), %ymm9 -; AVX-NEXT: vmovaps (%r10), %ymm11 -; AVX-NEXT: vmovaps (%rax), %ymm12 -; AVX-NEXT: vunpcklpd {{.*#+}} ymm8 = ymm12[0],ymm11[0],ymm12[2],ymm11[2] -; AVX-NEXT: vunpcklps {{.*#+}} ymm10 = ymm7[0],ymm9[0],ymm7[1],ymm9[1],ymm7[4],ymm9[4],ymm7[5],ymm9[5] -; AVX-NEXT: vshufps {{.*#+}} ymm8 = ymm10[0,1],ymm8[2,0],ymm10[4,5],ymm8[6,4] -; AVX-NEXT: vunpcklps {{.*#+}} ymm10 = ymm3[0],ymm4[0],ymm3[1],ymm4[1],ymm3[4],ymm4[4],ymm3[5],ymm4[5] -; AVX-NEXT: vextractf128 $1, %ymm10, %xmm10 -; AVX-NEXT: vunpcklpd {{.*#+}} ymm13 = ymm6[0],ymm5[0],ymm6[2],ymm5[2] -; AVX-NEXT: vshufps {{.*#+}} ymm13 = ymm13[0,1,2,0,4,5,6,4] -; AVX-NEXT: vextractf128 $1, %ymm13, %xmm13 -; AVX-NEXT: vblendps {{.*#+}} xmm10 = xmm10[0,1],xmm13[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm8 = ymm10[0,1,2,3],ymm8[4,5,6,7] -; AVX-NEXT: vmovups %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vunpcklps {{.*#+}} ymm8 = ymm11[0],ymm12[0],ymm11[1],ymm12[1],ymm11[4],ymm12[4],ymm11[5],ymm12[5] -; AVX-NEXT: vshufps {{.*#+}} ymm10 = ymm9[1,0],ymm7[1,0],ymm9[5,4],ymm7[5,4] -; AVX-NEXT: vshufps {{.*#+}} ymm8 = ymm10[2,0],ymm8[2,3],ymm10[6,4],ymm8[6,7] -; AVX-NEXT: vunpcklps {{.*#+}} ymm10 = ymm5[0],ymm6[0],ymm5[1],ymm6[1],ymm5[4],ymm6[4],ymm5[5],ymm6[5] -; AVX-NEXT: vextractf128 $1, %ymm10, %xmm10 -; AVX-NEXT: vshufps {{.*#+}} ymm13 = ymm4[1,0],ymm3[1,0],ymm4[5,4],ymm3[5,4] -; AVX-NEXT: vshufps {{.*#+}} ymm13 = ymm13[2,0,2,3,6,4,6,7] -; AVX-NEXT: vextractf128 $1, %ymm13, %xmm13 -; AVX-NEXT: vblendps {{.*#+}} xmm10 = xmm13[0,1],xmm10[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm8 = ymm10[0,1,2,3],ymm8[4,5,6,7] -; AVX-NEXT: vmovups %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vunpckhpd {{.*#+}} ymm8 = ymm12[1],ymm11[1],ymm12[3],ymm11[3] -; AVX-NEXT: vunpckhps {{.*#+}} ymm10 = ymm7[2],ymm9[2],ymm7[3],ymm9[3],ymm7[6],ymm9[6],ymm7[7],ymm9[7] -; AVX-NEXT: vshufps {{.*#+}} ymm10 = ymm10[0,1],ymm8[2,0],ymm10[4,5],ymm8[6,4] -; AVX-NEXT: vunpckhps {{.*#+}} ymm8 = ymm3[2],ymm4[2],ymm3[3],ymm4[3],ymm3[6],ymm4[6],ymm3[7],ymm4[7] -; AVX-NEXT: vextractf128 $1, %ymm8, %xmm8 -; AVX-NEXT: vunpckhpd {{.*#+}} ymm13 = ymm6[1],ymm5[1],ymm6[3],ymm5[3] -; AVX-NEXT: vshufps {{.*#+}} ymm13 = ymm13[0,1,2,0,4,5,6,4] -; AVX-NEXT: vextractf128 $1, %ymm13, %xmm13 -; AVX-NEXT: vblendps {{.*#+}} xmm13 = xmm8[0,1],xmm13[2,3] -; AVX-NEXT: vmovaps 32(%rcx), %ymm8 +; AVX-NEXT: vmovaps (%rdi), %ymm0 +; AVX-NEXT: vmovaps (%rsi), %ymm1 +; AVX-NEXT: vmovaps (%rdx), %ymm2 +; AVX-NEXT: vmovaps (%rcx), %ymm3 +; AVX-NEXT: vmovaps (%rcx), %xmm5 +; AVX-NEXT: vmovaps (%rdx), %xmm7 +; AVX-NEXT: vunpcklps {{.*#+}} xmm4 = xmm7[0],xmm5[0],xmm7[1],xmm5[1] +; AVX-NEXT: vmovaps (%rsi), %xmm10 +; AVX-NEXT: vmovaps (%rdi), %xmm12 +; AVX-NEXT: vmovaps (%r9), %xmm11 +; AVX-NEXT: vmovaps (%r8), %xmm13 +; AVX-NEXT: vunpcklps {{.*#+}} xmm6 = xmm13[0],xmm11[0],xmm13[1],xmm11[1] +; AVX-NEXT: vinsertf128 $1, %xmm6, %ymm0, %ymm6 +; AVX-NEXT: vmovaps (%rax), %xmm8 +; AVX-NEXT: vmovaps (%r10), %xmm9 +; AVX-NEXT: vunpcklps {{.*#+}} xmm14 = xmm9[0],xmm8[0],xmm9[1],xmm8[1] +; AVX-NEXT: vshufps {{.*#+}} xmm15 = xmm14[0,1,0,1] +; AVX-NEXT: vinsertf128 $1, %xmm15, %ymm0, %ymm15 +; AVX-NEXT: vblendps {{.*#+}} ymm6 = ymm6[0,1,2,3,4,5],ymm15[6,7] +; AVX-NEXT: vunpcklps {{.*#+}} xmm15 = xmm12[0],xmm10[0],xmm12[1],xmm10[1] +; AVX-NEXT: vmovlhps {{.*#+}} xmm15 = xmm15[0],xmm4[0] +; AVX-NEXT: vblendps {{.*#+}} ymm6 = ymm15[0,1,2,3],ymm6[4,5,6,7] +; AVX-NEXT: vmovups %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vshufps {{.*#+}} xmm6 = xmm12[1,1,1,1] +; AVX-NEXT: vblendps {{.*#+}} xmm6 = xmm6[0],xmm10[1],xmm6[2,3] +; AVX-NEXT: vblendps {{.*#+}} xmm6 = xmm6[0,1],xmm4[2,3] +; AVX-NEXT: vinsertf128 $1, %xmm14, %ymm0, %ymm4 +; AVX-NEXT: vshufps {{.*#+}} xmm14 = xmm13[1,1,1,1] +; AVX-NEXT: vblendps {{.*#+}} xmm14 = xmm14[0],xmm11[1],xmm14[2,3] +; AVX-NEXT: vinsertf128 $1, %xmm14, %ymm0, %ymm14 +; AVX-NEXT: vblendps {{.*#+}} ymm14 = ymm14[0,1,2,3,4,5],ymm4[6,7] +; AVX-NEXT: vmovaps (%r8), %ymm4 +; AVX-NEXT: vblendps {{.*#+}} ymm6 = ymm6[0,1,2,3],ymm14[4,5,6,7] +; AVX-NEXT: vmovups %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vmovaps (%r9), %ymm6 +; AVX-NEXT: vunpckhps {{.*#+}} xmm12 = xmm12[2],xmm10[2],xmm12[3],xmm10[3] +; AVX-NEXT: vunpckhps {{.*#+}} xmm11 = xmm13[2],xmm11[2],xmm13[3],xmm11[3] +; AVX-NEXT: vinsertf128 $1, %xmm11, %ymm0, %ymm10 +; AVX-NEXT: vshufps {{.*#+}} xmm13 = xmm8[2,2,2,2] +; AVX-NEXT: vblendps {{.*#+}} xmm13 = xmm9[0,1,2],xmm13[3] +; AVX-NEXT: vinsertf128 $1, %xmm13, %ymm0, %ymm13 +; AVX-NEXT: vblendps {{.*#+}} ymm10 = ymm10[0,1,2,3,4,5],ymm13[6,7] +; AVX-NEXT: vshufps {{.*#+}} xmm13 = xmm5[2,2,2,2] +; AVX-NEXT: vblendps {{.*#+}} xmm13 = xmm7[0,1,2],xmm13[3] +; AVX-NEXT: vblendps {{.*#+}} xmm13 = xmm12[0,1],xmm13[2,3] ; AVX-NEXT: vblendps {{.*#+}} ymm10 = ymm13[0,1,2,3],ymm10[4,5,6,7] ; AVX-NEXT: vmovups %ymm10, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vmovaps 32(%r8), %ymm10 -; AVX-NEXT: vunpckhps {{.*#+}} ymm12 = ymm11[2],ymm12[2],ymm11[3],ymm12[3],ymm11[6],ymm12[6],ymm11[7],ymm12[7] -; AVX-NEXT: vmovaps 32(%r9), %ymm11 -; AVX-NEXT: vshufps {{.*#+}} ymm9 = ymm9[3,0],ymm7[3,0],ymm9[7,4],ymm7[7,4] -; AVX-NEXT: vmovaps 32(%r10), %ymm7 -; AVX-NEXT: vshufps {{.*#+}} ymm12 = ymm9[2,0],ymm12[2,3],ymm9[6,4],ymm12[6,7] -; AVX-NEXT: vmovaps 32(%rax), %ymm9 -; AVX-NEXT: vunpckhps {{.*#+}} ymm5 = ymm5[2],ymm6[2],ymm5[3],ymm6[3],ymm5[6],ymm6[6],ymm5[7],ymm6[7] -; AVX-NEXT: vextractf128 $1, %ymm5, %xmm5 -; AVX-NEXT: vshufps {{.*#+}} ymm3 = ymm4[3,0],ymm3[3,0],ymm4[7,4],ymm3[7,4] -; AVX-NEXT: vshufps {{.*#+}} ymm3 = ymm3[2,0,2,3,6,4,6,7] -; AVX-NEXT: vextractf128 $1, %ymm3, %xmm3 -; AVX-NEXT: vblendps {{.*#+}} xmm3 = xmm3[0,1],xmm5[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm3 = ymm3[0,1,2,3],ymm12[4,5,6,7] -; AVX-NEXT: vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vunpcklpd {{.*#+}} ymm3 = ymm9[0],ymm7[0],ymm9[2],ymm7[2] -; AVX-NEXT: vunpcklps {{.*#+}} ymm4 = ymm10[0],ymm11[0],ymm10[1],ymm11[1],ymm10[4],ymm11[4],ymm10[5],ymm11[5] -; AVX-NEXT: vshufps {{.*#+}} ymm3 = ymm4[0,1],ymm3[2,0],ymm4[4,5],ymm3[6,4] -; AVX-NEXT: vunpcklps {{.*#+}} ymm4 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5] -; AVX-NEXT: vextractf128 $1, %ymm4, %xmm4 -; AVX-NEXT: vunpcklpd {{.*#+}} ymm5 = ymm8[0],ymm2[0],ymm8[2],ymm2[2] -; AVX-NEXT: vshufps {{.*#+}} ymm5 = ymm5[0,1,2,0,4,5,6,4] -; AVX-NEXT: vextractf128 $1, %ymm5, %xmm5 -; AVX-NEXT: vblendps {{.*#+}} xmm4 = xmm4[0,1],xmm5[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7] -; AVX-NEXT: vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vunpcklps {{.*#+}} ymm3 = ymm7[0],ymm9[0],ymm7[1],ymm9[1],ymm7[4],ymm9[4],ymm7[5],ymm9[5] -; AVX-NEXT: vshufps {{.*#+}} ymm4 = ymm11[1,0],ymm10[1,0],ymm11[5,4],ymm10[5,4] -; AVX-NEXT: vshufps {{.*#+}} ymm3 = ymm4[2,0],ymm3[2,3],ymm4[6,4],ymm3[6,7] -; AVX-NEXT: vunpcklps {{.*#+}} ymm4 = ymm2[0],ymm8[0],ymm2[1],ymm8[1],ymm2[4],ymm8[4],ymm2[5],ymm8[5] -; AVX-NEXT: vextractf128 $1, %ymm4, %xmm4 -; AVX-NEXT: vshufps {{.*#+}} ymm5 = ymm1[1,0],ymm0[1,0],ymm1[5,4],ymm0[5,4] -; AVX-NEXT: vshufps {{.*#+}} ymm5 = ymm5[2,0,2,3,6,4,6,7] -; AVX-NEXT: vextractf128 $1, %ymm5, %xmm5 -; AVX-NEXT: vblendps {{.*#+}} xmm4 = xmm5[0,1],xmm4[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7] -; AVX-NEXT: vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vunpckhpd {{.*#+}} ymm3 = ymm9[1],ymm7[1],ymm9[3],ymm7[3] -; AVX-NEXT: vunpckhps {{.*#+}} ymm4 = ymm10[2],ymm11[2],ymm10[3],ymm11[3],ymm10[6],ymm11[6],ymm10[7],ymm11[7] -; AVX-NEXT: vshufps {{.*#+}} ymm3 = ymm4[0,1],ymm3[2,0],ymm4[4,5],ymm3[6,4] -; AVX-NEXT: vunpckhps {{.*#+}} ymm4 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7] -; AVX-NEXT: vextractf128 $1, %ymm4, %xmm4 -; AVX-NEXT: vunpckhpd {{.*#+}} ymm5 = ymm8[1],ymm2[1],ymm8[3],ymm2[3] -; AVX-NEXT: vshufps {{.*#+}} ymm5 = ymm5[0,1,2,0,4,5,6,4] -; AVX-NEXT: vextractf128 $1, %ymm5, %xmm5 -; AVX-NEXT: vblendps {{.*#+}} xmm4 = xmm4[0,1],xmm5[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7] -; AVX-NEXT: vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vmovaps 64(%rdi), %ymm3 -; AVX-NEXT: vunpckhps {{.*#+}} ymm6 = ymm7[2],ymm9[2],ymm7[3],ymm9[3],ymm7[6],ymm9[6],ymm7[7],ymm9[7] -; AVX-NEXT: vmovaps 64(%rsi), %ymm4 -; AVX-NEXT: vshufps {{.*#+}} ymm7 = ymm11[3,0],ymm10[3,0],ymm11[7,4],ymm10[7,4] -; AVX-NEXT: vmovaps 64(%r8), %ymm5 -; AVX-NEXT: vshufps {{.*#+}} ymm7 = ymm7[2,0],ymm6[2,3],ymm7[6,4],ymm6[6,7] -; AVX-NEXT: vmovaps 64(%r9), %ymm6 -; AVX-NEXT: vunpckhps {{.*#+}} ymm8 = ymm2[2],ymm8[2],ymm2[3],ymm8[3],ymm2[6],ymm8[6],ymm2[7],ymm8[7] -; AVX-NEXT: vmovaps 64(%r10), %ymm2 -; AVX-NEXT: vshufps {{.*#+}} ymm1 = ymm1[3,0],ymm0[3,0],ymm1[7,4],ymm0[7,4] -; AVX-NEXT: vmovaps 64(%rax), %ymm0 -; AVX-NEXT: vextractf128 $1, %ymm8, %xmm8 -; AVX-NEXT: vshufps {{.*#+}} ymm1 = ymm1[2,0,2,3,6,4,6,7] -; AVX-NEXT: vextractf128 $1, %ymm1, %xmm1 -; AVX-NEXT: vblendps {{.*#+}} xmm1 = xmm1[0,1],xmm8[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm7[4,5,6,7] -; AVX-NEXT: vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vunpcklpd {{.*#+}} ymm1 = ymm0[0],ymm2[0],ymm0[2],ymm2[2] -; AVX-NEXT: vunpcklps {{.*#+}} ymm7 = ymm5[0],ymm6[0],ymm5[1],ymm6[1],ymm5[4],ymm6[4],ymm5[5],ymm6[5] -; AVX-NEXT: vshufps {{.*#+}} ymm7 = ymm7[0,1],ymm1[2,0],ymm7[4,5],ymm1[6,4] -; AVX-NEXT: vunpcklps {{.*#+}} ymm1 = ymm3[0],ymm4[0],ymm3[1],ymm4[1],ymm3[4],ymm4[4],ymm3[5],ymm4[5] -; AVX-NEXT: vextractf128 $1, %ymm1, %xmm8 -; AVX-NEXT: vmovaps 64(%rdx), %ymm1 -; AVX-NEXT: vmovaps 64(%rcx), %ymm9 -; AVX-NEXT: vunpcklpd {{.*#+}} ymm10 = ymm9[0],ymm1[0],ymm9[2],ymm1[2] -; AVX-NEXT: vshufps {{.*#+}} ymm10 = ymm10[0,1,2,0,4,5,6,4] -; AVX-NEXT: vextractf128 $1, %ymm10, %xmm10 -; AVX-NEXT: vblendps {{.*#+}} xmm8 = xmm8[0,1],xmm10[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm7 = ymm8[0,1,2,3],ymm7[4,5,6,7] -; AVX-NEXT: vmovups %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vunpcklps {{.*#+}} ymm7 = ymm2[0],ymm0[0],ymm2[1],ymm0[1],ymm2[4],ymm0[4],ymm2[5],ymm0[5] -; AVX-NEXT: vshufps {{.*#+}} ymm8 = ymm6[1,0],ymm5[1,0],ymm6[5,4],ymm5[5,4] -; AVX-NEXT: vshufps {{.*#+}} ymm7 = ymm8[2,0],ymm7[2,3],ymm8[6,4],ymm7[6,7] -; AVX-NEXT: vunpcklps {{.*#+}} ymm8 = ymm1[0],ymm9[0],ymm1[1],ymm9[1],ymm1[4],ymm9[4],ymm1[5],ymm9[5] -; AVX-NEXT: vextractf128 $1, %ymm8, %xmm8 -; AVX-NEXT: vshufps {{.*#+}} ymm10 = ymm4[1,0],ymm3[1,0],ymm4[5,4],ymm3[5,4] -; AVX-NEXT: vshufps {{.*#+}} ymm10 = ymm10[2,0,2,3,6,4,6,7] -; AVX-NEXT: vextractf128 $1, %ymm10, %xmm10 -; AVX-NEXT: vblendps {{.*#+}} xmm8 = xmm10[0,1],xmm8[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm7 = ymm8[0,1,2,3],ymm7[4,5,6,7] -; AVX-NEXT: vmovups %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vunpckhpd {{.*#+}} ymm7 = ymm0[1],ymm2[1],ymm0[3],ymm2[3] -; AVX-NEXT: vunpckhps {{.*#+}} ymm8 = ymm5[2],ymm6[2],ymm5[3],ymm6[3],ymm5[6],ymm6[6],ymm5[7],ymm6[7] -; AVX-NEXT: vshufps {{.*#+}} ymm7 = ymm8[0,1],ymm7[2,0],ymm8[4,5],ymm7[6,4] -; AVX-NEXT: vunpckhps {{.*#+}} ymm8 = ymm3[2],ymm4[2],ymm3[3],ymm4[3],ymm3[6],ymm4[6],ymm3[7],ymm4[7] -; AVX-NEXT: vextractf128 $1, %ymm8, %xmm8 -; AVX-NEXT: vunpckhpd {{.*#+}} ymm10 = ymm9[1],ymm1[1],ymm9[3],ymm1[3] -; AVX-NEXT: vshufps {{.*#+}} ymm10 = ymm10[0,1,2,0,4,5,6,4] -; AVX-NEXT: vextractf128 $1, %ymm10, %xmm10 -; AVX-NEXT: vblendps {{.*#+}} xmm8 = xmm8[0,1],xmm10[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm7 = ymm8[0,1,2,3],ymm7[4,5,6,7] -; AVX-NEXT: vmovups %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vunpckhps {{.*#+}} ymm0 = ymm2[2],ymm0[2],ymm2[3],ymm0[3],ymm2[6],ymm0[6],ymm2[7],ymm0[7] -; AVX-NEXT: vshufps {{.*#+}} ymm2 = ymm6[3,0],ymm5[3,0],ymm6[7,4],ymm5[7,4] -; AVX-NEXT: vshufps {{.*#+}} ymm5 = ymm2[2,0],ymm0[2,3],ymm2[6,4],ymm0[6,7] -; AVX-NEXT: vunpckhps {{.*#+}} ymm2 = ymm1[2],ymm9[2],ymm1[3],ymm9[3],ymm1[6],ymm9[6],ymm1[7],ymm9[7] -; AVX-NEXT: vmovaps 96(%r8), %ymm0 -; AVX-NEXT: vshufps {{.*#+}} ymm3 = ymm4[3,0],ymm3[3,0],ymm4[7,4],ymm3[7,4] -; AVX-NEXT: vmovaps 96(%r9), %ymm1 -; AVX-NEXT: vextractf128 $1, %ymm2, %xmm2 -; AVX-NEXT: vshufps {{.*#+}} ymm3 = ymm3[2,0,2,3,6,4,6,7] -; AVX-NEXT: vextractf128 $1, %ymm3, %xmm3 -; AVX-NEXT: vblendps {{.*#+}} xmm3 = xmm3[0,1],xmm2[2,3] -; AVX-NEXT: vmovaps 96(%r10), %ymm2 -; AVX-NEXT: vblendps {{.*#+}} ymm3 = ymm3[0,1,2,3],ymm5[4,5,6,7] -; AVX-NEXT: vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vmovaps 96(%rax), %ymm5 -; AVX-NEXT: vunpcklpd {{.*#+}} ymm3 = ymm5[0],ymm2[0],ymm5[2],ymm2[2] -; AVX-NEXT: vunpcklps {{.*#+}} ymm4 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5] -; AVX-NEXT: vshufps {{.*#+}} ymm7 = ymm4[0,1],ymm3[2,0],ymm4[4,5],ymm3[6,4] -; AVX-NEXT: vmovaps 96(%rdi), %ymm3 -; AVX-NEXT: vmovaps 96(%rsi), %ymm4 -; AVX-NEXT: vunpcklps {{.*#+}} ymm6 = ymm3[0],ymm4[0],ymm3[1],ymm4[1],ymm3[4],ymm4[4],ymm3[5],ymm4[5] -; AVX-NEXT: vextractf128 $1, %ymm6, %xmm8 -; AVX-NEXT: vmovaps 96(%rdx), %ymm6 -; AVX-NEXT: vmovaps 96(%rcx), %ymm9 -; AVX-NEXT: vunpcklpd {{.*#+}} ymm10 = ymm9[0],ymm6[0],ymm9[2],ymm6[2] -; AVX-NEXT: vshufps {{.*#+}} ymm10 = ymm10[0,1,2,0,4,5,6,4] -; AVX-NEXT: vextractf128 $1, %ymm10, %xmm10 -; AVX-NEXT: vblendps {{.*#+}} xmm8 = xmm8[0,1],xmm10[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm7 = ymm8[0,1,2,3],ymm7[4,5,6,7] -; AVX-NEXT: vmovups %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vunpcklps {{.*#+}} ymm7 = ymm2[0],ymm5[0],ymm2[1],ymm5[1],ymm2[4],ymm5[4],ymm2[5],ymm5[5] -; AVX-NEXT: vshufps {{.*#+}} ymm8 = ymm1[1,0],ymm0[1,0],ymm1[5,4],ymm0[5,4] -; AVX-NEXT: vshufps {{.*#+}} ymm7 = ymm8[2,0],ymm7[2,3],ymm8[6,4],ymm7[6,7] -; AVX-NEXT: vunpcklps {{.*#+}} ymm8 = ymm6[0],ymm9[0],ymm6[1],ymm9[1],ymm6[4],ymm9[4],ymm6[5],ymm9[5] -; AVX-NEXT: vextractf128 $1, %ymm8, %xmm8 -; AVX-NEXT: vshufps {{.*#+}} ymm10 = ymm4[1,0],ymm3[1,0],ymm4[5,4],ymm3[5,4] -; AVX-NEXT: vshufps {{.*#+}} ymm10 = ymm10[2,0,2,3,6,4,6,7] -; AVX-NEXT: vextractf128 $1, %ymm10, %xmm10 -; AVX-NEXT: vblendps {{.*#+}} xmm8 = xmm10[0,1],xmm8[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm7 = ymm8[0,1,2,3],ymm7[4,5,6,7] -; AVX-NEXT: vmovups %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vunpckhpd {{.*#+}} ymm7 = ymm5[1],ymm2[1],ymm5[3],ymm2[3] -; AVX-NEXT: vunpckhps {{.*#+}} ymm8 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7] -; AVX-NEXT: vshufps {{.*#+}} ymm7 = ymm8[0,1],ymm7[2,0],ymm8[4,5],ymm7[6,4] -; AVX-NEXT: vunpckhps {{.*#+}} ymm8 = ymm3[2],ymm4[2],ymm3[3],ymm4[3],ymm3[6],ymm4[6],ymm3[7],ymm4[7] -; AVX-NEXT: vextractf128 $1, %ymm8, %xmm8 -; AVX-NEXT: vunpckhpd {{.*#+}} ymm10 = ymm9[1],ymm6[1],ymm9[3],ymm6[3] -; AVX-NEXT: vshufps {{.*#+}} ymm10 = ymm10[0,1,2,0,4,5,6,4] -; AVX-NEXT: vextractf128 $1, %ymm10, %xmm10 -; AVX-NEXT: vblendps {{.*#+}} xmm8 = xmm8[0,1],xmm10[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm7 = ymm8[0,1,2,3],ymm7[4,5,6,7] -; AVX-NEXT: vmovups %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vunpckhps {{.*#+}} ymm2 = ymm2[2],ymm5[2],ymm2[3],ymm5[3],ymm2[6],ymm5[6],ymm2[7],ymm5[7] -; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm1[3,0],ymm0[3,0],ymm1[7,4],ymm0[7,4] -; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm0[2,0],ymm2[2,3],ymm0[6,4],ymm2[6,7] -; AVX-NEXT: vunpckhps {{.*#+}} ymm1 = ymm6[2],ymm9[2],ymm6[3],ymm9[3],ymm6[6],ymm9[6],ymm6[7],ymm9[7] -; AVX-NEXT: vshufps {{.*#+}} ymm2 = ymm4[3,0],ymm3[3,0],ymm4[7,4],ymm3[7,4] -; AVX-NEXT: vextractf128 $1, %ymm1, %xmm1 -; AVX-NEXT: vshufps {{.*#+}} ymm2 = ymm2[2,0,2,3,6,4,6,7] -; AVX-NEXT: vextractf128 $1, %ymm2, %xmm2 -; AVX-NEXT: vblendps {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7] -; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vmovaps 128(%r8), %ymm0 -; AVX-NEXT: vmovaps 128(%r9), %ymm1 -; AVX-NEXT: vmovaps 128(%r10), %ymm3 -; AVX-NEXT: vmovaps 128(%rax), %ymm5 -; AVX-NEXT: vunpcklpd {{.*#+}} ymm2 = ymm5[0],ymm3[0],ymm5[2],ymm3[2] -; AVX-NEXT: vunpcklps {{.*#+}} ymm4 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5] -; AVX-NEXT: vshufps {{.*#+}} ymm7 = ymm4[0,1],ymm2[2,0],ymm4[4,5],ymm2[6,4] -; AVX-NEXT: vmovaps 128(%rdi), %ymm2 -; AVX-NEXT: vmovaps 128(%rsi), %ymm4 -; AVX-NEXT: vunpcklps {{.*#+}} ymm6 = ymm2[0],ymm4[0],ymm2[1],ymm4[1],ymm2[4],ymm4[4],ymm2[5],ymm4[5] -; AVX-NEXT: vextractf128 $1, %ymm6, %xmm8 -; AVX-NEXT: vmovaps 128(%rdx), %ymm6 -; AVX-NEXT: vmovaps 128(%rcx), %ymm9 -; AVX-NEXT: vunpcklpd {{.*#+}} ymm10 = ymm9[0],ymm6[0],ymm9[2],ymm6[2] -; AVX-NEXT: vshufps {{.*#+}} ymm10 = ymm10[0,1,2,0,4,5,6,4] -; AVX-NEXT: vextractf128 $1, %ymm10, %xmm10 -; AVX-NEXT: vblendps {{.*#+}} xmm8 = xmm8[0,1],xmm10[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm7 = ymm8[0,1,2,3],ymm7[4,5,6,7] -; AVX-NEXT: vmovups %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vunpcklps {{.*#+}} ymm7 = ymm3[0],ymm5[0],ymm3[1],ymm5[1],ymm3[4],ymm5[4],ymm3[5],ymm5[5] -; AVX-NEXT: vshufps {{.*#+}} ymm8 = ymm1[1,0],ymm0[1,0],ymm1[5,4],ymm0[5,4] -; AVX-NEXT: vshufps {{.*#+}} ymm7 = ymm8[2,0],ymm7[2,3],ymm8[6,4],ymm7[6,7] -; AVX-NEXT: vunpcklps {{.*#+}} ymm8 = ymm6[0],ymm9[0],ymm6[1],ymm9[1],ymm6[4],ymm9[4],ymm6[5],ymm9[5] -; AVX-NEXT: vextractf128 $1, %ymm8, %xmm8 -; AVX-NEXT: vshufps {{.*#+}} ymm10 = ymm4[1,0],ymm2[1,0],ymm4[5,4],ymm2[5,4] -; AVX-NEXT: vshufps {{.*#+}} ymm10 = ymm10[2,0,2,3,6,4,6,7] -; AVX-NEXT: vextractf128 $1, %ymm10, %xmm10 -; AVX-NEXT: vblendps {{.*#+}} xmm8 = xmm10[0,1],xmm8[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm7 = ymm8[0,1,2,3],ymm7[4,5,6,7] +; AVX-NEXT: vmovaps (%r10), %ymm10 +; AVX-NEXT: vunpckhps {{.*#+}} xmm7 = xmm7[2],xmm5[2],xmm7[3],xmm5[3] +; AVX-NEXT: vmovaps (%rax), %ymm5 +; AVX-NEXT: vunpckhpd {{.*#+}} xmm7 = xmm12[1],xmm7[1] +; AVX-NEXT: vunpckhps {{.*#+}} xmm8 = xmm9[2],xmm8[2],xmm9[3],xmm8[3] +; AVX-NEXT: vinsertf128 $1, %xmm8, %ymm0, %ymm8 +; AVX-NEXT: vshufps {{.*#+}} xmm9 = xmm11[2,3,2,3] +; AVX-NEXT: vinsertf128 $1, %xmm9, %ymm0, %ymm9 +; AVX-NEXT: vblendps {{.*#+}} ymm8 = ymm9[0,1,2,3,4,5],ymm8[6,7] +; AVX-NEXT: vblendps {{.*#+}} ymm7 = ymm7[0,1,2,3],ymm8[4,5,6,7] ; AVX-NEXT: vmovups %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vunpckhpd {{.*#+}} ymm7 = ymm5[1],ymm3[1],ymm5[3],ymm3[3] -; AVX-NEXT: vunpckhps {{.*#+}} ymm8 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7] +; AVX-NEXT: vunpcklpd {{.*#+}} ymm7 = ymm5[0],ymm10[0],ymm5[2],ymm10[2] +; AVX-NEXT: vunpcklps {{.*#+}} ymm8 = ymm4[0],ymm6[0],ymm4[1],ymm6[1],ymm4[4],ymm6[4],ymm4[5],ymm6[5] ; AVX-NEXT: vshufps {{.*#+}} ymm7 = ymm8[0,1],ymm7[2,0],ymm8[4,5],ymm7[6,4] -; AVX-NEXT: vunpckhps {{.*#+}} ymm8 = ymm2[2],ymm4[2],ymm2[3],ymm4[3],ymm2[6],ymm4[6],ymm2[7],ymm4[7] +; AVX-NEXT: vunpcklpd {{.*#+}} ymm8 = ymm3[0],ymm2[0],ymm3[2],ymm2[2] ; AVX-NEXT: vextractf128 $1, %ymm8, %xmm8 -; AVX-NEXT: vunpckhpd {{.*#+}} ymm10 = ymm9[1],ymm6[1],ymm9[3],ymm6[3] -; AVX-NEXT: vshufps {{.*#+}} ymm10 = ymm10[0,1,2,0,4,5,6,4] -; AVX-NEXT: vextractf128 $1, %ymm10, %xmm10 -; AVX-NEXT: vblendps {{.*#+}} xmm8 = xmm8[0,1],xmm10[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm7 = ymm8[0,1,2,3],ymm7[4,5,6,7] -; AVX-NEXT: vmovups %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vunpckhps {{.*#+}} ymm3 = ymm3[2],ymm5[2],ymm3[3],ymm5[3],ymm3[6],ymm5[6],ymm3[7],ymm5[7] -; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm1[3,0],ymm0[3,0],ymm1[7,4],ymm0[7,4] -; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm0[2,0],ymm3[2,3],ymm0[6,4],ymm3[6,7] -; AVX-NEXT: vunpckhps {{.*#+}} ymm1 = ymm6[2],ymm9[2],ymm6[3],ymm9[3],ymm6[6],ymm9[6],ymm6[7],ymm9[7] -; AVX-NEXT: vshufps {{.*#+}} ymm2 = ymm4[3,0],ymm2[3,0],ymm4[7,4],ymm2[7,4] -; AVX-NEXT: vextractf128 $1, %ymm1, %xmm1 -; AVX-NEXT: vshufps {{.*#+}} ymm2 = ymm2[2,0,2,3,6,4,6,7] -; AVX-NEXT: vextractf128 $1, %ymm2, %xmm2 -; AVX-NEXT: vblendps {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7] -; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vmovaps 160(%r8), %ymm0 -; AVX-NEXT: vmovaps 160(%r9), %ymm1 -; AVX-NEXT: vmovaps 160(%r10), %ymm3 -; AVX-NEXT: vmovaps 160(%rax), %ymm5 -; AVX-NEXT: vunpcklpd {{.*#+}} ymm2 = ymm5[0],ymm3[0],ymm5[2],ymm3[2] -; AVX-NEXT: vunpcklps {{.*#+}} ymm4 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5] -; AVX-NEXT: vshufps {{.*#+}} ymm7 = ymm4[0,1],ymm2[2,0],ymm4[4,5],ymm2[6,4] -; AVX-NEXT: vmovaps 160(%rdi), %ymm2 -; AVX-NEXT: vmovaps 160(%rsi), %ymm4 -; AVX-NEXT: vunpcklps {{.*#+}} ymm6 = ymm2[0],ymm4[0],ymm2[1],ymm4[1],ymm2[4],ymm4[4],ymm2[5],ymm4[5] -; AVX-NEXT: vextractf128 $1, %ymm6, %xmm8 -; AVX-NEXT: vmovaps 160(%rdx), %ymm6 -; AVX-NEXT: vmovaps 160(%rcx), %ymm9 -; AVX-NEXT: vunpcklpd {{.*#+}} ymm10 = ymm9[0],ymm6[0],ymm9[2],ymm6[2] -; AVX-NEXT: vshufps {{.*#+}} ymm10 = ymm10[0,1,2,0,4,5,6,4] -; AVX-NEXT: vextractf128 $1, %ymm10, %xmm10 -; AVX-NEXT: vblendps {{.*#+}} xmm8 = xmm8[0,1],xmm10[2,3] +; AVX-NEXT: vunpcklps {{.*#+}} ymm9 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5] +; AVX-NEXT: vextractf128 $1, %ymm9, %xmm9 +; AVX-NEXT: vshufps {{.*#+}} xmm8 = xmm9[0,1],xmm8[2,0] ; AVX-NEXT: vblendps {{.*#+}} ymm7 = ymm8[0,1,2,3],ymm7[4,5,6,7] ; AVX-NEXT: vmovups %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vunpcklps {{.*#+}} ymm7 = ymm3[0],ymm5[0],ymm3[1],ymm5[1],ymm3[4],ymm5[4],ymm3[5],ymm5[5] -; AVX-NEXT: vshufps {{.*#+}} ymm8 = ymm1[1,0],ymm0[1,0],ymm1[5,4],ymm0[5,4] +; AVX-NEXT: vunpcklps {{.*#+}} ymm7 = ymm10[0],ymm5[0],ymm10[1],ymm5[1],ymm10[4],ymm5[4],ymm10[5],ymm5[5] +; AVX-NEXT: vshufps {{.*#+}} ymm8 = ymm6[1,0],ymm4[1,0],ymm6[5,4],ymm4[5,4] ; AVX-NEXT: vshufps {{.*#+}} ymm7 = ymm8[2,0],ymm7[2,3],ymm8[6,4],ymm7[6,7] -; AVX-NEXT: vunpcklps {{.*#+}} ymm8 = ymm6[0],ymm9[0],ymm6[1],ymm9[1],ymm6[4],ymm9[4],ymm6[5],ymm9[5] +; AVX-NEXT: vunpcklps {{.*#+}} ymm8 = ymm2[0],ymm3[0],ymm2[1],ymm3[1],ymm2[4],ymm3[4],ymm2[5],ymm3[5] ; AVX-NEXT: vextractf128 $1, %ymm8, %xmm8 -; AVX-NEXT: vshufps {{.*#+}} ymm10 = ymm4[1,0],ymm2[1,0],ymm4[5,4],ymm2[5,4] -; AVX-NEXT: vshufps {{.*#+}} ymm10 = ymm10[2,0,2,3,6,4,6,7] -; AVX-NEXT: vextractf128 $1, %ymm10, %xmm10 -; AVX-NEXT: vblendps {{.*#+}} xmm8 = xmm10[0,1],xmm8[2,3] +; AVX-NEXT: vshufps {{.*#+}} ymm9 = ymm1[1,0],ymm0[1,0],ymm1[5,4],ymm0[5,4] +; AVX-NEXT: vextractf128 $1, %ymm9, %xmm9 +; AVX-NEXT: vshufps {{.*#+}} xmm8 = xmm9[2,0],xmm8[2,3] ; AVX-NEXT: vblendps {{.*#+}} ymm7 = ymm8[0,1,2,3],ymm7[4,5,6,7] ; AVX-NEXT: vmovups %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vunpckhpd {{.*#+}} ymm7 = ymm5[1],ymm3[1],ymm5[3],ymm3[3] -; AVX-NEXT: vunpckhps {{.*#+}} ymm8 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7] +; AVX-NEXT: vunpckhpd {{.*#+}} ymm7 = ymm5[1],ymm10[1],ymm5[3],ymm10[3] +; AVX-NEXT: vunpckhps {{.*#+}} ymm8 = ymm4[2],ymm6[2],ymm4[3],ymm6[3],ymm4[6],ymm6[6],ymm4[7],ymm6[7] ; AVX-NEXT: vshufps {{.*#+}} ymm7 = ymm8[0,1],ymm7[2,0],ymm8[4,5],ymm7[6,4] -; AVX-NEXT: vunpckhps {{.*#+}} ymm8 = ymm2[2],ymm4[2],ymm2[3],ymm4[3],ymm2[6],ymm4[6],ymm2[7],ymm4[7] +; AVX-NEXT: vunpckhpd {{.*#+}} ymm8 = ymm3[1],ymm2[1],ymm3[3],ymm2[3] ; AVX-NEXT: vextractf128 $1, %ymm8, %xmm8 -; AVX-NEXT: vunpckhpd {{.*#+}} ymm10 = ymm9[1],ymm6[1],ymm9[3],ymm6[3] -; AVX-NEXT: vshufps {{.*#+}} ymm10 = ymm10[0,1,2,0,4,5,6,4] -; AVX-NEXT: vextractf128 $1, %ymm10, %xmm10 -; AVX-NEXT: vblendps {{.*#+}} xmm8 = xmm8[0,1],xmm10[2,3] +; AVX-NEXT: vunpckhps {{.*#+}} ymm9 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7] +; AVX-NEXT: vextractf128 $1, %ymm9, %xmm9 +; AVX-NEXT: vshufps {{.*#+}} xmm8 = xmm9[0,1],xmm8[2,0] ; AVX-NEXT: vblendps {{.*#+}} ymm7 = ymm8[0,1,2,3],ymm7[4,5,6,7] ; AVX-NEXT: vmovups %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vunpckhps {{.*#+}} ymm3 = ymm3[2],ymm5[2],ymm3[3],ymm5[3],ymm3[6],ymm5[6],ymm3[7],ymm5[7] -; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm1[3,0],ymm0[3,0],ymm1[7,4],ymm0[7,4] -; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm0[2,0],ymm3[2,3],ymm0[6,4],ymm3[6,7] -; AVX-NEXT: vunpckhps {{.*#+}} ymm1 = ymm6[2],ymm9[2],ymm6[3],ymm9[3],ymm6[6],ymm9[6],ymm6[7],ymm9[7] -; AVX-NEXT: vshufps {{.*#+}} ymm2 = ymm4[3,0],ymm2[3,0],ymm4[7,4],ymm2[7,4] -; AVX-NEXT: vextractf128 $1, %ymm1, %xmm1 -; AVX-NEXT: vshufps {{.*#+}} ymm2 = ymm2[2,0,2,3,6,4,6,7] +; AVX-NEXT: vmovaps 32(%rcx), %xmm7 +; AVX-NEXT: vunpckhps {{.*#+}} ymm8 = ymm10[2],ymm5[2],ymm10[3],ymm5[3],ymm10[6],ymm5[6],ymm10[7],ymm5[7] +; AVX-NEXT: vmovaps 32(%rax), %xmm5 +; AVX-NEXT: vshufps {{.*#+}} ymm6 = ymm6[3,0],ymm4[3,0],ymm6[7,4],ymm4[7,4] +; AVX-NEXT: vmovaps 32(%r10), %xmm4 +; AVX-NEXT: vshufps {{.*#+}} ymm6 = ymm6[2,0],ymm8[2,3],ymm6[6,4],ymm8[6,7] +; AVX-NEXT: vunpckhps {{.*#+}} ymm2 = ymm2[2],ymm3[2],ymm2[3],ymm3[3],ymm2[6],ymm3[6],ymm2[7],ymm3[7] ; AVX-NEXT: vextractf128 $1, %ymm2, %xmm2 -; AVX-NEXT: vblendps {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7] -; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vmovaps 192(%r8), %ymm0 -; AVX-NEXT: vmovaps 192(%r9), %ymm1 -; AVX-NEXT: vmovaps 192(%r10), %ymm3 -; AVX-NEXT: vmovaps 192(%rax), %ymm5 -; AVX-NEXT: vunpcklpd {{.*#+}} ymm2 = ymm5[0],ymm3[0],ymm5[2],ymm3[2] -; AVX-NEXT: vunpcklps {{.*#+}} ymm4 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5] -; AVX-NEXT: vshufps {{.*#+}} ymm7 = ymm4[0,1],ymm2[2,0],ymm4[4,5],ymm2[6,4] -; AVX-NEXT: vmovaps 192(%rdi), %ymm2 -; AVX-NEXT: vmovaps 192(%rsi), %ymm4 -; AVX-NEXT: vunpcklps {{.*#+}} ymm6 = ymm2[0],ymm4[0],ymm2[1],ymm4[1],ymm2[4],ymm4[4],ymm2[5],ymm4[5] -; AVX-NEXT: vextractf128 $1, %ymm6, %xmm8 -; AVX-NEXT: vmovaps 192(%rdx), %ymm6 -; AVX-NEXT: vmovaps 192(%rcx), %ymm9 -; AVX-NEXT: vunpcklpd {{.*#+}} ymm10 = ymm9[0],ymm6[0],ymm9[2],ymm6[2] -; AVX-NEXT: vshufps {{.*#+}} ymm10 = ymm10[0,1,2,0,4,5,6,4] -; AVX-NEXT: vextractf128 $1, %ymm10, %xmm10 -; AVX-NEXT: vblendps {{.*#+}} xmm8 = xmm8[0,1],xmm10[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm7 = ymm8[0,1,2,3],ymm7[4,5,6,7] -; AVX-NEXT: vmovups %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vunpcklps {{.*#+}} ymm7 = ymm3[0],ymm5[0],ymm3[1],ymm5[1],ymm3[4],ymm5[4],ymm3[5],ymm5[5] -; AVX-NEXT: vshufps {{.*#+}} ymm8 = ymm1[1,0],ymm0[1,0],ymm1[5,4],ymm0[5,4] -; AVX-NEXT: vshufps {{.*#+}} ymm7 = ymm8[2,0],ymm7[2,3],ymm8[6,4],ymm7[6,7] -; AVX-NEXT: vunpcklps {{.*#+}} ymm8 = ymm6[0],ymm9[0],ymm6[1],ymm9[1],ymm6[4],ymm9[4],ymm6[5],ymm9[5] -; AVX-NEXT: vextractf128 $1, %ymm8, %xmm8 -; AVX-NEXT: vshufps {{.*#+}} ymm10 = ymm4[1,0],ymm2[1,0],ymm4[5,4],ymm2[5,4] -; AVX-NEXT: vshufps {{.*#+}} ymm10 = ymm10[2,0,2,3,6,4,6,7] -; AVX-NEXT: vextractf128 $1, %ymm10, %xmm10 -; AVX-NEXT: vblendps {{.*#+}} xmm8 = xmm10[0,1],xmm8[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm7 = ymm8[0,1,2,3],ymm7[4,5,6,7] -; AVX-NEXT: vmovups %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vunpckhpd {{.*#+}} ymm7 = ymm5[1],ymm3[1],ymm5[3],ymm3[3] -; AVX-NEXT: vunpckhps {{.*#+}} ymm8 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7] -; AVX-NEXT: vshufps {{.*#+}} ymm7 = ymm8[0,1],ymm7[2,0],ymm8[4,5],ymm7[6,4] -; AVX-NEXT: vunpckhps {{.*#+}} ymm8 = ymm2[2],ymm4[2],ymm2[3],ymm4[3],ymm2[6],ymm4[6],ymm2[7],ymm4[7] -; AVX-NEXT: vextractf128 $1, %ymm8, %xmm8 -; AVX-NEXT: vunpckhpd {{.*#+}} ymm10 = ymm9[1],ymm6[1],ymm9[3],ymm6[3] -; AVX-NEXT: vshufps {{.*#+}} ymm10 = ymm10[0,1,2,0,4,5,6,4] -; AVX-NEXT: vextractf128 $1, %ymm10, %xmm10 -; AVX-NEXT: vblendps {{.*#+}} xmm8 = xmm8[0,1],xmm10[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm7 = ymm8[0,1,2,3],ymm7[4,5,6,7] -; AVX-NEXT: vmovups %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vunpckhps {{.*#+}} ymm3 = ymm3[2],ymm5[2],ymm3[3],ymm5[3],ymm3[6],ymm5[6],ymm3[7],ymm5[7] ; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm1[3,0],ymm0[3,0],ymm1[7,4],ymm0[7,4] -; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm0[2,0],ymm3[2,3],ymm0[6,4],ymm3[6,7] -; AVX-NEXT: vunpckhps {{.*#+}} ymm1 = ymm6[2],ymm9[2],ymm6[3],ymm9[3],ymm6[6],ymm9[6],ymm6[7],ymm9[7] -; AVX-NEXT: vshufps {{.*#+}} ymm2 = ymm4[3,0],ymm2[3,0],ymm4[7,4],ymm2[7,4] -; AVX-NEXT: vextractf128 $1, %ymm1, %xmm1 -; AVX-NEXT: vshufps {{.*#+}} ymm2 = ymm2[2,0,2,3,6,4,6,7] -; AVX-NEXT: vextractf128 $1, %ymm2, %xmm2 -; AVX-NEXT: vblendps {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7] +; AVX-NEXT: vextractf128 $1, %ymm0, %xmm0 +; AVX-NEXT: vshufps {{.*#+}} xmm0 = xmm0[2,0],xmm2[2,3] +; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm6[4,5,6,7] ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vmovaps 224(%r8), %ymm0 -; AVX-NEXT: vmovaps 224(%r9), %ymm1 -; AVX-NEXT: vmovaps 224(%r10), %ymm3 -; AVX-NEXT: vmovaps 224(%rax), %ymm5 -; AVX-NEXT: vunpcklpd {{.*#+}} ymm2 = ymm5[0],ymm3[0],ymm5[2],ymm3[2] -; AVX-NEXT: vunpcklps {{.*#+}} ymm4 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5] -; AVX-NEXT: vshufps {{.*#+}} ymm7 = ymm4[0,1],ymm2[2,0],ymm4[4,5],ymm2[6,4] -; AVX-NEXT: vmovaps 224(%rdi), %ymm2 -; AVX-NEXT: vmovaps 224(%rsi), %ymm4 -; AVX-NEXT: vunpcklps {{.*#+}} ymm6 = ymm2[0],ymm4[0],ymm2[1],ymm4[1],ymm2[4],ymm4[4],ymm2[5],ymm4[5] -; AVX-NEXT: vextractf128 $1, %ymm6, %xmm8 -; AVX-NEXT: vmovaps 224(%rdx), %ymm6 -; AVX-NEXT: vmovaps 224(%rcx), %ymm9 -; AVX-NEXT: vunpcklpd {{.*#+}} ymm10 = ymm9[0],ymm6[0],ymm9[2],ymm6[2] -; AVX-NEXT: vshufps {{.*#+}} ymm10 = ymm10[0,1,2,0,4,5,6,4] +; AVX-NEXT: vmovaps 32(%r9), %xmm1 +; AVX-NEXT: vmovaps 32(%r8), %xmm2 +; AVX-NEXT: vunpcklps {{.*#+}} xmm0 = xmm2[0],xmm1[0],xmm2[1],xmm1[1] +; AVX-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0 +; AVX-NEXT: vunpcklps {{.*#+}} xmm3 = xmm4[0],xmm5[0],xmm4[1],xmm5[1] +; AVX-NEXT: vshufps {{.*#+}} xmm6 = xmm3[0,1,0,1] +; AVX-NEXT: vinsertf128 $1, %xmm6, %ymm0, %ymm6 +; AVX-NEXT: vblendps {{.*#+}} ymm6 = ymm0[0,1,2,3,4,5],ymm6[6,7] +; AVX-NEXT: vmovaps 32(%rdx), %xmm0 +; AVX-NEXT: vunpcklps {{.*#+}} xmm8 = xmm0[0],xmm7[0],xmm0[1],xmm7[1] +; AVX-NEXT: vmovaps 32(%rsi), %xmm9 +; AVX-NEXT: vmovaps 32(%rdi), %xmm10 +; AVX-NEXT: vunpcklps {{.*#+}} xmm11 = xmm10[0],xmm9[0],xmm10[1],xmm9[1] +; AVX-NEXT: vmovlhps {{.*#+}} xmm11 = xmm11[0],xmm8[0] +; AVX-NEXT: vblendps {{.*#+}} ymm6 = ymm11[0,1,2,3],ymm6[4,5,6,7] +; AVX-NEXT: vmovups %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vshufps {{.*#+}} xmm6 = xmm10[1,1,1,1] +; AVX-NEXT: vblendps {{.*#+}} xmm6 = xmm6[0],xmm9[1],xmm6[2,3] +; AVX-NEXT: vblendps {{.*#+}} xmm6 = xmm6[0,1],xmm8[2,3] +; AVX-NEXT: vinsertf128 $1, %xmm3, %ymm0, %ymm3 +; AVX-NEXT: vshufps {{.*#+}} xmm8 = xmm2[1,1,1,1] +; AVX-NEXT: vblendps {{.*#+}} xmm8 = xmm8[0],xmm1[1],xmm8[2,3] +; AVX-NEXT: vinsertf128 $1, %xmm8, %ymm0, %ymm8 +; AVX-NEXT: vblendps {{.*#+}} ymm3 = ymm8[0,1,2,3,4,5],ymm3[6,7] +; AVX-NEXT: vblendps {{.*#+}} ymm3 = ymm6[0,1,2,3],ymm3[4,5,6,7] +; AVX-NEXT: vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vunpckhps {{.*#+}} xmm3 = xmm10[2],xmm9[2],xmm10[3],xmm9[3] +; AVX-NEXT: vunpckhps {{.*#+}} xmm2 = xmm2[2],xmm1[2],xmm2[3],xmm1[3] +; AVX-NEXT: vshufps {{.*#+}} xmm1 = xmm5[2,2,2,2] +; AVX-NEXT: vblendps {{.*#+}} xmm1 = xmm4[0,1,2],xmm1[3] +; AVX-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm1 +; AVX-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm6 +; AVX-NEXT: vblendps {{.*#+}} ymm1 = ymm6[0,1,2,3,4,5],ymm1[6,7] +; AVX-NEXT: vshufps {{.*#+}} xmm6 = xmm7[2,2,2,2] +; AVX-NEXT: vblendps {{.*#+}} xmm6 = xmm0[0,1,2],xmm6[3] +; AVX-NEXT: vblendps {{.*#+}} xmm6 = xmm3[0,1],xmm6[2,3] +; AVX-NEXT: vblendps {{.*#+}} ymm1 = ymm6[0,1,2,3],ymm1[4,5,6,7] +; AVX-NEXT: vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vunpckhps {{.*#+}} xmm0 = xmm0[2],xmm7[2],xmm0[3],xmm7[3] +; AVX-NEXT: vunpckhpd {{.*#+}} xmm3 = xmm3[1],xmm0[1] +; AVX-NEXT: vmovaps 32(%r8), %ymm0 +; AVX-NEXT: vunpckhps {{.*#+}} xmm4 = xmm4[2],xmm5[2],xmm4[3],xmm5[3] +; AVX-NEXT: vmovaps 32(%r9), %ymm1 +; AVX-NEXT: vinsertf128 $1, %xmm4, %ymm0, %ymm4 +; AVX-NEXT: vshufps {{.*#+}} xmm2 = xmm2[2,3,2,3] +; AVX-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm2 +; AVX-NEXT: vblendps {{.*#+}} ymm4 = ymm2[0,1,2,3,4,5],ymm4[6,7] +; AVX-NEXT: vmovaps 32(%r10), %ymm2 +; AVX-NEXT: vblendps {{.*#+}} ymm3 = ymm3[0,1,2,3],ymm4[4,5,6,7] +; AVX-NEXT: vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vmovaps 32(%rax), %ymm3 +; AVX-NEXT: vunpcklpd {{.*#+}} ymm4 = ymm3[0],ymm2[0],ymm3[2],ymm2[2] +; AVX-NEXT: vunpcklps {{.*#+}} ymm5 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5] +; AVX-NEXT: vshufps {{.*#+}} ymm6 = ymm5[0,1],ymm4[2,0],ymm5[4,5],ymm4[6,4] +; AVX-NEXT: vmovaps 32(%rdx), %ymm4 +; AVX-NEXT: vmovaps 32(%rcx), %ymm5 +; AVX-NEXT: vunpcklpd {{.*#+}} ymm7 = ymm5[0],ymm4[0],ymm5[2],ymm4[2] +; AVX-NEXT: vextractf128 $1, %ymm7, %xmm7 +; AVX-NEXT: vmovaps 32(%rdi), %ymm8 +; AVX-NEXT: vmovaps 32(%rsi), %ymm9 +; AVX-NEXT: vunpcklps {{.*#+}} ymm10 = ymm8[0],ymm9[0],ymm8[1],ymm9[1],ymm8[4],ymm9[4],ymm8[5],ymm9[5] ; AVX-NEXT: vextractf128 $1, %ymm10, %xmm10 -; AVX-NEXT: vblendps {{.*#+}} xmm8 = xmm8[0,1],xmm10[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm7 = ymm8[0,1,2,3],ymm7[4,5,6,7] -; AVX-NEXT: vmovups %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vunpcklps {{.*#+}} ymm7 = ymm3[0],ymm5[0],ymm3[1],ymm5[1],ymm3[4],ymm5[4],ymm3[5],ymm5[5] -; AVX-NEXT: vshufps {{.*#+}} ymm8 = ymm1[1,0],ymm0[1,0],ymm1[5,4],ymm0[5,4] -; AVX-NEXT: vshufps {{.*#+}} ymm7 = ymm8[2,0],ymm7[2,3],ymm8[6,4],ymm7[6,7] -; AVX-NEXT: vunpcklps {{.*#+}} ymm8 = ymm6[0],ymm9[0],ymm6[1],ymm9[1],ymm6[4],ymm9[4],ymm6[5],ymm9[5] -; AVX-NEXT: vextractf128 $1, %ymm8, %xmm8 -; AVX-NEXT: vshufps {{.*#+}} ymm10 = ymm4[1,0],ymm2[1,0],ymm4[5,4],ymm2[5,4] -; AVX-NEXT: vshufps {{.*#+}} ymm10 = ymm10[2,0,2,3,6,4,6,7] +; AVX-NEXT: vshufps {{.*#+}} xmm7 = xmm10[0,1],xmm7[2,0] +; AVX-NEXT: vblendps {{.*#+}} ymm6 = ymm7[0,1,2,3],ymm6[4,5,6,7] +; AVX-NEXT: vmovups %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vunpcklps {{.*#+}} ymm6 = ymm2[0],ymm3[0],ymm2[1],ymm3[1],ymm2[4],ymm3[4],ymm2[5],ymm3[5] +; AVX-NEXT: vshufps {{.*#+}} ymm7 = ymm1[1,0],ymm0[1,0],ymm1[5,4],ymm0[5,4] +; AVX-NEXT: vshufps {{.*#+}} ymm6 = ymm7[2,0],ymm6[2,3],ymm7[6,4],ymm6[6,7] +; AVX-NEXT: vunpcklps {{.*#+}} ymm7 = ymm4[0],ymm5[0],ymm4[1],ymm5[1],ymm4[4],ymm5[4],ymm4[5],ymm5[5] +; AVX-NEXT: vextractf128 $1, %ymm7, %xmm7 +; AVX-NEXT: vshufps {{.*#+}} ymm10 = ymm9[1,0],ymm8[1,0],ymm9[5,4],ymm8[5,4] ; AVX-NEXT: vextractf128 $1, %ymm10, %xmm10 -; AVX-NEXT: vblendps {{.*#+}} xmm8 = xmm10[0,1],xmm8[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm7 = ymm8[0,1,2,3],ymm7[4,5,6,7] -; AVX-NEXT: vmovups %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vunpckhpd {{.*#+}} ymm7 = ymm5[1],ymm3[1],ymm5[3],ymm3[3] -; AVX-NEXT: vunpckhps {{.*#+}} ymm8 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7] -; AVX-NEXT: vshufps {{.*#+}} ymm7 = ymm8[0,1],ymm7[2,0],ymm8[4,5],ymm7[6,4] -; AVX-NEXT: vunpckhps {{.*#+}} ymm8 = ymm2[2],ymm4[2],ymm2[3],ymm4[3],ymm2[6],ymm4[6],ymm2[7],ymm4[7] -; AVX-NEXT: vextractf128 $1, %ymm8, %xmm8 -; AVX-NEXT: vunpckhpd {{.*#+}} ymm10 = ymm9[1],ymm6[1],ymm9[3],ymm6[3] -; AVX-NEXT: vshufps {{.*#+}} ymm10 = ymm10[0,1,2,0,4,5,6,4] +; AVX-NEXT: vshufps {{.*#+}} xmm7 = xmm10[2,0],xmm7[2,3] +; AVX-NEXT: vblendps {{.*#+}} ymm6 = ymm7[0,1,2,3],ymm6[4,5,6,7] +; AVX-NEXT: vmovups %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vunpckhpd {{.*#+}} ymm6 = ymm3[1],ymm2[1],ymm3[3],ymm2[3] +; AVX-NEXT: vunpckhps {{.*#+}} ymm7 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7] +; AVX-NEXT: vshufps {{.*#+}} ymm6 = ymm7[0,1],ymm6[2,0],ymm7[4,5],ymm6[6,4] +; AVX-NEXT: vunpckhpd {{.*#+}} ymm7 = ymm5[1],ymm4[1],ymm5[3],ymm4[3] +; AVX-NEXT: vextractf128 $1, %ymm7, %xmm7 +; AVX-NEXT: vunpckhps {{.*#+}} ymm10 = ymm8[2],ymm9[2],ymm8[3],ymm9[3],ymm8[6],ymm9[6],ymm8[7],ymm9[7] ; AVX-NEXT: vextractf128 $1, %ymm10, %xmm10 -; AVX-NEXT: vblendps {{.*#+}} xmm8 = xmm8[0,1],xmm10[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm7 = ymm8[0,1,2,3],ymm7[4,5,6,7] -; AVX-NEXT: vmovups %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vunpckhps {{.*#+}} ymm3 = ymm3[2],ymm5[2],ymm3[3],ymm5[3],ymm3[6],ymm5[6],ymm3[7],ymm5[7] +; AVX-NEXT: vshufps {{.*#+}} xmm7 = xmm10[0,1],xmm7[2,0] +; AVX-NEXT: vblendps {{.*#+}} ymm6 = ymm7[0,1,2,3],ymm6[4,5,6,7] +; AVX-NEXT: vmovups %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vunpckhps {{.*#+}} ymm2 = ymm2[2],ymm3[2],ymm2[3],ymm3[3],ymm2[6],ymm3[6],ymm2[7],ymm3[7] ; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm1[3,0],ymm0[3,0],ymm1[7,4],ymm0[7,4] -; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm0[2,0],ymm3[2,3],ymm0[6,4],ymm3[6,7] -; AVX-NEXT: vunpckhps {{.*#+}} ymm1 = ymm6[2],ymm9[2],ymm6[3],ymm9[3],ymm6[6],ymm9[6],ymm6[7],ymm9[7] -; AVX-NEXT: vshufps {{.*#+}} ymm2 = ymm4[3,0],ymm2[3,0],ymm4[7,4],ymm2[7,4] +; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm0[2,0],ymm2[2,3],ymm0[6,4],ymm2[6,7] +; AVX-NEXT: vunpckhps {{.*#+}} ymm1 = ymm4[2],ymm5[2],ymm4[3],ymm5[3],ymm4[6],ymm5[6],ymm4[7],ymm5[7] +; AVX-NEXT: vshufps {{.*#+}} ymm2 = ymm9[3,0],ymm8[3,0],ymm9[7,4],ymm8[7,4] ; AVX-NEXT: vextractf128 $1, %ymm1, %xmm1 -; AVX-NEXT: vshufps {{.*#+}} ymm2 = ymm2[2,0,2,3,6,4,6,7] ; AVX-NEXT: vextractf128 $1, %ymm2, %xmm2 -; AVX-NEXT: vblendps {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3] +; AVX-NEXT: vshufps {{.*#+}} xmm1 = xmm2[2,0],xmm1[2,3] ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7] ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vmovaps (%r9), %xmm2 -; AVX-NEXT: vmovaps (%r8), %xmm3 -; AVX-NEXT: vunpcklps {{.*#+}} xmm0 = xmm3[0],xmm2[0],xmm3[1],xmm2[1] -; AVX-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm4 -; AVX-NEXT: vmovaps (%rax), %xmm0 -; AVX-NEXT: vmovaps (%r10), %xmm1 -; AVX-NEXT: vunpcklps {{.*#+}} xmm6 = xmm1[0],xmm0[0],xmm1[1],xmm0[1] -; AVX-NEXT: vshufps {{.*#+}} xmm5 = xmm6[0,1,0,1] -; AVX-NEXT: vinsertf128 $1, %xmm5, %ymm0, %ymm5 -; AVX-NEXT: vblendps {{.*#+}} ymm7 = ymm4[0,1,2,3,4,5],ymm5[6,7] -; AVX-NEXT: vmovaps (%rcx), %xmm4 -; AVX-NEXT: vmovaps (%rdx), %xmm5 -; AVX-NEXT: vunpcklps {{.*#+}} xmm8 = xmm5[0],xmm4[0],xmm5[1],xmm4[1] -; AVX-NEXT: vmovaps (%rsi), %xmm9 -; AVX-NEXT: vmovaps (%rdi), %xmm10 -; AVX-NEXT: vunpcklps {{.*#+}} xmm11 = xmm10[0],xmm9[0],xmm10[1],xmm9[1] -; AVX-NEXT: vmovlhps {{.*#+}} xmm11 = xmm11[0],xmm8[0] -; AVX-NEXT: vblendps {{.*#+}} ymm7 = ymm11[0,1,2,3],ymm7[4,5,6,7] -; AVX-NEXT: vmovups %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vshufps {{.*#+}} xmm7 = xmm10[1,1,1,1] -; AVX-NEXT: vblendps {{.*#+}} xmm7 = xmm7[0],xmm9[1],xmm7[2,3] -; AVX-NEXT: vblendps {{.*#+}} xmm7 = xmm7[0,1],xmm8[2,3] -; AVX-NEXT: vinsertf128 $1, %xmm6, %ymm0, %ymm6 -; AVX-NEXT: vshufps {{.*#+}} xmm8 = xmm3[1,1,1,1] -; AVX-NEXT: vblendps {{.*#+}} xmm8 = xmm8[0],xmm2[1],xmm8[2,3] -; AVX-NEXT: vinsertf128 $1, %xmm8, %ymm0, %ymm8 -; AVX-NEXT: vblendps {{.*#+}} ymm6 = ymm8[0,1,2,3,4,5],ymm6[6,7] -; AVX-NEXT: vblendps {{.*#+}} ymm6 = ymm7[0,1,2,3],ymm6[4,5,6,7] -; AVX-NEXT: vmovups %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vunpckhps {{.*#+}} xmm6 = xmm10[2],xmm9[2],xmm10[3],xmm9[3] -; AVX-NEXT: vunpckhps {{.*#+}} xmm2 = xmm3[2],xmm2[2],xmm3[3],xmm2[3] -; AVX-NEXT: vshufps {{.*#+}} xmm3 = xmm0[2,2,2,2] -; AVX-NEXT: vblendps {{.*#+}} xmm3 = xmm1[0,1,2],xmm3[3] -; AVX-NEXT: vinsertf128 $1, %xmm3, %ymm0, %ymm3 -; AVX-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm7 -; AVX-NEXT: vblendps {{.*#+}} ymm3 = ymm7[0,1,2,3,4,5],ymm3[6,7] -; AVX-NEXT: vshufps {{.*#+}} xmm7 = xmm4[2,2,2,2] -; AVX-NEXT: vblendps {{.*#+}} xmm7 = xmm5[0,1,2],xmm7[3] -; AVX-NEXT: vblendps {{.*#+}} xmm7 = xmm6[0,1],xmm7[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm3 = ymm7[0,1,2,3],ymm3[4,5,6,7] -; AVX-NEXT: vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vunpckhps {{.*#+}} xmm3 = xmm5[2],xmm4[2],xmm5[3],xmm4[3] -; AVX-NEXT: vunpckhpd {{.*#+}} xmm3 = xmm6[1],xmm3[1] -; AVX-NEXT: vunpckhps {{.*#+}} xmm0 = xmm1[2],xmm0[2],xmm1[3],xmm0[3] -; AVX-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0 -; AVX-NEXT: vshufps {{.*#+}} xmm1 = xmm2[2,3,2,3] -; AVX-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm1 -; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7] -; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm3[0,1,2,3],ymm0[4,5,6,7] -; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vmovaps 32(%r9), %xmm4 -; AVX-NEXT: vmovaps 32(%r8), %xmm5 +; AVX-NEXT: vmovaps 64(%r9), %xmm4 +; AVX-NEXT: vmovaps 64(%r8), %xmm5 ; AVX-NEXT: vunpcklps {{.*#+}} xmm0 = xmm5[0],xmm4[0],xmm5[1],xmm4[1] ; AVX-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm2 -; AVX-NEXT: vmovaps 32(%rax), %xmm0 -; AVX-NEXT: vmovaps 32(%r10), %xmm1 +; AVX-NEXT: vmovaps 64(%rax), %xmm0 +; AVX-NEXT: vmovaps 64(%r10), %xmm1 ; AVX-NEXT: vunpcklps {{.*#+}} xmm6 = xmm1[0],xmm0[0],xmm1[1],xmm0[1] ; AVX-NEXT: vshufps {{.*#+}} xmm3 = xmm6[0,1,0,1] ; AVX-NEXT: vinsertf128 $1, %xmm3, %ymm0, %ymm3 ; AVX-NEXT: vblendps {{.*#+}} ymm7 = ymm2[0,1,2,3,4,5],ymm3[6,7] -; AVX-NEXT: vmovaps 32(%rcx), %xmm2 -; AVX-NEXT: vmovaps 32(%rdx), %xmm3 +; AVX-NEXT: vmovaps 64(%rcx), %xmm2 +; AVX-NEXT: vmovaps 64(%rdx), %xmm3 ; AVX-NEXT: vunpcklps {{.*#+}} xmm8 = xmm3[0],xmm2[0],xmm3[1],xmm2[1] -; AVX-NEXT: vmovaps 32(%rsi), %xmm9 -; AVX-NEXT: vmovaps 32(%rdi), %xmm10 +; AVX-NEXT: vmovaps 64(%rsi), %xmm9 +; AVX-NEXT: vmovaps 64(%rdi), %xmm10 ; AVX-NEXT: vunpcklps {{.*#+}} xmm11 = xmm10[0],xmm9[0],xmm10[1],xmm9[1] ; AVX-NEXT: vmovlhps {{.*#+}} xmm11 = xmm11[0],xmm8[0] ; AVX-NEXT: vblendps {{.*#+}} ymm7 = ymm11[0,1,2,3],ymm7[4,5,6,7] @@ -10245,21 +9968,69 @@ define void @store_i32_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7] ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm0[4,5,6,7] ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vmovaps 64(%r9), %xmm4 -; AVX-NEXT: vmovaps 64(%r8), %xmm5 +; AVX-NEXT: vmovaps 64(%r8), %ymm0 +; AVX-NEXT: vmovaps 64(%r9), %ymm1 +; AVX-NEXT: vmovaps 64(%r10), %ymm2 +; AVX-NEXT: vmovaps 64(%rax), %ymm3 +; AVX-NEXT: vunpcklpd {{.*#+}} ymm4 = ymm3[0],ymm2[0],ymm3[2],ymm2[2] +; AVX-NEXT: vunpcklps {{.*#+}} ymm5 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5] +; AVX-NEXT: vshufps {{.*#+}} ymm6 = ymm5[0,1],ymm4[2,0],ymm5[4,5],ymm4[6,4] +; AVX-NEXT: vmovaps 64(%rdx), %ymm4 +; AVX-NEXT: vmovaps 64(%rcx), %ymm5 +; AVX-NEXT: vunpcklpd {{.*#+}} ymm7 = ymm5[0],ymm4[0],ymm5[2],ymm4[2] +; AVX-NEXT: vextractf128 $1, %ymm7, %xmm7 +; AVX-NEXT: vmovaps 64(%rdi), %ymm8 +; AVX-NEXT: vmovaps 64(%rsi), %ymm9 +; AVX-NEXT: vunpcklps {{.*#+}} ymm10 = ymm8[0],ymm9[0],ymm8[1],ymm9[1],ymm8[4],ymm9[4],ymm8[5],ymm9[5] +; AVX-NEXT: vextractf128 $1, %ymm10, %xmm10 +; AVX-NEXT: vshufps {{.*#+}} xmm7 = xmm10[0,1],xmm7[2,0] +; AVX-NEXT: vblendps {{.*#+}} ymm6 = ymm7[0,1,2,3],ymm6[4,5,6,7] +; AVX-NEXT: vmovups %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vunpcklps {{.*#+}} ymm6 = ymm2[0],ymm3[0],ymm2[1],ymm3[1],ymm2[4],ymm3[4],ymm2[5],ymm3[5] +; AVX-NEXT: vshufps {{.*#+}} ymm7 = ymm1[1,0],ymm0[1,0],ymm1[5,4],ymm0[5,4] +; AVX-NEXT: vshufps {{.*#+}} ymm6 = ymm7[2,0],ymm6[2,3],ymm7[6,4],ymm6[6,7] +; AVX-NEXT: vunpcklps {{.*#+}} ymm7 = ymm4[0],ymm5[0],ymm4[1],ymm5[1],ymm4[4],ymm5[4],ymm4[5],ymm5[5] +; AVX-NEXT: vextractf128 $1, %ymm7, %xmm7 +; AVX-NEXT: vshufps {{.*#+}} ymm10 = ymm9[1,0],ymm8[1,0],ymm9[5,4],ymm8[5,4] +; AVX-NEXT: vextractf128 $1, %ymm10, %xmm10 +; AVX-NEXT: vshufps {{.*#+}} xmm7 = xmm10[2,0],xmm7[2,3] +; AVX-NEXT: vblendps {{.*#+}} ymm6 = ymm7[0,1,2,3],ymm6[4,5,6,7] +; AVX-NEXT: vmovups %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vunpckhpd {{.*#+}} ymm6 = ymm3[1],ymm2[1],ymm3[3],ymm2[3] +; AVX-NEXT: vunpckhps {{.*#+}} ymm7 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7] +; AVX-NEXT: vshufps {{.*#+}} ymm6 = ymm7[0,1],ymm6[2,0],ymm7[4,5],ymm6[6,4] +; AVX-NEXT: vunpckhpd {{.*#+}} ymm7 = ymm5[1],ymm4[1],ymm5[3],ymm4[3] +; AVX-NEXT: vextractf128 $1, %ymm7, %xmm7 +; AVX-NEXT: vunpckhps {{.*#+}} ymm10 = ymm8[2],ymm9[2],ymm8[3],ymm9[3],ymm8[6],ymm9[6],ymm8[7],ymm9[7] +; AVX-NEXT: vextractf128 $1, %ymm10, %xmm10 +; AVX-NEXT: vshufps {{.*#+}} xmm7 = xmm10[0,1],xmm7[2,0] +; AVX-NEXT: vblendps {{.*#+}} ymm6 = ymm7[0,1,2,3],ymm6[4,5,6,7] +; AVX-NEXT: vmovups %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vunpckhps {{.*#+}} ymm2 = ymm2[2],ymm3[2],ymm2[3],ymm3[3],ymm2[6],ymm3[6],ymm2[7],ymm3[7] +; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm1[3,0],ymm0[3,0],ymm1[7,4],ymm0[7,4] +; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm0[2,0],ymm2[2,3],ymm0[6,4],ymm2[6,7] +; AVX-NEXT: vunpckhps {{.*#+}} ymm1 = ymm4[2],ymm5[2],ymm4[3],ymm5[3],ymm4[6],ymm5[6],ymm4[7],ymm5[7] +; AVX-NEXT: vshufps {{.*#+}} ymm2 = ymm9[3,0],ymm8[3,0],ymm9[7,4],ymm8[7,4] +; AVX-NEXT: vextractf128 $1, %ymm1, %xmm1 +; AVX-NEXT: vextractf128 $1, %ymm2, %xmm2 +; AVX-NEXT: vshufps {{.*#+}} xmm1 = xmm2[2,0],xmm1[2,3] +; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7] +; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vmovaps 96(%r9), %xmm4 +; AVX-NEXT: vmovaps 96(%r8), %xmm5 ; AVX-NEXT: vunpcklps {{.*#+}} xmm0 = xmm5[0],xmm4[0],xmm5[1],xmm4[1] ; AVX-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm2 -; AVX-NEXT: vmovaps 64(%rax), %xmm0 -; AVX-NEXT: vmovaps 64(%r10), %xmm1 +; AVX-NEXT: vmovaps 96(%rax), %xmm0 +; AVX-NEXT: vmovaps 96(%r10), %xmm1 ; AVX-NEXT: vunpcklps {{.*#+}} xmm6 = xmm1[0],xmm0[0],xmm1[1],xmm0[1] ; AVX-NEXT: vshufps {{.*#+}} xmm3 = xmm6[0,1,0,1] ; AVX-NEXT: vinsertf128 $1, %xmm3, %ymm0, %ymm3 ; AVX-NEXT: vblendps {{.*#+}} ymm7 = ymm2[0,1,2,3,4,5],ymm3[6,7] -; AVX-NEXT: vmovaps 64(%rcx), %xmm2 -; AVX-NEXT: vmovaps 64(%rdx), %xmm3 +; AVX-NEXT: vmovaps 96(%rcx), %xmm2 +; AVX-NEXT: vmovaps 96(%rdx), %xmm3 ; AVX-NEXT: vunpcklps {{.*#+}} xmm8 = xmm3[0],xmm2[0],xmm3[1],xmm2[1] -; AVX-NEXT: vmovaps 64(%rsi), %xmm9 -; AVX-NEXT: vmovaps 64(%rdi), %xmm10 +; AVX-NEXT: vmovaps 96(%rsi), %xmm9 +; AVX-NEXT: vmovaps 96(%rdi), %xmm10 ; AVX-NEXT: vunpcklps {{.*#+}} xmm11 = xmm10[0],xmm9[0],xmm10[1],xmm9[1] ; AVX-NEXT: vmovlhps {{.*#+}} xmm11 = xmm11[0],xmm8[0] ; AVX-NEXT: vblendps {{.*#+}} ymm7 = ymm11[0,1,2,3],ymm7[4,5,6,7] @@ -10295,21 +10066,69 @@ define void @store_i32_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7] ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm0[4,5,6,7] ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vmovaps 96(%r9), %xmm4 -; AVX-NEXT: vmovaps 96(%r8), %xmm5 +; AVX-NEXT: vmovaps 96(%r8), %ymm0 +; AVX-NEXT: vmovaps 96(%r9), %ymm1 +; AVX-NEXT: vmovaps 96(%r10), %ymm2 +; AVX-NEXT: vmovaps 96(%rax), %ymm3 +; AVX-NEXT: vunpcklpd {{.*#+}} ymm4 = ymm3[0],ymm2[0],ymm3[2],ymm2[2] +; AVX-NEXT: vunpcklps {{.*#+}} ymm5 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5] +; AVX-NEXT: vshufps {{.*#+}} ymm6 = ymm5[0,1],ymm4[2,0],ymm5[4,5],ymm4[6,4] +; AVX-NEXT: vmovaps 96(%rdx), %ymm4 +; AVX-NEXT: vmovaps 96(%rcx), %ymm5 +; AVX-NEXT: vunpcklpd {{.*#+}} ymm7 = ymm5[0],ymm4[0],ymm5[2],ymm4[2] +; AVX-NEXT: vextractf128 $1, %ymm7, %xmm7 +; AVX-NEXT: vmovaps 96(%rdi), %ymm8 +; AVX-NEXT: vmovaps 96(%rsi), %ymm9 +; AVX-NEXT: vunpcklps {{.*#+}} ymm10 = ymm8[0],ymm9[0],ymm8[1],ymm9[1],ymm8[4],ymm9[4],ymm8[5],ymm9[5] +; AVX-NEXT: vextractf128 $1, %ymm10, %xmm10 +; AVX-NEXT: vshufps {{.*#+}} xmm7 = xmm10[0,1],xmm7[2,0] +; AVX-NEXT: vblendps {{.*#+}} ymm6 = ymm7[0,1,2,3],ymm6[4,5,6,7] +; AVX-NEXT: vmovups %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vunpcklps {{.*#+}} ymm6 = ymm2[0],ymm3[0],ymm2[1],ymm3[1],ymm2[4],ymm3[4],ymm2[5],ymm3[5] +; AVX-NEXT: vshufps {{.*#+}} ymm7 = ymm1[1,0],ymm0[1,0],ymm1[5,4],ymm0[5,4] +; AVX-NEXT: vshufps {{.*#+}} ymm6 = ymm7[2,0],ymm6[2,3],ymm7[6,4],ymm6[6,7] +; AVX-NEXT: vunpcklps {{.*#+}} ymm7 = ymm4[0],ymm5[0],ymm4[1],ymm5[1],ymm4[4],ymm5[4],ymm4[5],ymm5[5] +; AVX-NEXT: vextractf128 $1, %ymm7, %xmm7 +; AVX-NEXT: vshufps {{.*#+}} ymm10 = ymm9[1,0],ymm8[1,0],ymm9[5,4],ymm8[5,4] +; AVX-NEXT: vextractf128 $1, %ymm10, %xmm10 +; AVX-NEXT: vshufps {{.*#+}} xmm7 = xmm10[2,0],xmm7[2,3] +; AVX-NEXT: vblendps {{.*#+}} ymm6 = ymm7[0,1,2,3],ymm6[4,5,6,7] +; AVX-NEXT: vmovups %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vunpckhpd {{.*#+}} ymm6 = ymm3[1],ymm2[1],ymm3[3],ymm2[3] +; AVX-NEXT: vunpckhps {{.*#+}} ymm7 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7] +; AVX-NEXT: vshufps {{.*#+}} ymm6 = ymm7[0,1],ymm6[2,0],ymm7[4,5],ymm6[6,4] +; AVX-NEXT: vunpckhpd {{.*#+}} ymm7 = ymm5[1],ymm4[1],ymm5[3],ymm4[3] +; AVX-NEXT: vextractf128 $1, %ymm7, %xmm7 +; AVX-NEXT: vunpckhps {{.*#+}} ymm10 = ymm8[2],ymm9[2],ymm8[3],ymm9[3],ymm8[6],ymm9[6],ymm8[7],ymm9[7] +; AVX-NEXT: vextractf128 $1, %ymm10, %xmm10 +; AVX-NEXT: vshufps {{.*#+}} xmm7 = xmm10[0,1],xmm7[2,0] +; AVX-NEXT: vblendps {{.*#+}} ymm6 = ymm7[0,1,2,3],ymm6[4,5,6,7] +; AVX-NEXT: vmovups %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vunpckhps {{.*#+}} ymm2 = ymm2[2],ymm3[2],ymm2[3],ymm3[3],ymm2[6],ymm3[6],ymm2[7],ymm3[7] +; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm1[3,0],ymm0[3,0],ymm1[7,4],ymm0[7,4] +; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm0[2,0],ymm2[2,3],ymm0[6,4],ymm2[6,7] +; AVX-NEXT: vunpckhps {{.*#+}} ymm1 = ymm4[2],ymm5[2],ymm4[3],ymm5[3],ymm4[6],ymm5[6],ymm4[7],ymm5[7] +; AVX-NEXT: vshufps {{.*#+}} ymm2 = ymm9[3,0],ymm8[3,0],ymm9[7,4],ymm8[7,4] +; AVX-NEXT: vextractf128 $1, %ymm1, %xmm1 +; AVX-NEXT: vextractf128 $1, %ymm2, %xmm2 +; AVX-NEXT: vshufps {{.*#+}} xmm1 = xmm2[2,0],xmm1[2,3] +; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7] +; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vmovaps 128(%r9), %xmm4 +; AVX-NEXT: vmovaps 128(%r8), %xmm5 ; AVX-NEXT: vunpcklps {{.*#+}} xmm0 = xmm5[0],xmm4[0],xmm5[1],xmm4[1] ; AVX-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm2 -; AVX-NEXT: vmovaps 96(%rax), %xmm0 -; AVX-NEXT: vmovaps 96(%r10), %xmm1 +; AVX-NEXT: vmovaps 128(%rax), %xmm0 +; AVX-NEXT: vmovaps 128(%r10), %xmm1 ; AVX-NEXT: vunpcklps {{.*#+}} xmm6 = xmm1[0],xmm0[0],xmm1[1],xmm0[1] ; AVX-NEXT: vshufps {{.*#+}} xmm3 = xmm6[0,1,0,1] ; AVX-NEXT: vinsertf128 $1, %xmm3, %ymm0, %ymm3 ; AVX-NEXT: vblendps {{.*#+}} ymm7 = ymm2[0,1,2,3,4,5],ymm3[6,7] -; AVX-NEXT: vmovaps 96(%rcx), %xmm2 -; AVX-NEXT: vmovaps 96(%rdx), %xmm3 +; AVX-NEXT: vmovaps 128(%rcx), %xmm2 +; AVX-NEXT: vmovaps 128(%rdx), %xmm3 ; AVX-NEXT: vunpcklps {{.*#+}} xmm8 = xmm3[0],xmm2[0],xmm3[1],xmm2[1] -; AVX-NEXT: vmovaps 96(%rsi), %xmm9 -; AVX-NEXT: vmovaps 96(%rdi), %xmm10 +; AVX-NEXT: vmovaps 128(%rsi), %xmm9 +; AVX-NEXT: vmovaps 128(%rdi), %xmm10 ; AVX-NEXT: vunpcklps {{.*#+}} xmm11 = xmm10[0],xmm9[0],xmm10[1],xmm9[1] ; AVX-NEXT: vmovlhps {{.*#+}} xmm11 = xmm11[0],xmm8[0] ; AVX-NEXT: vblendps {{.*#+}} ymm7 = ymm11[0,1,2,3],ymm7[4,5,6,7] @@ -10345,21 +10164,69 @@ define void @store_i32_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7] ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm0[4,5,6,7] ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vmovaps 128(%r9), %xmm4 -; AVX-NEXT: vmovaps 128(%r8), %xmm5 +; AVX-NEXT: vmovaps 128(%r8), %ymm0 +; AVX-NEXT: vmovaps 128(%r9), %ymm1 +; AVX-NEXT: vmovaps 128(%r10), %ymm2 +; AVX-NEXT: vmovaps 128(%rax), %ymm3 +; AVX-NEXT: vunpcklpd {{.*#+}} ymm4 = ymm3[0],ymm2[0],ymm3[2],ymm2[2] +; AVX-NEXT: vunpcklps {{.*#+}} ymm5 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5] +; AVX-NEXT: vshufps {{.*#+}} ymm6 = ymm5[0,1],ymm4[2,0],ymm5[4,5],ymm4[6,4] +; AVX-NEXT: vmovaps 128(%rdx), %ymm4 +; AVX-NEXT: vmovaps 128(%rcx), %ymm5 +; AVX-NEXT: vunpcklpd {{.*#+}} ymm7 = ymm5[0],ymm4[0],ymm5[2],ymm4[2] +; AVX-NEXT: vextractf128 $1, %ymm7, %xmm7 +; AVX-NEXT: vmovaps 128(%rdi), %ymm8 +; AVX-NEXT: vmovaps 128(%rsi), %ymm9 +; AVX-NEXT: vunpcklps {{.*#+}} ymm10 = ymm8[0],ymm9[0],ymm8[1],ymm9[1],ymm8[4],ymm9[4],ymm8[5],ymm9[5] +; AVX-NEXT: vextractf128 $1, %ymm10, %xmm10 +; AVX-NEXT: vshufps {{.*#+}} xmm7 = xmm10[0,1],xmm7[2,0] +; AVX-NEXT: vblendps {{.*#+}} ymm6 = ymm7[0,1,2,3],ymm6[4,5,6,7] +; AVX-NEXT: vmovups %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vunpcklps {{.*#+}} ymm6 = ymm2[0],ymm3[0],ymm2[1],ymm3[1],ymm2[4],ymm3[4],ymm2[5],ymm3[5] +; AVX-NEXT: vshufps {{.*#+}} ymm7 = ymm1[1,0],ymm0[1,0],ymm1[5,4],ymm0[5,4] +; AVX-NEXT: vshufps {{.*#+}} ymm6 = ymm7[2,0],ymm6[2,3],ymm7[6,4],ymm6[6,7] +; AVX-NEXT: vunpcklps {{.*#+}} ymm7 = ymm4[0],ymm5[0],ymm4[1],ymm5[1],ymm4[4],ymm5[4],ymm4[5],ymm5[5] +; AVX-NEXT: vextractf128 $1, %ymm7, %xmm7 +; AVX-NEXT: vshufps {{.*#+}} ymm10 = ymm9[1,0],ymm8[1,0],ymm9[5,4],ymm8[5,4] +; AVX-NEXT: vextractf128 $1, %ymm10, %xmm10 +; AVX-NEXT: vshufps {{.*#+}} xmm7 = xmm10[2,0],xmm7[2,3] +; AVX-NEXT: vblendps {{.*#+}} ymm6 = ymm7[0,1,2,3],ymm6[4,5,6,7] +; AVX-NEXT: vmovups %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vunpckhpd {{.*#+}} ymm6 = ymm3[1],ymm2[1],ymm3[3],ymm2[3] +; AVX-NEXT: vunpckhps {{.*#+}} ymm7 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7] +; AVX-NEXT: vshufps {{.*#+}} ymm6 = ymm7[0,1],ymm6[2,0],ymm7[4,5],ymm6[6,4] +; AVX-NEXT: vunpckhpd {{.*#+}} ymm7 = ymm5[1],ymm4[1],ymm5[3],ymm4[3] +; AVX-NEXT: vextractf128 $1, %ymm7, %xmm7 +; AVX-NEXT: vunpckhps {{.*#+}} ymm10 = ymm8[2],ymm9[2],ymm8[3],ymm9[3],ymm8[6],ymm9[6],ymm8[7],ymm9[7] +; AVX-NEXT: vextractf128 $1, %ymm10, %xmm10 +; AVX-NEXT: vshufps {{.*#+}} xmm7 = xmm10[0,1],xmm7[2,0] +; AVX-NEXT: vblendps {{.*#+}} ymm6 = ymm7[0,1,2,3],ymm6[4,5,6,7] +; AVX-NEXT: vmovups %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vunpckhps {{.*#+}} ymm2 = ymm2[2],ymm3[2],ymm2[3],ymm3[3],ymm2[6],ymm3[6],ymm2[7],ymm3[7] +; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm1[3,0],ymm0[3,0],ymm1[7,4],ymm0[7,4] +; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm0[2,0],ymm2[2,3],ymm0[6,4],ymm2[6,7] +; AVX-NEXT: vunpckhps {{.*#+}} ymm1 = ymm4[2],ymm5[2],ymm4[3],ymm5[3],ymm4[6],ymm5[6],ymm4[7],ymm5[7] +; AVX-NEXT: vshufps {{.*#+}} ymm2 = ymm9[3,0],ymm8[3,0],ymm9[7,4],ymm8[7,4] +; AVX-NEXT: vextractf128 $1, %ymm1, %xmm1 +; AVX-NEXT: vextractf128 $1, %ymm2, %xmm2 +; AVX-NEXT: vshufps {{.*#+}} xmm1 = xmm2[2,0],xmm1[2,3] +; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7] +; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vmovaps 160(%r9), %xmm4 +; AVX-NEXT: vmovaps 160(%r8), %xmm5 ; AVX-NEXT: vunpcklps {{.*#+}} xmm0 = xmm5[0],xmm4[0],xmm5[1],xmm4[1] ; AVX-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm2 -; AVX-NEXT: vmovaps 128(%rax), %xmm0 -; AVX-NEXT: vmovaps 128(%r10), %xmm1 +; AVX-NEXT: vmovaps 160(%rax), %xmm0 +; AVX-NEXT: vmovaps 160(%r10), %xmm1 ; AVX-NEXT: vunpcklps {{.*#+}} xmm6 = xmm1[0],xmm0[0],xmm1[1],xmm0[1] ; AVX-NEXT: vshufps {{.*#+}} xmm3 = xmm6[0,1,0,1] ; AVX-NEXT: vinsertf128 $1, %xmm3, %ymm0, %ymm3 ; AVX-NEXT: vblendps {{.*#+}} ymm7 = ymm2[0,1,2,3,4,5],ymm3[6,7] -; AVX-NEXT: vmovaps 128(%rcx), %xmm2 -; AVX-NEXT: vmovaps 128(%rdx), %xmm3 +; AVX-NEXT: vmovaps 160(%rcx), %xmm2 +; AVX-NEXT: vmovaps 160(%rdx), %xmm3 ; AVX-NEXT: vunpcklps {{.*#+}} xmm8 = xmm3[0],xmm2[0],xmm3[1],xmm2[1] -; AVX-NEXT: vmovaps 128(%rsi), %xmm9 -; AVX-NEXT: vmovaps 128(%rdi), %xmm10 +; AVX-NEXT: vmovaps 160(%rsi), %xmm9 +; AVX-NEXT: vmovaps 160(%rdi), %xmm10 ; AVX-NEXT: vunpcklps {{.*#+}} xmm11 = xmm10[0],xmm9[0],xmm10[1],xmm9[1] ; AVX-NEXT: vmovlhps {{.*#+}} xmm11 = xmm11[0],xmm8[0] ; AVX-NEXT: vblendps {{.*#+}} ymm7 = ymm11[0,1,2,3],ymm7[4,5,6,7] @@ -10394,22 +10261,70 @@ define void @store_i32_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm1 ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7] ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm0[4,5,6,7] -; AVX-NEXT: vmovups %ymm0, (%rsp) # 32-byte Spill -; AVX-NEXT: vmovaps 160(%r9), %xmm4 -; AVX-NEXT: vmovaps 160(%r8), %xmm5 +; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vmovaps 160(%r8), %ymm0 +; AVX-NEXT: vmovaps 160(%r9), %ymm1 +; AVX-NEXT: vmovaps 160(%r10), %ymm2 +; AVX-NEXT: vmovaps 160(%rax), %ymm3 +; AVX-NEXT: vunpcklpd {{.*#+}} ymm4 = ymm3[0],ymm2[0],ymm3[2],ymm2[2] +; AVX-NEXT: vunpcklps {{.*#+}} ymm5 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5] +; AVX-NEXT: vshufps {{.*#+}} ymm6 = ymm5[0,1],ymm4[2,0],ymm5[4,5],ymm4[6,4] +; AVX-NEXT: vmovaps 160(%rdx), %ymm4 +; AVX-NEXT: vmovaps 160(%rcx), %ymm5 +; AVX-NEXT: vunpcklpd {{.*#+}} ymm7 = ymm5[0],ymm4[0],ymm5[2],ymm4[2] +; AVX-NEXT: vextractf128 $1, %ymm7, %xmm7 +; AVX-NEXT: vmovaps 160(%rdi), %ymm8 +; AVX-NEXT: vmovaps 160(%rsi), %ymm9 +; AVX-NEXT: vunpcklps {{.*#+}} ymm10 = ymm8[0],ymm9[0],ymm8[1],ymm9[1],ymm8[4],ymm9[4],ymm8[5],ymm9[5] +; AVX-NEXT: vextractf128 $1, %ymm10, %xmm10 +; AVX-NEXT: vshufps {{.*#+}} xmm7 = xmm10[0,1],xmm7[2,0] +; AVX-NEXT: vblendps {{.*#+}} ymm6 = ymm7[0,1,2,3],ymm6[4,5,6,7] +; AVX-NEXT: vmovups %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vunpcklps {{.*#+}} ymm6 = ymm2[0],ymm3[0],ymm2[1],ymm3[1],ymm2[4],ymm3[4],ymm2[5],ymm3[5] +; AVX-NEXT: vshufps {{.*#+}} ymm7 = ymm1[1,0],ymm0[1,0],ymm1[5,4],ymm0[5,4] +; AVX-NEXT: vshufps {{.*#+}} ymm6 = ymm7[2,0],ymm6[2,3],ymm7[6,4],ymm6[6,7] +; AVX-NEXT: vunpcklps {{.*#+}} ymm7 = ymm4[0],ymm5[0],ymm4[1],ymm5[1],ymm4[4],ymm5[4],ymm4[5],ymm5[5] +; AVX-NEXT: vextractf128 $1, %ymm7, %xmm7 +; AVX-NEXT: vshufps {{.*#+}} ymm10 = ymm9[1,0],ymm8[1,0],ymm9[5,4],ymm8[5,4] +; AVX-NEXT: vextractf128 $1, %ymm10, %xmm10 +; AVX-NEXT: vshufps {{.*#+}} xmm7 = xmm10[2,0],xmm7[2,3] +; AVX-NEXT: vblendps {{.*#+}} ymm6 = ymm7[0,1,2,3],ymm6[4,5,6,7] +; AVX-NEXT: vmovups %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vunpckhpd {{.*#+}} ymm6 = ymm3[1],ymm2[1],ymm3[3],ymm2[3] +; AVX-NEXT: vunpckhps {{.*#+}} ymm7 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7] +; AVX-NEXT: vshufps {{.*#+}} ymm6 = ymm7[0,1],ymm6[2,0],ymm7[4,5],ymm6[6,4] +; AVX-NEXT: vunpckhpd {{.*#+}} ymm7 = ymm5[1],ymm4[1],ymm5[3],ymm4[3] +; AVX-NEXT: vextractf128 $1, %ymm7, %xmm7 +; AVX-NEXT: vunpckhps {{.*#+}} ymm10 = ymm8[2],ymm9[2],ymm8[3],ymm9[3],ymm8[6],ymm9[6],ymm8[7],ymm9[7] +; AVX-NEXT: vextractf128 $1, %ymm10, %xmm10 +; AVX-NEXT: vshufps {{.*#+}} xmm7 = xmm10[0,1],xmm7[2,0] +; AVX-NEXT: vblendps {{.*#+}} ymm6 = ymm7[0,1,2,3],ymm6[4,5,6,7] +; AVX-NEXT: vmovups %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vunpckhps {{.*#+}} ymm2 = ymm2[2],ymm3[2],ymm2[3],ymm3[3],ymm2[6],ymm3[6],ymm2[7],ymm3[7] +; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm1[3,0],ymm0[3,0],ymm1[7,4],ymm0[7,4] +; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm0[2,0],ymm2[2,3],ymm0[6,4],ymm2[6,7] +; AVX-NEXT: vunpckhps {{.*#+}} ymm1 = ymm4[2],ymm5[2],ymm4[3],ymm5[3],ymm4[6],ymm5[6],ymm4[7],ymm5[7] +; AVX-NEXT: vshufps {{.*#+}} ymm2 = ymm9[3,0],ymm8[3,0],ymm9[7,4],ymm8[7,4] +; AVX-NEXT: vextractf128 $1, %ymm1, %xmm1 +; AVX-NEXT: vextractf128 $1, %ymm2, %xmm2 +; AVX-NEXT: vshufps {{.*#+}} xmm1 = xmm2[2,0],xmm1[2,3] +; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7] +; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vmovaps 192(%r9), %xmm4 +; AVX-NEXT: vmovaps 192(%r8), %xmm5 ; AVX-NEXT: vunpcklps {{.*#+}} xmm0 = xmm5[0],xmm4[0],xmm5[1],xmm4[1] ; AVX-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm2 -; AVX-NEXT: vmovaps 160(%rax), %xmm0 -; AVX-NEXT: vmovaps 160(%r10), %xmm1 +; AVX-NEXT: vmovaps 192(%rax), %xmm0 +; AVX-NEXT: vmovaps 192(%r10), %xmm1 ; AVX-NEXT: vunpcklps {{.*#+}} xmm6 = xmm1[0],xmm0[0],xmm1[1],xmm0[1] ; AVX-NEXT: vshufps {{.*#+}} xmm3 = xmm6[0,1,0,1] ; AVX-NEXT: vinsertf128 $1, %xmm3, %ymm0, %ymm3 ; AVX-NEXT: vblendps {{.*#+}} ymm7 = ymm2[0,1,2,3,4,5],ymm3[6,7] -; AVX-NEXT: vmovaps 160(%rcx), %xmm2 -; AVX-NEXT: vmovaps 160(%rdx), %xmm3 +; AVX-NEXT: vmovaps 192(%rcx), %xmm2 +; AVX-NEXT: vmovaps 192(%rdx), %xmm3 ; AVX-NEXT: vunpcklps {{.*#+}} xmm8 = xmm3[0],xmm2[0],xmm3[1],xmm2[1] -; AVX-NEXT: vmovaps 160(%rsi), %xmm9 -; AVX-NEXT: vmovaps 160(%rdi), %xmm10 +; AVX-NEXT: vmovaps 192(%rsi), %xmm9 +; AVX-NEXT: vmovaps 192(%rdi), %xmm10 ; AVX-NEXT: vunpcklps {{.*#+}} xmm11 = xmm10[0],xmm9[0],xmm10[1],xmm9[1] ; AVX-NEXT: vmovlhps {{.*#+}} xmm11 = xmm11[0],xmm8[0] ; AVX-NEXT: vblendps {{.*#+}} ymm7 = ymm11[0,1,2,3],ymm7[4,5,6,7] @@ -10445,32 +10360,81 @@ define void @store_i32_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7] ; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm0[4,5,6,7] ; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX-NEXT: vmovaps 192(%r9), %xmm4 -; AVX-NEXT: vmovaps 192(%r8), %xmm5 +; AVX-NEXT: vmovaps 192(%r8), %ymm0 +; AVX-NEXT: vmovaps 192(%r9), %ymm1 +; AVX-NEXT: vmovaps 192(%r10), %ymm2 +; AVX-NEXT: vmovaps 192(%rax), %ymm3 +; AVX-NEXT: vunpcklpd {{.*#+}} ymm4 = ymm3[0],ymm2[0],ymm3[2],ymm2[2] +; AVX-NEXT: vunpcklps {{.*#+}} ymm5 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5] +; AVX-NEXT: vshufps {{.*#+}} ymm6 = ymm5[0,1],ymm4[2,0],ymm5[4,5],ymm4[6,4] +; AVX-NEXT: vmovaps 192(%rdx), %ymm4 +; AVX-NEXT: vmovaps 192(%rcx), %ymm5 +; AVX-NEXT: vunpcklpd {{.*#+}} ymm7 = ymm5[0],ymm4[0],ymm5[2],ymm4[2] +; AVX-NEXT: vextractf128 $1, %ymm7, %xmm7 +; AVX-NEXT: vmovaps 192(%rdi), %ymm8 +; AVX-NEXT: vmovaps 192(%rsi), %ymm9 +; AVX-NEXT: vunpcklps {{.*#+}} ymm10 = ymm8[0],ymm9[0],ymm8[1],ymm9[1],ymm8[4],ymm9[4],ymm8[5],ymm9[5] +; AVX-NEXT: vextractf128 $1, %ymm10, %xmm10 +; AVX-NEXT: vshufps {{.*#+}} xmm7 = xmm10[0,1],xmm7[2,0] +; AVX-NEXT: vblendps {{.*#+}} ymm6 = ymm7[0,1,2,3],ymm6[4,5,6,7] +; AVX-NEXT: vmovups %ymm6, (%rsp) # 32-byte Spill +; AVX-NEXT: vunpcklps {{.*#+}} ymm6 = ymm2[0],ymm3[0],ymm2[1],ymm3[1],ymm2[4],ymm3[4],ymm2[5],ymm3[5] +; AVX-NEXT: vshufps {{.*#+}} ymm7 = ymm1[1,0],ymm0[1,0],ymm1[5,4],ymm0[5,4] +; AVX-NEXT: vshufps {{.*#+}} ymm6 = ymm7[2,0],ymm6[2,3],ymm7[6,4],ymm6[6,7] +; AVX-NEXT: vunpcklps {{.*#+}} ymm7 = ymm4[0],ymm5[0],ymm4[1],ymm5[1],ymm4[4],ymm5[4],ymm4[5],ymm5[5] +; AVX-NEXT: vextractf128 $1, %ymm7, %xmm7 +; AVX-NEXT: vshufps {{.*#+}} ymm10 = ymm9[1,0],ymm8[1,0],ymm9[5,4],ymm8[5,4] +; AVX-NEXT: vextractf128 $1, %ymm10, %xmm10 +; AVX-NEXT: vshufps {{.*#+}} xmm7 = xmm10[2,0],xmm7[2,3] +; AVX-NEXT: vblendps {{.*#+}} ymm6 = ymm7[0,1,2,3],ymm6[4,5,6,7] +; AVX-NEXT: vmovups %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vunpckhpd {{.*#+}} ymm6 = ymm3[1],ymm2[1],ymm3[3],ymm2[3] +; AVX-NEXT: vunpckhps {{.*#+}} ymm7 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7] +; AVX-NEXT: vshufps {{.*#+}} ymm6 = ymm7[0,1],ymm6[2,0],ymm7[4,5],ymm6[6,4] +; AVX-NEXT: vunpckhpd {{.*#+}} ymm7 = ymm5[1],ymm4[1],ymm5[3],ymm4[3] +; AVX-NEXT: vextractf128 $1, %ymm7, %xmm7 +; AVX-NEXT: vunpckhps {{.*#+}} ymm10 = ymm8[2],ymm9[2],ymm8[3],ymm9[3],ymm8[6],ymm9[6],ymm8[7],ymm9[7] +; AVX-NEXT: vextractf128 $1, %ymm10, %xmm10 +; AVX-NEXT: vshufps {{.*#+}} xmm7 = xmm10[0,1],xmm7[2,0] +; AVX-NEXT: vblendps {{.*#+}} ymm6 = ymm7[0,1,2,3],ymm6[4,5,6,7] +; AVX-NEXT: vmovups %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vunpckhps {{.*#+}} ymm2 = ymm2[2],ymm3[2],ymm2[3],ymm3[3],ymm2[6],ymm3[6],ymm2[7],ymm3[7] +; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm1[3,0],ymm0[3,0],ymm1[7,4],ymm0[7,4] +; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm0[2,0],ymm2[2,3],ymm0[6,4],ymm2[6,7] +; AVX-NEXT: vunpckhps {{.*#+}} ymm1 = ymm4[2],ymm5[2],ymm4[3],ymm5[3],ymm4[6],ymm5[6],ymm4[7],ymm5[7] +; AVX-NEXT: vshufps {{.*#+}} ymm2 = ymm9[3,0],ymm8[3,0],ymm9[7,4],ymm8[7,4] +; AVX-NEXT: vextractf128 $1, %ymm1, %xmm1 +; AVX-NEXT: vextractf128 $1, %ymm2, %xmm2 +; AVX-NEXT: vshufps {{.*#+}} xmm1 = xmm2[2,0],xmm1[2,3] +; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7] +; AVX-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vmovaps 224(%r9), %xmm4 +; AVX-NEXT: vmovaps 224(%r8), %xmm5 ; AVX-NEXT: vunpcklps {{.*#+}} xmm0 = xmm5[0],xmm4[0],xmm5[1],xmm4[1] ; AVX-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm2 -; AVX-NEXT: vmovaps 192(%rax), %xmm9 -; AVX-NEXT: vmovaps 192(%r10), %xmm1 +; AVX-NEXT: vmovaps 224(%rax), %xmm9 +; AVX-NEXT: vmovaps 224(%r10), %xmm1 ; AVX-NEXT: vunpcklps {{.*#+}} xmm6 = xmm1[0],xmm9[0],xmm1[1],xmm9[1] ; AVX-NEXT: vshufps {{.*#+}} xmm3 = xmm6[0,1,0,1] ; AVX-NEXT: vinsertf128 $1, %xmm3, %ymm0, %ymm3 ; AVX-NEXT: vblendps {{.*#+}} ymm10 = ymm2[0,1,2,3,4,5],ymm3[6,7] -; AVX-NEXT: vmovaps 192(%rcx), %xmm2 -; AVX-NEXT: vmovaps 192(%rdx), %xmm3 +; AVX-NEXT: vmovaps 224(%rcx), %xmm2 +; AVX-NEXT: vmovaps 224(%rdx), %xmm3 ; AVX-NEXT: vunpcklps {{.*#+}} xmm8 = xmm3[0],xmm2[0],xmm3[1],xmm2[1] -; AVX-NEXT: vmovaps 192(%rsi), %xmm0 -; AVX-NEXT: vmovaps 192(%rdi), %xmm7 -; AVX-NEXT: vunpcklps {{.*#+}} xmm15 = xmm7[0],xmm0[0],xmm7[1],xmm0[1] -; AVX-NEXT: vmovlhps {{.*#+}} xmm15 = xmm15[0],xmm8[0] -; AVX-NEXT: vblendps {{.*#+}} ymm13 = ymm15[0,1,2,3],ymm10[4,5,6,7] -; AVX-NEXT: vshufps {{.*#+}} xmm15 = xmm7[1,1,1,1] -; AVX-NEXT: vblendps {{.*#+}} xmm15 = xmm15[0],xmm0[1],xmm15[2,3] -; AVX-NEXT: vblendps {{.*#+}} xmm8 = xmm15[0,1],xmm8[2,3] +; AVX-NEXT: vmovaps 224(%rsi), %xmm0 +; AVX-NEXT: vmovaps 224(%rdi), %xmm7 +; AVX-NEXT: vunpcklps {{.*#+}} xmm11 = xmm7[0],xmm0[0],xmm7[1],xmm0[1] +; AVX-NEXT: vmovlhps {{.*#+}} xmm11 = xmm11[0],xmm8[0] +; AVX-NEXT: vblendps {{.*#+}} ymm10 = ymm11[0,1,2,3],ymm10[4,5,6,7] +; AVX-NEXT: vmovups %ymm10, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX-NEXT: vshufps {{.*#+}} xmm10 = xmm7[1,1,1,1] +; AVX-NEXT: vblendps {{.*#+}} xmm10 = xmm10[0],xmm0[1],xmm10[2,3] +; AVX-NEXT: vblendps {{.*#+}} xmm8 = xmm10[0,1],xmm8[2,3] ; AVX-NEXT: vinsertf128 $1, %xmm6, %ymm0, %ymm6 -; AVX-NEXT: vshufps {{.*#+}} xmm15 = xmm5[1,1,1,1] -; AVX-NEXT: vblendps {{.*#+}} xmm15 = xmm15[0],xmm4[1],xmm15[2,3] -; AVX-NEXT: vinsertf128 $1, %xmm15, %ymm0, %ymm15 -; AVX-NEXT: vblendps {{.*#+}} ymm6 = ymm15[0,1,2,3,4,5],ymm6[6,7] +; AVX-NEXT: vshufps {{.*#+}} xmm10 = xmm5[1,1,1,1] +; AVX-NEXT: vblendps {{.*#+}} xmm10 = xmm10[0],xmm4[1],xmm10[2,3] +; AVX-NEXT: vinsertf128 $1, %xmm10, %ymm0, %ymm10 +; AVX-NEXT: vblendps {{.*#+}} ymm6 = ymm10[0,1,2,3,4,5],ymm6[6,7] ; AVX-NEXT: vblendps {{.*#+}} ymm12 = ymm8[0,1,2,3],ymm6[4,5,6,7] ; AVX-NEXT: vunpckhps {{.*#+}} xmm0 = xmm7[2],xmm0[2],xmm7[3],xmm0[3] ; AVX-NEXT: vunpckhps {{.*#+}} xmm4 = xmm5[2],xmm4[2],xmm5[3],xmm4[3] @@ -10491,126 +10455,77 @@ define void @store_i32_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm2 ; AVX-NEXT: vblendps {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5],ymm1[6,7] ; AVX-NEXT: vblendps {{.*#+}} ymm10 = ymm0[0,1,2,3],ymm1[4,5,6,7] -; AVX-NEXT: vmovaps 224(%r9), %xmm3 -; AVX-NEXT: vmovaps 224(%r8), %xmm9 -; AVX-NEXT: vunpcklps {{.*#+}} xmm0 = xmm9[0],xmm3[0],xmm9[1],xmm3[1] -; AVX-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0 -; AVX-NEXT: vmovaps 224(%rax), %xmm7 -; AVX-NEXT: vmovaps 224(%r10), %xmm6 -; AVX-NEXT: vunpcklps {{.*#+}} xmm15 = xmm6[0],xmm7[0],xmm6[1],xmm7[1] -; AVX-NEXT: vshufps {{.*#+}} xmm2 = xmm15[0,1,0,1] -; AVX-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm2 -; AVX-NEXT: vblendps {{.*#+}} ymm8 = ymm0[0,1,2,3,4,5],ymm2[6,7] -; AVX-NEXT: vmovaps 224(%rcx), %xmm5 -; AVX-NEXT: vmovaps 224(%rdx), %xmm4 -; AVX-NEXT: vunpcklps {{.*#+}} xmm2 = xmm4[0],xmm5[0],xmm4[1],xmm5[1] -; AVX-NEXT: vmovaps 224(%rsi), %xmm1 -; AVX-NEXT: vmovaps 224(%rdi), %xmm0 -; AVX-NEXT: vunpcklps {{.*#+}} xmm14 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] -; AVX-NEXT: vmovlhps {{.*#+}} xmm14 = xmm14[0],xmm2[0] +; AVX-NEXT: vmovaps 224(%r8), %ymm7 +; AVX-NEXT: vmovaps 224(%r9), %ymm6 +; AVX-NEXT: vmovaps 224(%r10), %ymm5 +; AVX-NEXT: vmovaps 224(%rax), %ymm4 +; AVX-NEXT: vunpcklpd {{.*#+}} ymm0 = ymm4[0],ymm5[0],ymm4[2],ymm5[2] +; AVX-NEXT: vunpcklps {{.*#+}} ymm1 = ymm7[0],ymm6[0],ymm7[1],ymm6[1],ymm7[4],ymm6[4],ymm7[5],ymm6[5] +; AVX-NEXT: vshufps {{.*#+}} ymm8 = ymm1[0,1],ymm0[2,0],ymm1[4,5],ymm0[6,4] +; AVX-NEXT: vmovaps 224(%rdx), %ymm3 +; AVX-NEXT: vmovaps 224(%rcx), %ymm2 +; AVX-NEXT: vunpcklpd {{.*#+}} ymm9 = ymm2[0],ymm3[0],ymm2[2],ymm3[2] +; AVX-NEXT: vextractf128 $1, %ymm9, %xmm9 +; AVX-NEXT: vmovaps 224(%rdi), %ymm1 +; AVX-NEXT: vmovaps 224(%rsi), %ymm0 +; AVX-NEXT: vunpcklps {{.*#+}} ymm15 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[4],ymm0[4],ymm1[5],ymm0[5] +; AVX-NEXT: vextractf128 $1, %ymm15, %xmm15 +; AVX-NEXT: vshufps {{.*#+}} xmm9 = xmm15[0,1],xmm9[2,0] +; AVX-NEXT: vblendps {{.*#+}} ymm9 = ymm9[0,1,2,3],ymm8[4,5,6,7] +; AVX-NEXT: vunpcklps {{.*#+}} ymm8 = ymm5[0],ymm4[0],ymm5[1],ymm4[1],ymm5[4],ymm4[4],ymm5[5],ymm4[5] +; AVX-NEXT: vshufps {{.*#+}} ymm15 = ymm6[1,0],ymm7[1,0],ymm6[5,4],ymm7[5,4] +; AVX-NEXT: vshufps {{.*#+}} ymm8 = ymm15[2,0],ymm8[2,3],ymm15[6,4],ymm8[6,7] +; AVX-NEXT: vunpcklps {{.*#+}} ymm15 = ymm3[0],ymm2[0],ymm3[1],ymm2[1],ymm3[4],ymm2[4],ymm3[5],ymm2[5] +; AVX-NEXT: vextractf128 $1, %ymm15, %xmm15 +; AVX-NEXT: vshufps {{.*#+}} ymm14 = ymm0[1,0],ymm1[1,0],ymm0[5,4],ymm1[5,4] +; AVX-NEXT: vextractf128 $1, %ymm14, %xmm14 +; AVX-NEXT: vshufps {{.*#+}} xmm14 = xmm14[2,0],xmm15[2,3] ; AVX-NEXT: vblendps {{.*#+}} ymm8 = ymm14[0,1,2,3],ymm8[4,5,6,7] -; AVX-NEXT: vshufps {{.*#+}} xmm14 = xmm0[1,1,1,1] -; AVX-NEXT: vblendps {{.*#+}} xmm14 = xmm14[0],xmm1[1],xmm14[2,3] -; AVX-NEXT: vblendps {{.*#+}} xmm2 = xmm14[0,1],xmm2[2,3] -; AVX-NEXT: vinsertf128 $1, %xmm15, %ymm0, %ymm14 -; AVX-NEXT: vshufps {{.*#+}} xmm15 = xmm9[1,1,1,1] -; AVX-NEXT: vblendps {{.*#+}} xmm15 = xmm15[0],xmm3[1],xmm15[2,3] -; AVX-NEXT: vinsertf128 $1, %xmm15, %ymm0, %ymm15 -; AVX-NEXT: vblendps {{.*#+}} ymm14 = ymm15[0,1,2,3,4,5],ymm14[6,7] -; AVX-NEXT: vblendps {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm14[4,5,6,7] -; AVX-NEXT: vunpckhps {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3] -; AVX-NEXT: vunpckhps {{.*#+}} xmm1 = xmm9[2],xmm3[2],xmm9[3],xmm3[3] -; AVX-NEXT: vshufps {{.*#+}} xmm3 = xmm7[2,2,2,2] -; AVX-NEXT: vblendps {{.*#+}} xmm3 = xmm6[0,1,2],xmm3[3] -; AVX-NEXT: vinsertf128 $1, %xmm3, %ymm0, %ymm3 -; AVX-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm9 -; AVX-NEXT: vblendps {{.*#+}} ymm3 = ymm9[0,1,2,3,4,5],ymm3[6,7] -; AVX-NEXT: vshufps {{.*#+}} xmm9 = xmm5[2,2,2,2] -; AVX-NEXT: vblendps {{.*#+}} xmm9 = xmm4[0,1,2],xmm9[3] -; AVX-NEXT: vblendps {{.*#+}} xmm9 = xmm0[0,1],xmm9[2,3] -; AVX-NEXT: vblendps {{.*#+}} ymm3 = ymm9[0,1,2,3],ymm3[4,5,6,7] -; AVX-NEXT: vunpckhps {{.*#+}} xmm4 = xmm4[2],xmm5[2],xmm4[3],xmm5[3] -; AVX-NEXT: vunpckhpd {{.*#+}} xmm0 = xmm0[1],xmm4[1] -; AVX-NEXT: vunpckhps {{.*#+}} xmm4 = xmm6[2],xmm7[2],xmm6[3],xmm7[3] -; AVX-NEXT: vinsertf128 $1, %xmm4, %ymm0, %ymm4 -; AVX-NEXT: vshufps {{.*#+}} xmm1 = xmm1[2,3,2,3] -; AVX-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm1 -; AVX-NEXT: vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5],ymm4[6,7] -; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7] +; AVX-NEXT: vunpckhpd {{.*#+}} ymm14 = ymm4[1],ymm5[1],ymm4[3],ymm5[3] +; AVX-NEXT: vunpckhps {{.*#+}} ymm15 = ymm7[2],ymm6[2],ymm7[3],ymm6[3],ymm7[6],ymm6[6],ymm7[7],ymm6[7] +; AVX-NEXT: vshufps {{.*#+}} ymm14 = ymm15[0,1],ymm14[2,0],ymm15[4,5],ymm14[6,4] +; AVX-NEXT: vunpckhpd {{.*#+}} ymm15 = ymm2[1],ymm3[1],ymm2[3],ymm3[3] +; AVX-NEXT: vextractf128 $1, %ymm15, %xmm15 +; AVX-NEXT: vunpckhps {{.*#+}} ymm13 = ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[6],ymm0[6],ymm1[7],ymm0[7] +; AVX-NEXT: vextractf128 $1, %ymm13, %xmm13 +; AVX-NEXT: vshufps {{.*#+}} xmm13 = xmm13[0,1],xmm15[2,0] +; AVX-NEXT: vblendps {{.*#+}} ymm13 = ymm13[0,1,2,3],ymm14[4,5,6,7] +; AVX-NEXT: vunpckhps {{.*#+}} ymm4 = ymm5[2],ymm4[2],ymm5[3],ymm4[3],ymm5[6],ymm4[6],ymm5[7],ymm4[7] +; AVX-NEXT: vshufps {{.*#+}} ymm5 = ymm6[3,0],ymm7[3,0],ymm6[7,4],ymm7[7,4] +; AVX-NEXT: vshufps {{.*#+}} ymm4 = ymm5[2,0],ymm4[2,3],ymm5[6,4],ymm4[6,7] +; AVX-NEXT: vunpckhps {{.*#+}} ymm2 = ymm3[2],ymm2[2],ymm3[3],ymm2[3],ymm3[6],ymm2[6],ymm3[7],ymm2[7] +; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm0[3,0],ymm1[3,0],ymm0[7,4],ymm1[7,4] +; AVX-NEXT: vextractf128 $1, %ymm2, %xmm1 +; AVX-NEXT: vextractf128 $1, %ymm0, %xmm0 +; AVX-NEXT: vshufps {{.*#+}} xmm0 = xmm0[2,0],xmm1[2,3] +; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm4[4,5,6,7] ; AVX-NEXT: movq {{[0-9]+}}(%rsp), %rax -; AVX-NEXT: vmovaps %ymm0, 1888(%rax) -; AVX-NEXT: vmovaps %ymm3, 1856(%rax) -; AVX-NEXT: vmovaps %ymm2, 1824(%rax) -; AVX-NEXT: vmovaps %ymm8, 1792(%rax) -; AVX-NEXT: vmovaps %ymm10, 1632(%rax) -; AVX-NEXT: vmovaps %ymm11, 1600(%rax) -; AVX-NEXT: vmovaps %ymm12, 1568(%rax) -; AVX-NEXT: vmovaps %ymm13, 1536(%rax) -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm0, 1376(%rax) -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm0, 1344(%rax) -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm0, 1312(%rax) -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm0, 1280(%rax) -; AVX-NEXT: vmovups (%rsp), %ymm0 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm0, 1120(%rax) -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm0, 1088(%rax) -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm0, 1056(%rax) -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm0, 1024(%rax) -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm0, 864(%rax) -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm0, 832(%rax) -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm0, 800(%rax) -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm0, 768(%rax) -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm0, 608(%rax) -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm0, 576(%rax) -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm0, 544(%rax) -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm0, 512(%rax) -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm0, 352(%rax) -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm0, 320(%rax) -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm0, 288(%rax) -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm0, 256(%rax) -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm0, 96(%rax) -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm0, 64(%rax) -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm0, 32(%rax) -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm0, (%rax) -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload ; AVX-NEXT: vmovaps %ymm0, 2016(%rax) +; AVX-NEXT: vmovaps %ymm13, 1984(%rax) +; AVX-NEXT: vmovaps %ymm8, 1952(%rax) +; AVX-NEXT: vmovaps %ymm9, 1920(%rax) +; AVX-NEXT: vmovaps %ymm10, 1888(%rax) +; AVX-NEXT: vmovaps %ymm11, 1856(%rax) +; AVX-NEXT: vmovaps %ymm12, 1824(%rax) ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm0, 1984(%rax) -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm0, 1952(%rax) -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload -; AVX-NEXT: vmovaps %ymm0, 1920(%rax) +; AVX-NEXT: vmovaps %ymm0, 1792(%rax) ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload ; AVX-NEXT: vmovaps %ymm0, 1760(%rax) ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload ; AVX-NEXT: vmovaps %ymm0, 1728(%rax) ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload ; AVX-NEXT: vmovaps %ymm0, 1696(%rax) -; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload +; AVX-NEXT: vmovups (%rsp), %ymm0 # 32-byte Reload ; AVX-NEXT: vmovaps %ymm0, 1664(%rax) ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm0, 1632(%rax) +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm0, 1600(%rax) +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm0, 1568(%rax) +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm0, 1536(%rax) +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload ; AVX-NEXT: vmovaps %ymm0, 1504(%rax) ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload ; AVX-NEXT: vmovaps %ymm0, 1472(%rax) @@ -10619,6 +10534,14 @@ define void @store_i32_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload ; AVX-NEXT: vmovaps %ymm0, 1408(%rax) ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm0, 1376(%rax) +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm0, 1344(%rax) +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm0, 1312(%rax) +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm0, 1280(%rax) +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload ; AVX-NEXT: vmovaps %ymm0, 1248(%rax) ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload ; AVX-NEXT: vmovaps %ymm0, 1216(%rax) @@ -10627,6 +10550,14 @@ define void @store_i32_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload ; AVX-NEXT: vmovaps %ymm0, 1152(%rax) ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm0, 1120(%rax) +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm0, 1088(%rax) +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm0, 1056(%rax) +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm0, 1024(%rax) +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload ; AVX-NEXT: vmovaps %ymm0, 992(%rax) ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload ; AVX-NEXT: vmovaps %ymm0, 960(%rax) @@ -10635,6 +10566,14 @@ define void @store_i32_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload ; AVX-NEXT: vmovaps %ymm0, 896(%rax) ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm0, 864(%rax) +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm0, 832(%rax) +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm0, 800(%rax) +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm0, 768(%rax) +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload ; AVX-NEXT: vmovaps %ymm0, 736(%rax) ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload ; AVX-NEXT: vmovaps %ymm0, 704(%rax) @@ -10643,6 +10582,14 @@ define void @store_i32_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload ; AVX-NEXT: vmovaps %ymm0, 640(%rax) ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm0, 608(%rax) +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm0, 576(%rax) +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm0, 544(%rax) +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm0, 512(%rax) +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload ; AVX-NEXT: vmovaps %ymm0, 480(%rax) ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload ; AVX-NEXT: vmovaps %ymm0, 448(%rax) @@ -10651,6 +10598,14 @@ define void @store_i32_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload ; AVX-NEXT: vmovaps %ymm0, 384(%rax) ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm0, 352(%rax) +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm0, 320(%rax) +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm0, 288(%rax) +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm0, 256(%rax) +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload ; AVX-NEXT: vmovaps %ymm0, 224(%rax) ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload ; AVX-NEXT: vmovaps %ymm0, 192(%rax) @@ -10658,7 +10613,15 @@ define void @store_i32_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX-NEXT: vmovaps %ymm0, 160(%rax) ; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload ; AVX-NEXT: vmovaps %ymm0, 128(%rax) -; AVX-NEXT: addq $1672, %rsp # imm = 0x688 +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm0, 96(%rax) +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm0, 64(%rax) +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm0, 32(%rax) +; AVX-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload +; AVX-NEXT: vmovaps %ymm0, (%rax) +; AVX-NEXT: addq $1704, %rsp # imm = 0x6A8 ; AVX-NEXT: vzeroupper ; AVX-NEXT: retq ; diff --git a/llvm/test/CodeGen/X86/vector-interleaved-store-i64-stride-5.ll b/llvm/test/CodeGen/X86/vector-interleaved-store-i64-stride-5.ll index 8dded96acfc1..bb609b9938a8 100644 --- a/llvm/test/CodeGen/X86/vector-interleaved-store-i64-stride-5.ll +++ b/llvm/test/CodeGen/X86/vector-interleaved-store-i64-stride-5.ll @@ -123,12 +123,12 @@ define void @store_i64_stride5_vf2(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512-NEXT: vinserti128 $1, (%rcx), %ymm1, %ymm1 ; AVX512-NEXT: vinserti128 $1, (%rsi), %ymm0, %ymm0 ; AVX512-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0 -; AVX512-NEXT: vpmovsxbq {{.*#+}} xmm1 = [7,9] +; AVX512-NEXT: vpmovsxbq {{.*#+}} xmm3 = [3,9] +; AVX512-NEXT: vpermi2q %zmm2, %zmm1, %zmm3 +; AVX512-NEXT: vpmovsxbq {{.*#+}} zmm1 = [0,2,4,6,8,1,3,5] ; AVX512-NEXT: vpermi2q %zmm2, %zmm0, %zmm1 -; AVX512-NEXT: vpmovsxbq {{.*#+}} zmm3 = [0,2,4,6,8,1,3,5] -; AVX512-NEXT: vpermi2q %zmm2, %zmm0, %zmm3 -; AVX512-NEXT: vmovdqa64 %zmm3, (%r9) -; AVX512-NEXT: vmovdqa %xmm1, 64(%r9) +; AVX512-NEXT: vmovdqa %xmm3, 64(%r9) +; AVX512-NEXT: vmovdqa64 %zmm1, (%r9) ; AVX512-NEXT: vzeroupper ; AVX512-NEXT: retq ; @@ -140,12 +140,12 @@ define void @store_i64_stride5_vf2(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512-FCP-NEXT: vinserti128 $1, (%rcx), %ymm1, %ymm1 ; AVX512-FCP-NEXT: vinserti128 $1, (%rsi), %ymm0, %ymm0 ; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0 -; AVX512-FCP-NEXT: vpmovsxbq {{.*#+}} xmm1 = [7,9] +; AVX512-FCP-NEXT: vpmovsxbq {{.*#+}} xmm3 = [3,9] +; AVX512-FCP-NEXT: vpermi2q %zmm2, %zmm1, %zmm3 +; AVX512-FCP-NEXT: vpmovsxbq {{.*#+}} zmm1 = [0,2,4,6,8,1,3,5] ; AVX512-FCP-NEXT: vpermi2q %zmm2, %zmm0, %zmm1 -; AVX512-FCP-NEXT: vpmovsxbq {{.*#+}} zmm3 = [0,2,4,6,8,1,3,5] -; AVX512-FCP-NEXT: vpermi2q %zmm2, %zmm0, %zmm3 -; AVX512-FCP-NEXT: vmovdqa64 %zmm3, (%r9) -; AVX512-FCP-NEXT: vmovdqa %xmm1, 64(%r9) +; AVX512-FCP-NEXT: vmovdqa %xmm3, 64(%r9) +; AVX512-FCP-NEXT: vmovdqa64 %zmm1, (%r9) ; AVX512-FCP-NEXT: vzeroupper ; AVX512-FCP-NEXT: retq ; @@ -157,12 +157,12 @@ define void @store_i64_stride5_vf2(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512DQ-NEXT: vinserti128 $1, (%rcx), %ymm1, %ymm1 ; AVX512DQ-NEXT: vinserti128 $1, (%rsi), %ymm0, %ymm0 ; AVX512DQ-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0 -; AVX512DQ-NEXT: vpmovsxbq {{.*#+}} xmm1 = [7,9] +; AVX512DQ-NEXT: vpmovsxbq {{.*#+}} xmm3 = [3,9] +; AVX512DQ-NEXT: vpermi2q %zmm2, %zmm1, %zmm3 +; AVX512DQ-NEXT: vpmovsxbq {{.*#+}} zmm1 = [0,2,4,6,8,1,3,5] ; AVX512DQ-NEXT: vpermi2q %zmm2, %zmm0, %zmm1 -; AVX512DQ-NEXT: vpmovsxbq {{.*#+}} zmm3 = [0,2,4,6,8,1,3,5] -; AVX512DQ-NEXT: vpermi2q %zmm2, %zmm0, %zmm3 -; AVX512DQ-NEXT: vmovdqa64 %zmm3, (%r9) -; AVX512DQ-NEXT: vmovdqa %xmm1, 64(%r9) +; AVX512DQ-NEXT: vmovdqa %xmm3, 64(%r9) +; AVX512DQ-NEXT: vmovdqa64 %zmm1, (%r9) ; AVX512DQ-NEXT: vzeroupper ; AVX512DQ-NEXT: retq ; @@ -174,12 +174,12 @@ define void @store_i64_stride5_vf2(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512DQ-FCP-NEXT: vinserti128 $1, (%rcx), %ymm1, %ymm1 ; AVX512DQ-FCP-NEXT: vinserti128 $1, (%rsi), %ymm0, %ymm0 ; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0 -; AVX512DQ-FCP-NEXT: vpmovsxbq {{.*#+}} xmm1 = [7,9] +; AVX512DQ-FCP-NEXT: vpmovsxbq {{.*#+}} xmm3 = [3,9] +; AVX512DQ-FCP-NEXT: vpermi2q %zmm2, %zmm1, %zmm3 +; AVX512DQ-FCP-NEXT: vpmovsxbq {{.*#+}} zmm1 = [0,2,4,6,8,1,3,5] ; AVX512DQ-FCP-NEXT: vpermi2q %zmm2, %zmm0, %zmm1 -; AVX512DQ-FCP-NEXT: vpmovsxbq {{.*#+}} zmm3 = [0,2,4,6,8,1,3,5] -; AVX512DQ-FCP-NEXT: vpermi2q %zmm2, %zmm0, %zmm3 -; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm3, (%r9) -; AVX512DQ-FCP-NEXT: vmovdqa %xmm1, 64(%r9) +; AVX512DQ-FCP-NEXT: vmovdqa %xmm3, 64(%r9) +; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm1, (%r9) ; AVX512DQ-FCP-NEXT: vzeroupper ; AVX512DQ-FCP-NEXT: retq ; @@ -191,12 +191,12 @@ define void @store_i64_stride5_vf2(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512BW-NEXT: vinserti128 $1, (%rcx), %ymm1, %ymm1 ; AVX512BW-NEXT: vinserti128 $1, (%rsi), %ymm0, %ymm0 ; AVX512BW-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0 -; AVX512BW-NEXT: vpmovsxbq {{.*#+}} xmm1 = [7,9] +; AVX512BW-NEXT: vpmovsxbq {{.*#+}} xmm3 = [3,9] +; AVX512BW-NEXT: vpermi2q %zmm2, %zmm1, %zmm3 +; AVX512BW-NEXT: vpmovsxbq {{.*#+}} zmm1 = [0,2,4,6,8,1,3,5] ; AVX512BW-NEXT: vpermi2q %zmm2, %zmm0, %zmm1 -; AVX512BW-NEXT: vpmovsxbq {{.*#+}} zmm3 = [0,2,4,6,8,1,3,5] -; AVX512BW-NEXT: vpermi2q %zmm2, %zmm0, %zmm3 -; AVX512BW-NEXT: vmovdqa64 %zmm3, (%r9) -; AVX512BW-NEXT: vmovdqa %xmm1, 64(%r9) +; AVX512BW-NEXT: vmovdqa %xmm3, 64(%r9) +; AVX512BW-NEXT: vmovdqa64 %zmm1, (%r9) ; AVX512BW-NEXT: vzeroupper ; AVX512BW-NEXT: retq ; @@ -208,12 +208,12 @@ define void @store_i64_stride5_vf2(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512BW-FCP-NEXT: vinserti128 $1, (%rcx), %ymm1, %ymm1 ; AVX512BW-FCP-NEXT: vinserti128 $1, (%rsi), %ymm0, %ymm0 ; AVX512BW-FCP-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0 -; AVX512BW-FCP-NEXT: vpmovsxbq {{.*#+}} xmm1 = [7,9] +; AVX512BW-FCP-NEXT: vpmovsxbq {{.*#+}} xmm3 = [3,9] +; AVX512BW-FCP-NEXT: vpermi2q %zmm2, %zmm1, %zmm3 +; AVX512BW-FCP-NEXT: vpmovsxbq {{.*#+}} zmm1 = [0,2,4,6,8,1,3,5] ; AVX512BW-FCP-NEXT: vpermi2q %zmm2, %zmm0, %zmm1 -; AVX512BW-FCP-NEXT: vpmovsxbq {{.*#+}} zmm3 = [0,2,4,6,8,1,3,5] -; AVX512BW-FCP-NEXT: vpermi2q %zmm2, %zmm0, %zmm3 -; AVX512BW-FCP-NEXT: vmovdqa64 %zmm3, (%r9) -; AVX512BW-FCP-NEXT: vmovdqa %xmm1, 64(%r9) +; AVX512BW-FCP-NEXT: vmovdqa %xmm3, 64(%r9) +; AVX512BW-FCP-NEXT: vmovdqa64 %zmm1, (%r9) ; AVX512BW-FCP-NEXT: vzeroupper ; AVX512BW-FCP-NEXT: retq ; @@ -225,12 +225,12 @@ define void @store_i64_stride5_vf2(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512DQ-BW-NEXT: vinserti128 $1, (%rcx), %ymm1, %ymm1 ; AVX512DQ-BW-NEXT: vinserti128 $1, (%rsi), %ymm0, %ymm0 ; AVX512DQ-BW-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0 -; AVX512DQ-BW-NEXT: vpmovsxbq {{.*#+}} xmm1 = [7,9] +; AVX512DQ-BW-NEXT: vpmovsxbq {{.*#+}} xmm3 = [3,9] +; AVX512DQ-BW-NEXT: vpermi2q %zmm2, %zmm1, %zmm3 +; AVX512DQ-BW-NEXT: vpmovsxbq {{.*#+}} zmm1 = [0,2,4,6,8,1,3,5] ; AVX512DQ-BW-NEXT: vpermi2q %zmm2, %zmm0, %zmm1 -; AVX512DQ-BW-NEXT: vpmovsxbq {{.*#+}} zmm3 = [0,2,4,6,8,1,3,5] -; AVX512DQ-BW-NEXT: vpermi2q %zmm2, %zmm0, %zmm3 -; AVX512DQ-BW-NEXT: vmovdqa64 %zmm3, (%r9) -; AVX512DQ-BW-NEXT: vmovdqa %xmm1, 64(%r9) +; AVX512DQ-BW-NEXT: vmovdqa %xmm3, 64(%r9) +; AVX512DQ-BW-NEXT: vmovdqa64 %zmm1, (%r9) ; AVX512DQ-BW-NEXT: vzeroupper ; AVX512DQ-BW-NEXT: retq ; @@ -242,12 +242,12 @@ define void @store_i64_stride5_vf2(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512DQ-BW-FCP-NEXT: vinserti128 $1, (%rcx), %ymm1, %ymm1 ; AVX512DQ-BW-FCP-NEXT: vinserti128 $1, (%rsi), %ymm0, %ymm0 ; AVX512DQ-BW-FCP-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0 -; AVX512DQ-BW-FCP-NEXT: vpmovsxbq {{.*#+}} xmm1 = [7,9] +; AVX512DQ-BW-FCP-NEXT: vpmovsxbq {{.*#+}} xmm3 = [3,9] +; AVX512DQ-BW-FCP-NEXT: vpermi2q %zmm2, %zmm1, %zmm3 +; AVX512DQ-BW-FCP-NEXT: vpmovsxbq {{.*#+}} zmm1 = [0,2,4,6,8,1,3,5] ; AVX512DQ-BW-FCP-NEXT: vpermi2q %zmm2, %zmm0, %zmm1 -; AVX512DQ-BW-FCP-NEXT: vpmovsxbq {{.*#+}} zmm3 = [0,2,4,6,8,1,3,5] -; AVX512DQ-BW-FCP-NEXT: vpermi2q %zmm2, %zmm0, %zmm3 -; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm3, (%r9) -; AVX512DQ-BW-FCP-NEXT: vmovdqa %xmm1, 64(%r9) +; AVX512DQ-BW-FCP-NEXT: vmovdqa %xmm3, 64(%r9) +; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm1, (%r9) ; AVX512DQ-BW-FCP-NEXT: vzeroupper ; AVX512DQ-BW-FCP-NEXT: retq %in.vec0 = load <2 x i64>, ptr %in.vecptr0, align 64 diff --git a/llvm/test/CodeGen/X86/vector-interleaved-store-i64-stride-6.ll b/llvm/test/CodeGen/X86/vector-interleaved-store-i64-stride-6.ll index e70975addc67..c2f1723d8031 100644 --- a/llvm/test/CodeGen/X86/vector-interleaved-store-i64-stride-6.ll +++ b/llvm/test/CodeGen/X86/vector-interleaved-store-i64-stride-6.ll @@ -138,13 +138,13 @@ define void @store_i64_stride6_vf2(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512-NEXT: vinserti128 $1, (%rcx), %ymm1, %ymm1 ; AVX512-NEXT: vinserti128 $1, (%rsi), %ymm0, %ymm0 ; AVX512-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0 -; AVX512-NEXT: vinserti32x4 $1, (%r9), %zmm2, %zmm1 -; AVX512-NEXT: vpmovsxbq {{.*#+}} ymm2 = [5,7,9,11] -; AVX512-NEXT: vpermi2q %zmm1, %zmm0, %zmm2 -; AVX512-NEXT: vpmovsxbq {{.*#+}} zmm3 = [0,2,4,6,8,10,1,3] -; AVX512-NEXT: vpermi2q %zmm1, %zmm0, %zmm3 -; AVX512-NEXT: vmovdqa64 %zmm3, (%rax) -; AVX512-NEXT: vmovdqa %ymm2, 64(%rax) +; AVX512-NEXT: vinserti32x4 $1, (%r9), %zmm2, %zmm2 +; AVX512-NEXT: vpmovsxbq {{.*#+}} ymm3 = [1,3,9,11] +; AVX512-NEXT: vpermi2q %zmm2, %zmm1, %zmm3 +; AVX512-NEXT: vpmovsxbq {{.*#+}} zmm1 = [0,2,4,6,8,10,1,3] +; AVX512-NEXT: vpermi2q %zmm2, %zmm0, %zmm1 +; AVX512-NEXT: vmovdqa %ymm3, 64(%rax) +; AVX512-NEXT: vmovdqa64 %zmm1, (%rax) ; AVX512-NEXT: vzeroupper ; AVX512-NEXT: retq ; @@ -157,13 +157,13 @@ define void @store_i64_stride6_vf2(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512-FCP-NEXT: vinserti128 $1, (%rcx), %ymm1, %ymm1 ; AVX512-FCP-NEXT: vinserti128 $1, (%rsi), %ymm0, %ymm0 ; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0 -; AVX512-FCP-NEXT: vinserti32x4 $1, (%r9), %zmm2, %zmm1 -; AVX512-FCP-NEXT: vpmovsxbq {{.*#+}} ymm2 = [5,7,9,11] -; AVX512-FCP-NEXT: vpermi2q %zmm1, %zmm0, %zmm2 -; AVX512-FCP-NEXT: vpmovsxbq {{.*#+}} zmm3 = [0,2,4,6,8,10,1,3] -; AVX512-FCP-NEXT: vpermi2q %zmm1, %zmm0, %zmm3 -; AVX512-FCP-NEXT: vmovdqa64 %zmm3, (%rax) -; AVX512-FCP-NEXT: vmovdqa %ymm2, 64(%rax) +; AVX512-FCP-NEXT: vinserti32x4 $1, (%r9), %zmm2, %zmm2 +; AVX512-FCP-NEXT: vpmovsxbq {{.*#+}} ymm3 = [1,3,9,11] +; AVX512-FCP-NEXT: vpermi2q %zmm2, %zmm1, %zmm3 +; AVX512-FCP-NEXT: vpmovsxbq {{.*#+}} zmm1 = [0,2,4,6,8,10,1,3] +; AVX512-FCP-NEXT: vpermi2q %zmm2, %zmm0, %zmm1 +; AVX512-FCP-NEXT: vmovdqa %ymm3, 64(%rax) +; AVX512-FCP-NEXT: vmovdqa64 %zmm1, (%rax) ; AVX512-FCP-NEXT: vzeroupper ; AVX512-FCP-NEXT: retq ; @@ -176,13 +176,13 @@ define void @store_i64_stride6_vf2(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512DQ-NEXT: vinserti128 $1, (%rcx), %ymm1, %ymm1 ; AVX512DQ-NEXT: vinserti128 $1, (%rsi), %ymm0, %ymm0 ; AVX512DQ-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0 -; AVX512DQ-NEXT: vinserti32x4 $1, (%r9), %zmm2, %zmm1 -; AVX512DQ-NEXT: vpmovsxbq {{.*#+}} ymm2 = [5,7,9,11] -; AVX512DQ-NEXT: vpermi2q %zmm1, %zmm0, %zmm2 -; AVX512DQ-NEXT: vpmovsxbq {{.*#+}} zmm3 = [0,2,4,6,8,10,1,3] -; AVX512DQ-NEXT: vpermi2q %zmm1, %zmm0, %zmm3 -; AVX512DQ-NEXT: vmovdqa64 %zmm3, (%rax) -; AVX512DQ-NEXT: vmovdqa %ymm2, 64(%rax) +; AVX512DQ-NEXT: vinserti32x4 $1, (%r9), %zmm2, %zmm2 +; AVX512DQ-NEXT: vpmovsxbq {{.*#+}} ymm3 = [1,3,9,11] +; AVX512DQ-NEXT: vpermi2q %zmm2, %zmm1, %zmm3 +; AVX512DQ-NEXT: vpmovsxbq {{.*#+}} zmm1 = [0,2,4,6,8,10,1,3] +; AVX512DQ-NEXT: vpermi2q %zmm2, %zmm0, %zmm1 +; AVX512DQ-NEXT: vmovdqa %ymm3, 64(%rax) +; AVX512DQ-NEXT: vmovdqa64 %zmm1, (%rax) ; AVX512DQ-NEXT: vzeroupper ; AVX512DQ-NEXT: retq ; @@ -195,13 +195,13 @@ define void @store_i64_stride6_vf2(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512DQ-FCP-NEXT: vinserti128 $1, (%rcx), %ymm1, %ymm1 ; AVX512DQ-FCP-NEXT: vinserti128 $1, (%rsi), %ymm0, %ymm0 ; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0 -; AVX512DQ-FCP-NEXT: vinserti32x4 $1, (%r9), %zmm2, %zmm1 -; AVX512DQ-FCP-NEXT: vpmovsxbq {{.*#+}} ymm2 = [5,7,9,11] -; AVX512DQ-FCP-NEXT: vpermi2q %zmm1, %zmm0, %zmm2 -; AVX512DQ-FCP-NEXT: vpmovsxbq {{.*#+}} zmm3 = [0,2,4,6,8,10,1,3] -; AVX512DQ-FCP-NEXT: vpermi2q %zmm1, %zmm0, %zmm3 -; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm3, (%rax) -; AVX512DQ-FCP-NEXT: vmovdqa %ymm2, 64(%rax) +; AVX512DQ-FCP-NEXT: vinserti32x4 $1, (%r9), %zmm2, %zmm2 +; AVX512DQ-FCP-NEXT: vpmovsxbq {{.*#+}} ymm3 = [1,3,9,11] +; AVX512DQ-FCP-NEXT: vpermi2q %zmm2, %zmm1, %zmm3 +; AVX512DQ-FCP-NEXT: vpmovsxbq {{.*#+}} zmm1 = [0,2,4,6,8,10,1,3] +; AVX512DQ-FCP-NEXT: vpermi2q %zmm2, %zmm0, %zmm1 +; AVX512DQ-FCP-NEXT: vmovdqa %ymm3, 64(%rax) +; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm1, (%rax) ; AVX512DQ-FCP-NEXT: vzeroupper ; AVX512DQ-FCP-NEXT: retq ; @@ -214,13 +214,13 @@ define void @store_i64_stride6_vf2(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512BW-NEXT: vinserti128 $1, (%rcx), %ymm1, %ymm1 ; AVX512BW-NEXT: vinserti128 $1, (%rsi), %ymm0, %ymm0 ; AVX512BW-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0 -; AVX512BW-NEXT: vinserti32x4 $1, (%r9), %zmm2, %zmm1 -; AVX512BW-NEXT: vpmovsxbq {{.*#+}} ymm2 = [5,7,9,11] -; AVX512BW-NEXT: vpermi2q %zmm1, %zmm0, %zmm2 -; AVX512BW-NEXT: vpmovsxbq {{.*#+}} zmm3 = [0,2,4,6,8,10,1,3] -; AVX512BW-NEXT: vpermi2q %zmm1, %zmm0, %zmm3 -; AVX512BW-NEXT: vmovdqa64 %zmm3, (%rax) -; AVX512BW-NEXT: vmovdqa %ymm2, 64(%rax) +; AVX512BW-NEXT: vinserti32x4 $1, (%r9), %zmm2, %zmm2 +; AVX512BW-NEXT: vpmovsxbq {{.*#+}} ymm3 = [1,3,9,11] +; AVX512BW-NEXT: vpermi2q %zmm2, %zmm1, %zmm3 +; AVX512BW-NEXT: vpmovsxbq {{.*#+}} zmm1 = [0,2,4,6,8,10,1,3] +; AVX512BW-NEXT: vpermi2q %zmm2, %zmm0, %zmm1 +; AVX512BW-NEXT: vmovdqa %ymm3, 64(%rax) +; AVX512BW-NEXT: vmovdqa64 %zmm1, (%rax) ; AVX512BW-NEXT: vzeroupper ; AVX512BW-NEXT: retq ; @@ -233,13 +233,13 @@ define void @store_i64_stride6_vf2(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512BW-FCP-NEXT: vinserti128 $1, (%rcx), %ymm1, %ymm1 ; AVX512BW-FCP-NEXT: vinserti128 $1, (%rsi), %ymm0, %ymm0 ; AVX512BW-FCP-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0 -; AVX512BW-FCP-NEXT: vinserti32x4 $1, (%r9), %zmm2, %zmm1 -; AVX512BW-FCP-NEXT: vpmovsxbq {{.*#+}} ymm2 = [5,7,9,11] -; AVX512BW-FCP-NEXT: vpermi2q %zmm1, %zmm0, %zmm2 -; AVX512BW-FCP-NEXT: vpmovsxbq {{.*#+}} zmm3 = [0,2,4,6,8,10,1,3] -; AVX512BW-FCP-NEXT: vpermi2q %zmm1, %zmm0, %zmm3 -; AVX512BW-FCP-NEXT: vmovdqa64 %zmm3, (%rax) -; AVX512BW-FCP-NEXT: vmovdqa %ymm2, 64(%rax) +; AVX512BW-FCP-NEXT: vinserti32x4 $1, (%r9), %zmm2, %zmm2 +; AVX512BW-FCP-NEXT: vpmovsxbq {{.*#+}} ymm3 = [1,3,9,11] +; AVX512BW-FCP-NEXT: vpermi2q %zmm2, %zmm1, %zmm3 +; AVX512BW-FCP-NEXT: vpmovsxbq {{.*#+}} zmm1 = [0,2,4,6,8,10,1,3] +; AVX512BW-FCP-NEXT: vpermi2q %zmm2, %zmm0, %zmm1 +; AVX512BW-FCP-NEXT: vmovdqa %ymm3, 64(%rax) +; AVX512BW-FCP-NEXT: vmovdqa64 %zmm1, (%rax) ; AVX512BW-FCP-NEXT: vzeroupper ; AVX512BW-FCP-NEXT: retq ; @@ -252,13 +252,13 @@ define void @store_i64_stride6_vf2(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512DQ-BW-NEXT: vinserti128 $1, (%rcx), %ymm1, %ymm1 ; AVX512DQ-BW-NEXT: vinserti128 $1, (%rsi), %ymm0, %ymm0 ; AVX512DQ-BW-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0 -; AVX512DQ-BW-NEXT: vinserti32x4 $1, (%r9), %zmm2, %zmm1 -; AVX512DQ-BW-NEXT: vpmovsxbq {{.*#+}} ymm2 = [5,7,9,11] -; AVX512DQ-BW-NEXT: vpermi2q %zmm1, %zmm0, %zmm2 -; AVX512DQ-BW-NEXT: vpmovsxbq {{.*#+}} zmm3 = [0,2,4,6,8,10,1,3] -; AVX512DQ-BW-NEXT: vpermi2q %zmm1, %zmm0, %zmm3 -; AVX512DQ-BW-NEXT: vmovdqa64 %zmm3, (%rax) -; AVX512DQ-BW-NEXT: vmovdqa %ymm2, 64(%rax) +; AVX512DQ-BW-NEXT: vinserti32x4 $1, (%r9), %zmm2, %zmm2 +; AVX512DQ-BW-NEXT: vpmovsxbq {{.*#+}} ymm3 = [1,3,9,11] +; AVX512DQ-BW-NEXT: vpermi2q %zmm2, %zmm1, %zmm3 +; AVX512DQ-BW-NEXT: vpmovsxbq {{.*#+}} zmm1 = [0,2,4,6,8,10,1,3] +; AVX512DQ-BW-NEXT: vpermi2q %zmm2, %zmm0, %zmm1 +; AVX512DQ-BW-NEXT: vmovdqa %ymm3, 64(%rax) +; AVX512DQ-BW-NEXT: vmovdqa64 %zmm1, (%rax) ; AVX512DQ-BW-NEXT: vzeroupper ; AVX512DQ-BW-NEXT: retq ; @@ -271,13 +271,13 @@ define void @store_i64_stride6_vf2(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512DQ-BW-FCP-NEXT: vinserti128 $1, (%rcx), %ymm1, %ymm1 ; AVX512DQ-BW-FCP-NEXT: vinserti128 $1, (%rsi), %ymm0, %ymm0 ; AVX512DQ-BW-FCP-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0 -; AVX512DQ-BW-FCP-NEXT: vinserti32x4 $1, (%r9), %zmm2, %zmm1 -; AVX512DQ-BW-FCP-NEXT: vpmovsxbq {{.*#+}} ymm2 = [5,7,9,11] -; AVX512DQ-BW-FCP-NEXT: vpermi2q %zmm1, %zmm0, %zmm2 -; AVX512DQ-BW-FCP-NEXT: vpmovsxbq {{.*#+}} zmm3 = [0,2,4,6,8,10,1,3] -; AVX512DQ-BW-FCP-NEXT: vpermi2q %zmm1, %zmm0, %zmm3 -; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm3, (%rax) -; AVX512DQ-BW-FCP-NEXT: vmovdqa %ymm2, 64(%rax) +; AVX512DQ-BW-FCP-NEXT: vinserti32x4 $1, (%r9), %zmm2, %zmm2 +; AVX512DQ-BW-FCP-NEXT: vpmovsxbq {{.*#+}} ymm3 = [1,3,9,11] +; AVX512DQ-BW-FCP-NEXT: vpermi2q %zmm2, %zmm1, %zmm3 +; AVX512DQ-BW-FCP-NEXT: vpmovsxbq {{.*#+}} zmm1 = [0,2,4,6,8,10,1,3] +; AVX512DQ-BW-FCP-NEXT: vpermi2q %zmm2, %zmm0, %zmm1 +; AVX512DQ-BW-FCP-NEXT: vmovdqa %ymm3, 64(%rax) +; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm1, (%rax) ; AVX512DQ-BW-FCP-NEXT: vzeroupper ; AVX512DQ-BW-FCP-NEXT: retq %in.vec0 = load <2 x i64>, ptr %in.vecptr0, align 64 diff --git a/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-5.ll b/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-5.ll index b87abded1081..941b18db0931 100644 --- a/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-5.ll +++ b/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-5.ll @@ -5585,12 +5585,12 @@ define void @store_i8_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512BW-FCP-LABEL: store_i8_stride5_vf64: ; AVX512BW-FCP: # %bb.0: ; AVX512BW-FCP-NEXT: vmovdqa64 (%r8), %zmm0 -; AVX512BW-FCP-NEXT: vmovdqa64 32(%rdx), %ymm21 +; AVX512BW-FCP-NEXT: vmovdqa 32(%rdx), %ymm8 ; AVX512BW-FCP-NEXT: vmovdqa {{.*#+}} ymm6 = [128,128,12,13,128,128,128,128,14,128,128,128,14,15,128,128,128,128,16,128,128,128,16,17,128,128,128,128,18,128,128,128] -; AVX512BW-FCP-NEXT: vpshufb %ymm6, %ymm21, %ymm1 -; AVX512BW-FCP-NEXT: vmovdqa 32(%rcx), %ymm8 +; AVX512BW-FCP-NEXT: vpshufb %ymm6, %ymm8, %ymm1 +; AVX512BW-FCP-NEXT: vmovdqa64 32(%rcx), %ymm21 ; AVX512BW-FCP-NEXT: vmovdqa {{.*#+}} ymm7 = [128,128,128,128,13,128,128,128,128,14,128,128,128,128,15,128,128,128,128,16,128,128,128,128,17,128,128,128,128,18,128,128] -; AVX512BW-FCP-NEXT: vpshufb %ymm7, %ymm8, %ymm2 +; AVX512BW-FCP-NEXT: vpshufb %ymm7, %ymm21, %ymm2 ; AVX512BW-FCP-NEXT: vpor %ymm1, %ymm2, %ymm1 ; AVX512BW-FCP-NEXT: vmovdqa (%rcx), %xmm11 ; AVX512BW-FCP-NEXT: vmovdqa 32(%rcx), %xmm2 @@ -5598,77 +5598,76 @@ define void @store_i8_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512BW-FCP-NEXT: vpshufb %xmm14, %xmm2, %xmm3 ; AVX512BW-FCP-NEXT: vmovdqa (%rdx), %xmm12 ; AVX512BW-FCP-NEXT: vmovdqa 32(%rdx), %xmm4 -; AVX512BW-FCP-NEXT: vmovdqa {{.*#+}} xmm15 = [6,128,8,128,u,7,128,9,128,11,128,u,10,128,12,128] -; AVX512BW-FCP-NEXT: vpshufb %xmm15, %xmm4, %xmm5 +; AVX512BW-FCP-NEXT: vmovdqa64 {{.*#+}} xmm17 = [6,128,8,128,u,7,128,9,128,11,128,u,10,128,12,128] +; AVX512BW-FCP-NEXT: vpshufb %xmm17, %xmm4, %xmm5 ; AVX512BW-FCP-NEXT: vpor %xmm3, %xmm5, %xmm3 ; AVX512BW-FCP-NEXT: vpermq {{.*#+}} ymm3 = ymm3[0,0,1,1] ; AVX512BW-FCP-NEXT: vinserti64x4 $1, %ymm1, %zmm3, %zmm1 ; AVX512BW-FCP-NEXT: vmovdqa (%rdi), %xmm13 ; AVX512BW-FCP-NEXT: vmovdqa 32(%rdi), %xmm3 -; AVX512BW-FCP-NEXT: vmovdqa64 {{.*#+}} xmm17 = [8,128,u,7,128,9,128,u,128,u,10,128,12,128,u,11] -; AVX512BW-FCP-NEXT: vpshufb %xmm17, %xmm3, %xmm9 -; AVX512BW-FCP-NEXT: vmovdqa64 (%rsi), %xmm16 +; AVX512BW-FCP-NEXT: vmovdqa64 {{.*#+}} xmm19 = [8,128,u,7,128,9,128,u,128,u,10,128,12,128,u,11] +; AVX512BW-FCP-NEXT: vpshufb %xmm19, %xmm3, %xmm9 +; AVX512BW-FCP-NEXT: vmovdqa64 (%rsi), %xmm18 ; AVX512BW-FCP-NEXT: vmovdqa 32(%rsi), %xmm5 ; AVX512BW-FCP-NEXT: vmovdqa64 {{.*#+}} xmm20 = [128,8,u,128,7,128,9,u,11,u,128,10,128,12,u,128] ; AVX512BW-FCP-NEXT: vpshufb %xmm20, %xmm5, %xmm10 ; AVX512BW-FCP-NEXT: vpor %xmm9, %xmm10, %xmm9 -; AVX512BW-FCP-NEXT: vpermq {{.*#+}} ymm18 = ymm9[0,0,1,1] -; AVX512BW-FCP-NEXT: vmovdqa64 32(%rdi), %ymm19 +; AVX512BW-FCP-NEXT: vpermq {{.*#+}} ymm15 = ymm9[0,0,1,1] +; AVX512BW-FCP-NEXT: vmovdqa64 32(%rdi), %ymm16 ; AVX512BW-FCP-NEXT: vpmovsxbd {{.*#+}} ymm9 = [3,3,3,0,4,4,4,4] -; AVX512BW-FCP-NEXT: vpermd %ymm19, %ymm9, %ymm22 +; AVX512BW-FCP-NEXT: vpermd %ymm16, %ymm9, %ymm22 ; AVX512BW-FCP-NEXT: vmovdqa64 32(%rsi), %ymm23 ; AVX512BW-FCP-NEXT: vpbroadcastq {{.*#+}} ymm10 = [0,0,13,2,15,0,1,14,0,0,13,2,15,0,1,14,0,0,13,2,15,0,1,14,0,0,13,2,15,0,1,14] ; AVX512BW-FCP-NEXT: movl $138547332, %eax # imm = 0x8421084 ; AVX512BW-FCP-NEXT: kmovd %eax, %k1 ; AVX512BW-FCP-NEXT: vpshufb %ymm10, %ymm23, %ymm22 {%k1} -; AVX512BW-FCP-NEXT: vinserti64x4 $1, %ymm22, %zmm18, %zmm18 +; AVX512BW-FCP-NEXT: vinserti64x4 $1, %ymm22, %zmm15, %zmm15 ; AVX512BW-FCP-NEXT: movabsq $-8330787646191410408, %rax # imm = 0x8C6318C6318C6318 ; AVX512BW-FCP-NEXT: kmovq %rax, %k2 -; AVX512BW-FCP-NEXT: vmovdqu8 %zmm18, %zmm1 {%k2} -; AVX512BW-FCP-NEXT: vmovdqa64 32(%r8), %ymm22 -; AVX512BW-FCP-NEXT: vpmovsxbd {{.*#+}} zmm18 = [1,1,2,2,2,2,2,2,27,27,27,27,0,28,28,28] -; AVX512BW-FCP-NEXT: vpermi2d %zmm0, %zmm22, %zmm18 +; AVX512BW-FCP-NEXT: vmovdqu8 %zmm15, %zmm1 {%k2} +; AVX512BW-FCP-NEXT: vpmovsxbd {{.*#+}} zmm15 = [9,9,10,10,10,10,10,10,11,11,11,11,0,12,12,12] +; AVX512BW-FCP-NEXT: vpermd %zmm0, %zmm15, %zmm15 ; AVX512BW-FCP-NEXT: movabsq $4760450083537948804, %rax # imm = 0x4210842108421084 ; AVX512BW-FCP-NEXT: kmovq %rax, %k3 -; AVX512BW-FCP-NEXT: vmovdqu8 %zmm18, %zmm1 {%k3} -; AVX512BW-FCP-NEXT: vshufi64x2 {{.*#+}} zmm23 = zmm23[0,1,2,3],mem[4,5,6,7] -; AVX512BW-FCP-NEXT: vmovdqa64 {{.*#+}} zmm18 = [u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,19,128,21,128,128,20,128,22,128,24,128,128,23,128,25,128,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,27,128,128,26,128,28,128,128,128,128,29,128,31,128,128,30] -; AVX512BW-FCP-NEXT: vpshufb %zmm18, %zmm23, %zmm23 -; AVX512BW-FCP-NEXT: vshufi64x2 {{.*#+}} zmm24 = zmm19[0,1,2,3],mem[4,5,6,7] -; AVX512BW-FCP-NEXT: vmovdqa64 {{.*#+}} zmm19 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,21,128,128,20,128,22,128,24,128,128,23,128,25,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,26,128,28,128,128,27,128,29,128,31,128,128,30,128] -; AVX512BW-FCP-NEXT: vpshufb %zmm19, %zmm24, %zmm24 -; AVX512BW-FCP-NEXT: vporq %zmm23, %zmm24, %zmm23 -; AVX512BW-FCP-NEXT: vpermq {{.*#+}} zmm23 = zmm23[2,2,3,3,6,6,7,7] +; AVX512BW-FCP-NEXT: vmovdqu8 %zmm15, %zmm1 {%k3} +; AVX512BW-FCP-NEXT: vshufi64x2 {{.*#+}} zmm22 = zmm23[0,1,2,3],mem[4,5,6,7] +; AVX512BW-FCP-NEXT: vmovdqa64 {{.*#+}} zmm15 = [u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,19,128,21,128,128,20,128,22,128,24,128,128,23,128,25,128,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,27,128,128,26,128,28,128,128,128,128,29,128,31,128,128,30] +; AVX512BW-FCP-NEXT: vpshufb %zmm15, %zmm22, %zmm22 +; AVX512BW-FCP-NEXT: vshufi64x2 {{.*#+}} zmm23 = zmm16[0,1,2,3],mem[4,5,6,7] +; AVX512BW-FCP-NEXT: vmovdqa64 {{.*#+}} zmm16 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,21,128,128,20,128,22,128,24,128,128,23,128,25,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,26,128,28,128,128,27,128,29,128,31,128,128,30,128] +; AVX512BW-FCP-NEXT: vpshufb %zmm16, %zmm23, %zmm23 +; AVX512BW-FCP-NEXT: vporq %zmm22, %zmm23, %zmm22 +; AVX512BW-FCP-NEXT: vpermq {{.*#+}} zmm22 = zmm22[2,2,3,3,6,6,7,7] +; AVX512BW-FCP-NEXT: vshufi64x2 {{.*#+}} zmm8 = zmm8[0,1,2,3],mem[4,5,6,7] +; AVX512BW-FCP-NEXT: vmovdqa64 {{.*#+}} zmm23 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,18,19,128,21,128,21,20,128,22,128,24,128,22,23,128,25,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,25,128,27,128,128,26,128,28,128,30,128,128,29,128,31,128] +; AVX512BW-FCP-NEXT: vpshufb %zmm23, %zmm8, %zmm8 ; AVX512BW-FCP-NEXT: vshufi64x2 {{.*#+}} zmm21 = zmm21[0,1,2,3],mem[4,5,6,7] -; AVX512BW-FCP-NEXT: vmovdqa64 {{.*#+}} zmm24 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,18,19,128,21,128,21,20,128,22,128,24,128,22,23,128,25,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,25,128,27,128,128,26,128,28,128,30,128,128,29,128,31,128] +; AVX512BW-FCP-NEXT: vmovdqa64 {{.*#+}} zmm24 = [u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,128,128,19,128,21,128,128,20,128,22,128,24,128,128,23,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,27,128,128,26,128,28,128,30,128,128,29,128,31,128,128] ; AVX512BW-FCP-NEXT: vpshufb %zmm24, %zmm21, %zmm21 -; AVX512BW-FCP-NEXT: vshufi64x2 {{.*#+}} zmm8 = zmm8[0,1,2,3],mem[4,5,6,7] -; AVX512BW-FCP-NEXT: vmovdqa64 {{.*#+}} zmm25 = [u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,128,128,19,128,21,128,128,20,128,22,128,24,128,128,23,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,27,128,128,26,128,28,128,30,128,128,29,128,31,128,128] -; AVX512BW-FCP-NEXT: vpshufb %zmm25, %zmm8, %zmm8 -; AVX512BW-FCP-NEXT: vporq %zmm21, %zmm8, %zmm8 +; AVX512BW-FCP-NEXT: vporq %zmm8, %zmm21, %zmm8 ; AVX512BW-FCP-NEXT: vpermq {{.*#+}} zmm8 = zmm8[2,2,3,3,6,6,7,7] ; AVX512BW-FCP-NEXT: movabsq $1785168781326730801, %rax # imm = 0x18C6318C6318C631 ; AVX512BW-FCP-NEXT: kmovq %rax, %k3 -; AVX512BW-FCP-NEXT: vmovdqu8 %zmm23, %zmm8 {%k3} -; AVX512BW-FCP-NEXT: vpmovsxbd {{.*#+}} zmm21 = [4,6,5,5,5,5,4,6,30,30,30,30,31,31,31,31] -; AVX512BW-FCP-NEXT: vpermi2d %zmm0, %zmm22, %zmm21 +; AVX512BW-FCP-NEXT: vmovdqu8 %zmm22, %zmm8 {%k3} +; AVX512BW-FCP-NEXT: vpmovsxbd {{.*#+}} zmm21 = [12,14,13,13,13,13,12,14,14,14,14,14,15,15,15,15] +; AVX512BW-FCP-NEXT: vpermd %zmm0, %zmm21, %zmm21 ; AVX512BW-FCP-NEXT: movabsq $-8925843906633654008, %rax # imm = 0x8421084210842108 ; AVX512BW-FCP-NEXT: kmovq %rax, %k4 ; AVX512BW-FCP-NEXT: vmovdqu8 %zmm21, %zmm8 {%k4} ; AVX512BW-FCP-NEXT: vpshufb %xmm14, %xmm11, %xmm14 -; AVX512BW-FCP-NEXT: vpshufb %xmm15, %xmm12, %xmm15 -; AVX512BW-FCP-NEXT: vpor %xmm14, %xmm15, %xmm14 +; AVX512BW-FCP-NEXT: vpshufb %xmm17, %xmm12, %xmm17 +; AVX512BW-FCP-NEXT: vporq %xmm14, %xmm17, %xmm14 ; AVX512BW-FCP-NEXT: vpunpcklbw {{.*#+}} xmm11 = xmm11[0],xmm12[0],xmm11[1],xmm12[1],xmm11[2],xmm12[2],xmm11[3],xmm12[3],xmm11[4],xmm12[4],xmm11[5],xmm12[5],xmm11[6],xmm12[6],xmm11[7],xmm12[7] ; AVX512BW-FCP-NEXT: vmovdqa {{.*#+}} xmm12 = [2,u,1,0,5,4,u,3,u,7,6,11,10,u,9,8] ; AVX512BW-FCP-NEXT: vpshufb %xmm12, %xmm11, %xmm11 ; AVX512BW-FCP-NEXT: vinserti32x4 $2, %xmm14, %zmm11, %zmm11 ; AVX512BW-FCP-NEXT: vpermq {{.*#+}} zmm14 = zmm11[0,0,1,1,4,4,5,5] -; AVX512BW-FCP-NEXT: vpshufb %xmm17, %xmm13, %xmm11 -; AVX512BW-FCP-NEXT: vpshufb %xmm20, %xmm16, %xmm15 -; AVX512BW-FCP-NEXT: vpor %xmm11, %xmm15, %xmm11 -; AVX512BW-FCP-NEXT: vpunpcklbw {{.*#+}} xmm13 = xmm13[0],xmm16[0],xmm13[1],xmm16[1],xmm13[2],xmm16[2],xmm13[3],xmm16[3],xmm13[4],xmm16[4],xmm13[5],xmm16[5],xmm13[6],xmm16[6],xmm13[7],xmm16[7] -; AVX512BW-FCP-NEXT: vmovdqa {{.*#+}} xmm15 = [0,1,4,5,u,2,3,6,7,10,11,u,8,9,12,13] -; AVX512BW-FCP-NEXT: vpshufb %xmm15, %xmm13, %xmm13 +; AVX512BW-FCP-NEXT: vpshufb %xmm19, %xmm13, %xmm11 +; AVX512BW-FCP-NEXT: vpshufb %xmm20, %xmm18, %xmm17 +; AVX512BW-FCP-NEXT: vporq %xmm11, %xmm17, %xmm11 +; AVX512BW-FCP-NEXT: vpunpcklbw {{.*#+}} xmm13 = xmm13[0],xmm18[0],xmm13[1],xmm18[1],xmm13[2],xmm18[2],xmm13[3],xmm18[3],xmm13[4],xmm18[4],xmm13[5],xmm18[5],xmm13[6],xmm18[6],xmm13[7],xmm18[7] +; AVX512BW-FCP-NEXT: vmovdqa64 {{.*#+}} xmm17 = [0,1,4,5,u,2,3,6,7,10,11,u,8,9,12,13] +; AVX512BW-FCP-NEXT: vpshufb %xmm17, %xmm13, %xmm13 ; AVX512BW-FCP-NEXT: vinserti32x4 $2, %xmm11, %zmm13, %zmm11 ; AVX512BW-FCP-NEXT: vpermq {{.*#+}} zmm11 = zmm11[0,0,1,1,4,4,5,5] ; AVX512BW-FCP-NEXT: movabsq $-4165393823095705204, %rax # imm = 0xC6318C6318C6318C @@ -5682,23 +5681,23 @@ define void @store_i8_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512BW-FCP-NEXT: vmovdqu8 %zmm14, %zmm11 {%k4} ; AVX512BW-FCP-NEXT: vmovdqa (%rdx), %ymm14 ; AVX512BW-FCP-NEXT: vpshufb %ymm6, %ymm14, %ymm6 -; AVX512BW-FCP-NEXT: vmovdqa64 (%rcx), %ymm16 -; AVX512BW-FCP-NEXT: vpshufb %ymm7, %ymm16, %ymm7 +; AVX512BW-FCP-NEXT: vmovdqa64 (%rcx), %ymm18 +; AVX512BW-FCP-NEXT: vpshufb %ymm7, %ymm18, %ymm7 ; AVX512BW-FCP-NEXT: vpor %ymm6, %ymm7, %ymm6 -; AVX512BW-FCP-NEXT: vpshufb %ymm24, %ymm14, %ymm7 -; AVX512BW-FCP-NEXT: vpshufb %ymm25, %ymm16, %ymm17 -; AVX512BW-FCP-NEXT: vporq %ymm7, %ymm17, %ymm7 +; AVX512BW-FCP-NEXT: vpshufb %ymm23, %ymm14, %ymm7 +; AVX512BW-FCP-NEXT: vpshufb %ymm24, %ymm18, %ymm19 +; AVX512BW-FCP-NEXT: vporq %ymm7, %ymm19, %ymm7 ; AVX512BW-FCP-NEXT: vpermq {{.*#+}} ymm7 = ymm7[2,2,3,3] ; AVX512BW-FCP-NEXT: vinserti64x4 $1, %ymm7, %zmm6, %zmm6 ; AVX512BW-FCP-NEXT: vmovdqa (%rsi), %ymm7 -; AVX512BW-FCP-NEXT: vpshufb %ymm18, %ymm7, %ymm17 -; AVX512BW-FCP-NEXT: vmovdqa64 (%rdi), %ymm18 -; AVX512BW-FCP-NEXT: vpshufb %ymm19, %ymm18, %ymm19 -; AVX512BW-FCP-NEXT: vporq %ymm17, %ymm19, %ymm17 -; AVX512BW-FCP-NEXT: vpermq {{.*#+}} ymm17 = ymm17[2,2,3,3] -; AVX512BW-FCP-NEXT: vpermd %ymm18, %ymm9, %ymm9 +; AVX512BW-FCP-NEXT: vpshufb %ymm15, %ymm7, %ymm15 +; AVX512BW-FCP-NEXT: vmovdqa64 (%rdi), %ymm19 +; AVX512BW-FCP-NEXT: vpshufb %ymm16, %ymm19, %ymm16 +; AVX512BW-FCP-NEXT: vporq %ymm15, %ymm16, %ymm15 +; AVX512BW-FCP-NEXT: vpermq {{.*#+}} ymm15 = ymm15[2,2,3,3] +; AVX512BW-FCP-NEXT: vpermd %ymm19, %ymm9, %ymm9 ; AVX512BW-FCP-NEXT: vpshufb %ymm10, %ymm7, %ymm9 {%k1} -; AVX512BW-FCP-NEXT: vinserti64x4 $1, %ymm17, %zmm9, %zmm9 +; AVX512BW-FCP-NEXT: vinserti64x4 $1, %ymm15, %zmm9, %zmm9 ; AVX512BW-FCP-NEXT: vmovdqu8 %zmm6, %zmm9 {%k2} ; AVX512BW-FCP-NEXT: vpmovsxbd {{.*#+}} zmm6 = [3,3,3,3,0,4,4,4,12,14,13,13,13,13,12,14] ; AVX512BW-FCP-NEXT: vpermd %zmm13, %zmm6, %zmm6 @@ -5707,15 +5706,15 @@ define void @store_i8_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512BW-FCP-NEXT: vmovdqu8 %zmm6, %zmm9 {%k1} ; AVX512BW-FCP-NEXT: vpunpcklbw {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3],xmm2[4],xmm4[4],xmm2[5],xmm4[5],xmm2[6],xmm4[6],xmm2[7],xmm4[7] ; AVX512BW-FCP-NEXT: vpshufb %xmm12, %xmm2, %xmm2 -; AVX512BW-FCP-NEXT: vpshufb {{.*#+}} ymm4 = ymm16[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,25],zero,ymm16[27],zero,zero,ymm16[26],zero,ymm16[28],zero,ymm16[30],zero,zero,ymm16[29],zero,ymm16[31],zero +; AVX512BW-FCP-NEXT: vpshufb {{.*#+}} ymm4 = ymm18[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,25],zero,ymm18[27],zero,zero,ymm18[26],zero,ymm18[28],zero,ymm18[30],zero,zero,ymm18[29],zero,ymm18[31],zero ; AVX512BW-FCP-NEXT: vpshufb {{.*#+}} ymm6 = ymm14[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm14[27],zero,zero,ymm14[26],zero,ymm14[28],zero,ymm14[30],zero,zero,ymm14[29],zero,ymm14[31],zero,zero ; AVX512BW-FCP-NEXT: vpor %ymm4, %ymm6, %ymm4 ; AVX512BW-FCP-NEXT: vpmovsxbq {{.*#+}} zmm6 = [2,2,3,3,8,8,9,9] ; AVX512BW-FCP-NEXT: vpermt2q %zmm2, %zmm6, %zmm4 ; AVX512BW-FCP-NEXT: vpunpcklbw {{.*#+}} xmm2 = xmm3[0],xmm5[0],xmm3[1],xmm5[1],xmm3[2],xmm5[2],xmm3[3],xmm5[3],xmm3[4],xmm5[4],xmm3[5],xmm5[5],xmm3[6],xmm5[6],xmm3[7],xmm5[7] -; AVX512BW-FCP-NEXT: vpshufb %xmm15, %xmm2, %xmm2 +; AVX512BW-FCP-NEXT: vpshufb %xmm17, %xmm2, %xmm2 ; AVX512BW-FCP-NEXT: vpshufb {{.*#+}} ymm3 = ymm7[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,27],zero,zero,ymm7[26],zero,ymm7[28],zero,zero,zero,zero,ymm7[29],zero,ymm7[31],zero,zero,ymm7[30] -; AVX512BW-FCP-NEXT: vpshufb {{.*#+}} ymm5 = ymm18[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,ymm18[26],zero,ymm18[28],zero,zero,ymm18[27],zero,ymm18[29],zero,ymm18[31],zero,zero,ymm18[30],zero +; AVX512BW-FCP-NEXT: vpshufb {{.*#+}} ymm5 = ymm19[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,ymm19[26],zero,ymm19[28],zero,zero,ymm19[27],zero,ymm19[29],zero,ymm19[31],zero,zero,ymm19[30],zero ; AVX512BW-FCP-NEXT: vpor %ymm3, %ymm5, %ymm3 ; AVX512BW-FCP-NEXT: vpermt2q %zmm2, %zmm6, %zmm3 ; AVX512BW-FCP-NEXT: vmovdqu8 %zmm4, %zmm3 {%k3} @@ -5909,12 +5908,12 @@ define void @store_i8_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512DQ-BW-FCP-LABEL: store_i8_stride5_vf64: ; AVX512DQ-BW-FCP: # %bb.0: ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 (%r8), %zmm0 -; AVX512DQ-BW-FCP-NEXT: vmovdqa64 32(%rdx), %ymm21 +; AVX512DQ-BW-FCP-NEXT: vmovdqa 32(%rdx), %ymm8 ; AVX512DQ-BW-FCP-NEXT: vmovdqa {{.*#+}} ymm6 = [128,128,12,13,128,128,128,128,14,128,128,128,14,15,128,128,128,128,16,128,128,128,16,17,128,128,128,128,18,128,128,128] -; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm6, %ymm21, %ymm1 -; AVX512DQ-BW-FCP-NEXT: vmovdqa 32(%rcx), %ymm8 +; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm6, %ymm8, %ymm1 +; AVX512DQ-BW-FCP-NEXT: vmovdqa64 32(%rcx), %ymm21 ; AVX512DQ-BW-FCP-NEXT: vmovdqa {{.*#+}} ymm7 = [128,128,128,128,13,128,128,128,128,14,128,128,128,128,15,128,128,128,128,16,128,128,128,128,17,128,128,128,128,18,128,128] -; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm7, %ymm8, %ymm2 +; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm7, %ymm21, %ymm2 ; AVX512DQ-BW-FCP-NEXT: vpor %ymm1, %ymm2, %ymm1 ; AVX512DQ-BW-FCP-NEXT: vmovdqa (%rcx), %xmm11 ; AVX512DQ-BW-FCP-NEXT: vmovdqa 32(%rcx), %xmm2 @@ -5922,77 +5921,76 @@ define void @store_i8_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm14, %xmm2, %xmm3 ; AVX512DQ-BW-FCP-NEXT: vmovdqa (%rdx), %xmm12 ; AVX512DQ-BW-FCP-NEXT: vmovdqa 32(%rdx), %xmm4 -; AVX512DQ-BW-FCP-NEXT: vmovdqa {{.*#+}} xmm15 = [6,128,8,128,u,7,128,9,128,11,128,u,10,128,12,128] -; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm15, %xmm4, %xmm5 +; AVX512DQ-BW-FCP-NEXT: vmovdqa64 {{.*#+}} xmm17 = [6,128,8,128,u,7,128,9,128,11,128,u,10,128,12,128] +; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm17, %xmm4, %xmm5 ; AVX512DQ-BW-FCP-NEXT: vpor %xmm3, %xmm5, %xmm3 ; AVX512DQ-BW-FCP-NEXT: vpermq {{.*#+}} ymm3 = ymm3[0,0,1,1] ; AVX512DQ-BW-FCP-NEXT: vinserti64x4 $1, %ymm1, %zmm3, %zmm1 ; AVX512DQ-BW-FCP-NEXT: vmovdqa (%rdi), %xmm13 ; AVX512DQ-BW-FCP-NEXT: vmovdqa 32(%rdi), %xmm3 -; AVX512DQ-BW-FCP-NEXT: vmovdqa64 {{.*#+}} xmm17 = [8,128,u,7,128,9,128,u,128,u,10,128,12,128,u,11] -; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm17, %xmm3, %xmm9 -; AVX512DQ-BW-FCP-NEXT: vmovdqa64 (%rsi), %xmm16 +; AVX512DQ-BW-FCP-NEXT: vmovdqa64 {{.*#+}} xmm19 = [8,128,u,7,128,9,128,u,128,u,10,128,12,128,u,11] +; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm19, %xmm3, %xmm9 +; AVX512DQ-BW-FCP-NEXT: vmovdqa64 (%rsi), %xmm18 ; AVX512DQ-BW-FCP-NEXT: vmovdqa 32(%rsi), %xmm5 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 {{.*#+}} xmm20 = [128,8,u,128,7,128,9,u,11,u,128,10,128,12,u,128] ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm20, %xmm5, %xmm10 ; AVX512DQ-BW-FCP-NEXT: vpor %xmm9, %xmm10, %xmm9 -; AVX512DQ-BW-FCP-NEXT: vpermq {{.*#+}} ymm18 = ymm9[0,0,1,1] -; AVX512DQ-BW-FCP-NEXT: vmovdqa64 32(%rdi), %ymm19 +; AVX512DQ-BW-FCP-NEXT: vpermq {{.*#+}} ymm15 = ymm9[0,0,1,1] +; AVX512DQ-BW-FCP-NEXT: vmovdqa64 32(%rdi), %ymm16 ; AVX512DQ-BW-FCP-NEXT: vpmovsxbd {{.*#+}} ymm9 = [3,3,3,0,4,4,4,4] -; AVX512DQ-BW-FCP-NEXT: vpermd %ymm19, %ymm9, %ymm22 +; AVX512DQ-BW-FCP-NEXT: vpermd %ymm16, %ymm9, %ymm22 ; AVX512DQ-BW-FCP-NEXT: vmovdqa64 32(%rsi), %ymm23 ; AVX512DQ-BW-FCP-NEXT: vpbroadcastq {{.*#+}} ymm10 = [0,0,13,2,15,0,1,14,0,0,13,2,15,0,1,14,0,0,13,2,15,0,1,14,0,0,13,2,15,0,1,14] ; AVX512DQ-BW-FCP-NEXT: movl $138547332, %eax # imm = 0x8421084 ; AVX512DQ-BW-FCP-NEXT: kmovd %eax, %k1 ; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm10, %ymm23, %ymm22 {%k1} -; AVX512DQ-BW-FCP-NEXT: vinserti64x4 $1, %ymm22, %zmm18, %zmm18 +; AVX512DQ-BW-FCP-NEXT: vinserti64x4 $1, %ymm22, %zmm15, %zmm15 ; AVX512DQ-BW-FCP-NEXT: movabsq $-8330787646191410408, %rax # imm = 0x8C6318C6318C6318 ; AVX512DQ-BW-FCP-NEXT: kmovq %rax, %k2 -; AVX512DQ-BW-FCP-NEXT: vmovdqu8 %zmm18, %zmm1 {%k2} -; AVX512DQ-BW-FCP-NEXT: vmovdqa64 32(%r8), %ymm22 -; AVX512DQ-BW-FCP-NEXT: vpmovsxbd {{.*#+}} zmm18 = [1,1,2,2,2,2,2,2,27,27,27,27,0,28,28,28] -; AVX512DQ-BW-FCP-NEXT: vpermi2d %zmm0, %zmm22, %zmm18 +; AVX512DQ-BW-FCP-NEXT: vmovdqu8 %zmm15, %zmm1 {%k2} +; AVX512DQ-BW-FCP-NEXT: vpmovsxbd {{.*#+}} zmm15 = [9,9,10,10,10,10,10,10,11,11,11,11,0,12,12,12] +; AVX512DQ-BW-FCP-NEXT: vpermd %zmm0, %zmm15, %zmm15 ; AVX512DQ-BW-FCP-NEXT: movabsq $4760450083537948804, %rax # imm = 0x4210842108421084 ; AVX512DQ-BW-FCP-NEXT: kmovq %rax, %k3 -; AVX512DQ-BW-FCP-NEXT: vmovdqu8 %zmm18, %zmm1 {%k3} -; AVX512DQ-BW-FCP-NEXT: vshufi64x2 {{.*#+}} zmm23 = zmm23[0,1,2,3],mem[4,5,6,7] -; AVX512DQ-BW-FCP-NEXT: vmovdqa64 {{.*#+}} zmm18 = [u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,19,128,21,128,128,20,128,22,128,24,128,128,23,128,25,128,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,27,128,128,26,128,28,128,128,128,128,29,128,31,128,128,30] -; AVX512DQ-BW-FCP-NEXT: vpshufb %zmm18, %zmm23, %zmm23 -; AVX512DQ-BW-FCP-NEXT: vshufi64x2 {{.*#+}} zmm24 = zmm19[0,1,2,3],mem[4,5,6,7] -; AVX512DQ-BW-FCP-NEXT: vmovdqa64 {{.*#+}} zmm19 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,21,128,128,20,128,22,128,24,128,128,23,128,25,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,26,128,28,128,128,27,128,29,128,31,128,128,30,128] -; AVX512DQ-BW-FCP-NEXT: vpshufb %zmm19, %zmm24, %zmm24 -; AVX512DQ-BW-FCP-NEXT: vporq %zmm23, %zmm24, %zmm23 -; AVX512DQ-BW-FCP-NEXT: vpermq {{.*#+}} zmm23 = zmm23[2,2,3,3,6,6,7,7] +; AVX512DQ-BW-FCP-NEXT: vmovdqu8 %zmm15, %zmm1 {%k3} +; AVX512DQ-BW-FCP-NEXT: vshufi64x2 {{.*#+}} zmm22 = zmm23[0,1,2,3],mem[4,5,6,7] +; AVX512DQ-BW-FCP-NEXT: vmovdqa64 {{.*#+}} zmm15 = [u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,19,128,21,128,128,20,128,22,128,24,128,128,23,128,25,128,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,27,128,128,26,128,28,128,128,128,128,29,128,31,128,128,30] +; AVX512DQ-BW-FCP-NEXT: vpshufb %zmm15, %zmm22, %zmm22 +; AVX512DQ-BW-FCP-NEXT: vshufi64x2 {{.*#+}} zmm23 = zmm16[0,1,2,3],mem[4,5,6,7] +; AVX512DQ-BW-FCP-NEXT: vmovdqa64 {{.*#+}} zmm16 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,21,128,128,20,128,22,128,24,128,128,23,128,25,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,26,128,28,128,128,27,128,29,128,31,128,128,30,128] +; AVX512DQ-BW-FCP-NEXT: vpshufb %zmm16, %zmm23, %zmm23 +; AVX512DQ-BW-FCP-NEXT: vporq %zmm22, %zmm23, %zmm22 +; AVX512DQ-BW-FCP-NEXT: vpermq {{.*#+}} zmm22 = zmm22[2,2,3,3,6,6,7,7] +; AVX512DQ-BW-FCP-NEXT: vshufi64x2 {{.*#+}} zmm8 = zmm8[0,1,2,3],mem[4,5,6,7] +; AVX512DQ-BW-FCP-NEXT: vmovdqa64 {{.*#+}} zmm23 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,18,19,128,21,128,21,20,128,22,128,24,128,22,23,128,25,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,25,128,27,128,128,26,128,28,128,30,128,128,29,128,31,128] +; AVX512DQ-BW-FCP-NEXT: vpshufb %zmm23, %zmm8, %zmm8 ; AVX512DQ-BW-FCP-NEXT: vshufi64x2 {{.*#+}} zmm21 = zmm21[0,1,2,3],mem[4,5,6,7] -; AVX512DQ-BW-FCP-NEXT: vmovdqa64 {{.*#+}} zmm24 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,18,19,128,21,128,21,20,128,22,128,24,128,22,23,128,25,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,25,128,27,128,128,26,128,28,128,30,128,128,29,128,31,128] +; AVX512DQ-BW-FCP-NEXT: vmovdqa64 {{.*#+}} zmm24 = [u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,128,128,19,128,21,128,128,20,128,22,128,24,128,128,23,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,27,128,128,26,128,28,128,30,128,128,29,128,31,128,128] ; AVX512DQ-BW-FCP-NEXT: vpshufb %zmm24, %zmm21, %zmm21 -; AVX512DQ-BW-FCP-NEXT: vshufi64x2 {{.*#+}} zmm8 = zmm8[0,1,2,3],mem[4,5,6,7] -; AVX512DQ-BW-FCP-NEXT: vmovdqa64 {{.*#+}} zmm25 = [u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,128,128,19,128,21,128,128,20,128,22,128,24,128,128,23,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,27,128,128,26,128,28,128,30,128,128,29,128,31,128,128] -; AVX512DQ-BW-FCP-NEXT: vpshufb %zmm25, %zmm8, %zmm8 -; AVX512DQ-BW-FCP-NEXT: vporq %zmm21, %zmm8, %zmm8 +; AVX512DQ-BW-FCP-NEXT: vporq %zmm8, %zmm21, %zmm8 ; AVX512DQ-BW-FCP-NEXT: vpermq {{.*#+}} zmm8 = zmm8[2,2,3,3,6,6,7,7] ; AVX512DQ-BW-FCP-NEXT: movabsq $1785168781326730801, %rax # imm = 0x18C6318C6318C631 ; AVX512DQ-BW-FCP-NEXT: kmovq %rax, %k3 -; AVX512DQ-BW-FCP-NEXT: vmovdqu8 %zmm23, %zmm8 {%k3} -; AVX512DQ-BW-FCP-NEXT: vpmovsxbd {{.*#+}} zmm21 = [4,6,5,5,5,5,4,6,30,30,30,30,31,31,31,31] -; AVX512DQ-BW-FCP-NEXT: vpermi2d %zmm0, %zmm22, %zmm21 +; AVX512DQ-BW-FCP-NEXT: vmovdqu8 %zmm22, %zmm8 {%k3} +; AVX512DQ-BW-FCP-NEXT: vpmovsxbd {{.*#+}} zmm21 = [12,14,13,13,13,13,12,14,14,14,14,14,15,15,15,15] +; AVX512DQ-BW-FCP-NEXT: vpermd %zmm0, %zmm21, %zmm21 ; AVX512DQ-BW-FCP-NEXT: movabsq $-8925843906633654008, %rax # imm = 0x8421084210842108 ; AVX512DQ-BW-FCP-NEXT: kmovq %rax, %k4 ; AVX512DQ-BW-FCP-NEXT: vmovdqu8 %zmm21, %zmm8 {%k4} ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm14, %xmm11, %xmm14 -; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm15, %xmm12, %xmm15 -; AVX512DQ-BW-FCP-NEXT: vpor %xmm14, %xmm15, %xmm14 +; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm17, %xmm12, %xmm17 +; AVX512DQ-BW-FCP-NEXT: vporq %xmm14, %xmm17, %xmm14 ; AVX512DQ-BW-FCP-NEXT: vpunpcklbw {{.*#+}} xmm11 = xmm11[0],xmm12[0],xmm11[1],xmm12[1],xmm11[2],xmm12[2],xmm11[3],xmm12[3],xmm11[4],xmm12[4],xmm11[5],xmm12[5],xmm11[6],xmm12[6],xmm11[7],xmm12[7] ; AVX512DQ-BW-FCP-NEXT: vmovdqa {{.*#+}} xmm12 = [2,u,1,0,5,4,u,3,u,7,6,11,10,u,9,8] ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm12, %xmm11, %xmm11 ; AVX512DQ-BW-FCP-NEXT: vinserti32x4 $2, %xmm14, %zmm11, %zmm11 ; AVX512DQ-BW-FCP-NEXT: vpermq {{.*#+}} zmm14 = zmm11[0,0,1,1,4,4,5,5] -; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm17, %xmm13, %xmm11 -; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm20, %xmm16, %xmm15 -; AVX512DQ-BW-FCP-NEXT: vpor %xmm11, %xmm15, %xmm11 -; AVX512DQ-BW-FCP-NEXT: vpunpcklbw {{.*#+}} xmm13 = xmm13[0],xmm16[0],xmm13[1],xmm16[1],xmm13[2],xmm16[2],xmm13[3],xmm16[3],xmm13[4],xmm16[4],xmm13[5],xmm16[5],xmm13[6],xmm16[6],xmm13[7],xmm16[7] -; AVX512DQ-BW-FCP-NEXT: vmovdqa {{.*#+}} xmm15 = [0,1,4,5,u,2,3,6,7,10,11,u,8,9,12,13] -; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm15, %xmm13, %xmm13 +; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm19, %xmm13, %xmm11 +; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm20, %xmm18, %xmm17 +; AVX512DQ-BW-FCP-NEXT: vporq %xmm11, %xmm17, %xmm11 +; AVX512DQ-BW-FCP-NEXT: vpunpcklbw {{.*#+}} xmm13 = xmm13[0],xmm18[0],xmm13[1],xmm18[1],xmm13[2],xmm18[2],xmm13[3],xmm18[3],xmm13[4],xmm18[4],xmm13[5],xmm18[5],xmm13[6],xmm18[6],xmm13[7],xmm18[7] +; AVX512DQ-BW-FCP-NEXT: vmovdqa64 {{.*#+}} xmm17 = [0,1,4,5,u,2,3,6,7,10,11,u,8,9,12,13] +; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm17, %xmm13, %xmm13 ; AVX512DQ-BW-FCP-NEXT: vinserti32x4 $2, %xmm11, %zmm13, %zmm11 ; AVX512DQ-BW-FCP-NEXT: vpermq {{.*#+}} zmm11 = zmm11[0,0,1,1,4,4,5,5] ; AVX512DQ-BW-FCP-NEXT: movabsq $-4165393823095705204, %rax # imm = 0xC6318C6318C6318C @@ -6006,23 +6004,23 @@ define void @store_i8_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512DQ-BW-FCP-NEXT: vmovdqu8 %zmm14, %zmm11 {%k4} ; AVX512DQ-BW-FCP-NEXT: vmovdqa (%rdx), %ymm14 ; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm6, %ymm14, %ymm6 -; AVX512DQ-BW-FCP-NEXT: vmovdqa64 (%rcx), %ymm16 -; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm7, %ymm16, %ymm7 +; AVX512DQ-BW-FCP-NEXT: vmovdqa64 (%rcx), %ymm18 +; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm7, %ymm18, %ymm7 ; AVX512DQ-BW-FCP-NEXT: vpor %ymm6, %ymm7, %ymm6 -; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm24, %ymm14, %ymm7 -; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm25, %ymm16, %ymm17 -; AVX512DQ-BW-FCP-NEXT: vporq %ymm7, %ymm17, %ymm7 +; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm23, %ymm14, %ymm7 +; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm24, %ymm18, %ymm19 +; AVX512DQ-BW-FCP-NEXT: vporq %ymm7, %ymm19, %ymm7 ; AVX512DQ-BW-FCP-NEXT: vpermq {{.*#+}} ymm7 = ymm7[2,2,3,3] ; AVX512DQ-BW-FCP-NEXT: vinserti64x4 $1, %ymm7, %zmm6, %zmm6 ; AVX512DQ-BW-FCP-NEXT: vmovdqa (%rsi), %ymm7 -; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm18, %ymm7, %ymm17 -; AVX512DQ-BW-FCP-NEXT: vmovdqa64 (%rdi), %ymm18 -; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm19, %ymm18, %ymm19 -; AVX512DQ-BW-FCP-NEXT: vporq %ymm17, %ymm19, %ymm17 -; AVX512DQ-BW-FCP-NEXT: vpermq {{.*#+}} ymm17 = ymm17[2,2,3,3] -; AVX512DQ-BW-FCP-NEXT: vpermd %ymm18, %ymm9, %ymm9 +; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm15, %ymm7, %ymm15 +; AVX512DQ-BW-FCP-NEXT: vmovdqa64 (%rdi), %ymm19 +; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm16, %ymm19, %ymm16 +; AVX512DQ-BW-FCP-NEXT: vporq %ymm15, %ymm16, %ymm15 +; AVX512DQ-BW-FCP-NEXT: vpermq {{.*#+}} ymm15 = ymm15[2,2,3,3] +; AVX512DQ-BW-FCP-NEXT: vpermd %ymm19, %ymm9, %ymm9 ; AVX512DQ-BW-FCP-NEXT: vpshufb %ymm10, %ymm7, %ymm9 {%k1} -; AVX512DQ-BW-FCP-NEXT: vinserti64x4 $1, %ymm17, %zmm9, %zmm9 +; AVX512DQ-BW-FCP-NEXT: vinserti64x4 $1, %ymm15, %zmm9, %zmm9 ; AVX512DQ-BW-FCP-NEXT: vmovdqu8 %zmm6, %zmm9 {%k2} ; AVX512DQ-BW-FCP-NEXT: vpmovsxbd {{.*#+}} zmm6 = [3,3,3,3,0,4,4,4,12,14,13,13,13,13,12,14] ; AVX512DQ-BW-FCP-NEXT: vpermd %zmm13, %zmm6, %zmm6 @@ -6031,15 +6029,15 @@ define void @store_i8_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512DQ-BW-FCP-NEXT: vmovdqu8 %zmm6, %zmm9 {%k1} ; AVX512DQ-BW-FCP-NEXT: vpunpcklbw {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3],xmm2[4],xmm4[4],xmm2[5],xmm4[5],xmm2[6],xmm4[6],xmm2[7],xmm4[7] ; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm12, %xmm2, %xmm2 -; AVX512DQ-BW-FCP-NEXT: vpshufb {{.*#+}} ymm4 = ymm16[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,25],zero,ymm16[27],zero,zero,ymm16[26],zero,ymm16[28],zero,ymm16[30],zero,zero,ymm16[29],zero,ymm16[31],zero +; AVX512DQ-BW-FCP-NEXT: vpshufb {{.*#+}} ymm4 = ymm18[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,25],zero,ymm18[27],zero,zero,ymm18[26],zero,ymm18[28],zero,ymm18[30],zero,zero,ymm18[29],zero,ymm18[31],zero ; AVX512DQ-BW-FCP-NEXT: vpshufb {{.*#+}} ymm6 = ymm14[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm14[27],zero,zero,ymm14[26],zero,ymm14[28],zero,ymm14[30],zero,zero,ymm14[29],zero,ymm14[31],zero,zero ; AVX512DQ-BW-FCP-NEXT: vpor %ymm4, %ymm6, %ymm4 ; AVX512DQ-BW-FCP-NEXT: vpmovsxbq {{.*#+}} zmm6 = [2,2,3,3,8,8,9,9] ; AVX512DQ-BW-FCP-NEXT: vpermt2q %zmm2, %zmm6, %zmm4 ; AVX512DQ-BW-FCP-NEXT: vpunpcklbw {{.*#+}} xmm2 = xmm3[0],xmm5[0],xmm3[1],xmm5[1],xmm3[2],xmm5[2],xmm3[3],xmm5[3],xmm3[4],xmm5[4],xmm3[5],xmm5[5],xmm3[6],xmm5[6],xmm3[7],xmm5[7] -; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm15, %xmm2, %xmm2 +; AVX512DQ-BW-FCP-NEXT: vpshufb %xmm17, %xmm2, %xmm2 ; AVX512DQ-BW-FCP-NEXT: vpshufb {{.*#+}} ymm3 = ymm7[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,27],zero,zero,ymm7[26],zero,ymm7[28],zero,zero,zero,zero,ymm7[29],zero,ymm7[31],zero,zero,ymm7[30] -; AVX512DQ-BW-FCP-NEXT: vpshufb {{.*#+}} ymm5 = ymm18[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,ymm18[26],zero,ymm18[28],zero,zero,ymm18[27],zero,ymm18[29],zero,ymm18[31],zero,zero,ymm18[30],zero +; AVX512DQ-BW-FCP-NEXT: vpshufb {{.*#+}} ymm5 = ymm19[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,ymm19[26],zero,ymm19[28],zero,zero,ymm19[27],zero,ymm19[29],zero,ymm19[31],zero,zero,ymm19[30],zero ; AVX512DQ-BW-FCP-NEXT: vpor %ymm3, %ymm5, %ymm3 ; AVX512DQ-BW-FCP-NEXT: vpermt2q %zmm2, %zmm6, %zmm3 ; AVX512DQ-BW-FCP-NEXT: vmovdqu8 %zmm4, %zmm3 {%k3} diff --git a/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-6.ll b/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-6.ll index 4d9cc3c8a7dc..37919128ba9f 100644 --- a/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-6.ll +++ b/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-6.ll @@ -1599,15 +1599,15 @@ define void @store_i8_stride6_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512BW-FCP-NEXT: vinserti128 $1, (%rsi), %ymm0, %ymm0 ; AVX512BW-FCP-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm3 ; AVX512BW-FCP-NEXT: vinserti32x4 $1, (%r9), %zmm2, %zmm2 -; AVX512BW-FCP-NEXT: vpmovsxbq {{.*#+}} zmm4 = [0,2,0,2,8,10,9,11] -; AVX512BW-FCP-NEXT: vmovdqa64 %zmm1, %zmm5 -; AVX512BW-FCP-NEXT: vpermt2q %zmm3, %zmm4, %zmm5 -; AVX512BW-FCP-NEXT: vpshufb {{.*#+}} zmm5 = zmm5[u,u,0,8,u,u,u,u,1,9,u,u,u,u,2,10,u,u,u,u,19,27,u,u,u,u,20,28,u,u,u,u,u,u,u,u,38,46,u,u,u,u,39,47,u,u,u,u,48,56,u,u,u,u,49,57,u,u,u,u,50,58,u,u] -; AVX512BW-FCP-NEXT: vpermt2q %zmm1, %zmm4, %zmm3 +; AVX512BW-FCP-NEXT: vpmovsxbq {{.*#+}} zmm4 = [4,6,4,6,0,2,1,3] +; AVX512BW-FCP-NEXT: vpermq %zmm3, %zmm4, %zmm4 +; AVX512BW-FCP-NEXT: vpshufb {{.*#+}} zmm4 = zmm4[u,u,0,8,u,u,u,u,1,9,u,u,u,u,2,10,u,u,u,u,19,27,u,u,u,u,20,28,u,u,u,u,u,u,u,u,38,46,u,u,u,u,39,47,u,u,u,u,48,56,u,u,u,u,49,57,u,u,u,u,50,58,u,u] +; AVX512BW-FCP-NEXT: vpmovsxbq {{.*#+}} zmm5 = [0,2,0,2,4,6,5,7] +; AVX512BW-FCP-NEXT: vpermq %zmm3, %zmm5, %zmm3 ; AVX512BW-FCP-NEXT: vpshufb {{.*#+}} zmm3 = zmm3[0,8,u,u,u,u,1,9,u,u,u,u,2,10,u,u,u,u,19,27,u,u,u,u,20,28,u,u,u,u,21,29,37,45,u,u,u,u,38,46,u,u,u,u,39,47,u,u,u,u,48,56,u,u,u,u,49,57,u,u,u,u,50,58] ; AVX512BW-FCP-NEXT: movl $1227105426, %ecx # imm = 0x49242492 ; AVX512BW-FCP-NEXT: kmovd %ecx, %k1 -; AVX512BW-FCP-NEXT: vmovdqu16 %zmm5, %zmm3 {%k1} +; AVX512BW-FCP-NEXT: vmovdqu16 %zmm4, %zmm3 {%k1} ; AVX512BW-FCP-NEXT: vpmovsxbq {{.*#+}} zmm4 = [0,2,0,2,0,2,1,3] ; AVX512BW-FCP-NEXT: vpermq %zmm2, %zmm4, %zmm4 ; AVX512BW-FCP-NEXT: vpshufb {{.*#+}} zmm4 = zmm4[u,u,u,u,0,8,u,u,u,u,1,9,u,u,u,u,18,26,u,u,u,u,19,27,u,u,u,u,20,28,u,u,u,u,37,45,u,u,u,u,38,46,u,u,u,u,39,47,u,u,u,u,48,56,u,u,u,u,49,57,u,u,u,u] @@ -1685,15 +1685,15 @@ define void @store_i8_stride6_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512DQ-BW-FCP-NEXT: vinserti128 $1, (%rsi), %ymm0, %ymm0 ; AVX512DQ-BW-FCP-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm3 ; AVX512DQ-BW-FCP-NEXT: vinserti32x4 $1, (%r9), %zmm2, %zmm2 -; AVX512DQ-BW-FCP-NEXT: vpmovsxbq {{.*#+}} zmm4 = [0,2,0,2,8,10,9,11] -; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm1, %zmm5 -; AVX512DQ-BW-FCP-NEXT: vpermt2q %zmm3, %zmm4, %zmm5 -; AVX512DQ-BW-FCP-NEXT: vpshufb {{.*#+}} zmm5 = zmm5[u,u,0,8,u,u,u,u,1,9,u,u,u,u,2,10,u,u,u,u,19,27,u,u,u,u,20,28,u,u,u,u,u,u,u,u,38,46,u,u,u,u,39,47,u,u,u,u,48,56,u,u,u,u,49,57,u,u,u,u,50,58,u,u] -; AVX512DQ-BW-FCP-NEXT: vpermt2q %zmm1, %zmm4, %zmm3 +; AVX512DQ-BW-FCP-NEXT: vpmovsxbq {{.*#+}} zmm4 = [4,6,4,6,0,2,1,3] +; AVX512DQ-BW-FCP-NEXT: vpermq %zmm3, %zmm4, %zmm4 +; AVX512DQ-BW-FCP-NEXT: vpshufb {{.*#+}} zmm4 = zmm4[u,u,0,8,u,u,u,u,1,9,u,u,u,u,2,10,u,u,u,u,19,27,u,u,u,u,20,28,u,u,u,u,u,u,u,u,38,46,u,u,u,u,39,47,u,u,u,u,48,56,u,u,u,u,49,57,u,u,u,u,50,58,u,u] +; AVX512DQ-BW-FCP-NEXT: vpmovsxbq {{.*#+}} zmm5 = [0,2,0,2,4,6,5,7] +; AVX512DQ-BW-FCP-NEXT: vpermq %zmm3, %zmm5, %zmm3 ; AVX512DQ-BW-FCP-NEXT: vpshufb {{.*#+}} zmm3 = zmm3[0,8,u,u,u,u,1,9,u,u,u,u,2,10,u,u,u,u,19,27,u,u,u,u,20,28,u,u,u,u,21,29,37,45,u,u,u,u,38,46,u,u,u,u,39,47,u,u,u,u,48,56,u,u,u,u,49,57,u,u,u,u,50,58] ; AVX512DQ-BW-FCP-NEXT: movl $1227105426, %ecx # imm = 0x49242492 ; AVX512DQ-BW-FCP-NEXT: kmovd %ecx, %k1 -; AVX512DQ-BW-FCP-NEXT: vmovdqu16 %zmm5, %zmm3 {%k1} +; AVX512DQ-BW-FCP-NEXT: vmovdqu16 %zmm4, %zmm3 {%k1} ; AVX512DQ-BW-FCP-NEXT: vpmovsxbq {{.*#+}} zmm4 = [0,2,0,2,0,2,1,3] ; AVX512DQ-BW-FCP-NEXT: vpermq %zmm2, %zmm4, %zmm4 ; AVX512DQ-BW-FCP-NEXT: vpshufb {{.*#+}} zmm4 = zmm4[u,u,u,u,0,8,u,u,u,u,1,9,u,u,u,u,18,26,u,u,u,u,19,27,u,u,u,u,20,28,u,u,u,u,37,45,u,u,u,u,38,46,u,u,u,u,39,47,u,u,u,u,48,56,u,u,u,u,49,57,u,u,u,u] diff --git a/llvm/test/CodeGen/X86/vector-reduce-fmax-nnan.ll b/llvm/test/CodeGen/X86/vector-reduce-fmax-nnan.ll index b214bf082f23..eafee9e65345 100644 --- a/llvm/test/CodeGen/X86/vector-reduce-fmax-nnan.ll +++ b/llvm/test/CodeGen/X86/vector-reduce-fmax-nnan.ll @@ -413,9 +413,12 @@ define half @test_v2f16(<2 x half> %a0) nounwind { ; AVX512F: # %bb.0: ; AVX512F-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0 ; AVX512F-NEXT: vpsrld $16, %xmm0, %xmm1 -; AVX512F-NEXT: vcvtph2ps %xmm0, %ymm2 -; AVX512F-NEXT: vcvtph2ps %xmm1, %ymm3 -; AVX512F-NEXT: vcmpltps %zmm2, %zmm3, %k1 +; AVX512F-NEXT: vcvtph2ps %xmm0, %xmm2 +; AVX512F-NEXT: vcvtph2ps %xmm1, %xmm3 +; AVX512F-NEXT: vucomiss %xmm3, %xmm2 +; AVX512F-NEXT: seta %al +; AVX512F-NEXT: negb %al +; AVX512F-NEXT: kmovd %eax, %k1 ; AVX512F-NEXT: vmovdqu16 %zmm0, %zmm1 {%k1} ; AVX512F-NEXT: vmovdqa %xmm1, %xmm0 ; AVX512F-NEXT: vzeroupper diff --git a/llvm/test/CodeGen/X86/vector-reduce-fmin-nnan.ll b/llvm/test/CodeGen/X86/vector-reduce-fmin-nnan.ll index 9f37df716b6c..e8f9c7f7b524 100644 --- a/llvm/test/CodeGen/X86/vector-reduce-fmin-nnan.ll +++ b/llvm/test/CodeGen/X86/vector-reduce-fmin-nnan.ll @@ -412,9 +412,12 @@ define half @test_v2f16(<2 x half> %a0) nounwind { ; AVX512F: # %bb.0: ; AVX512F-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0 ; AVX512F-NEXT: vpsrld $16, %xmm0, %xmm1 -; AVX512F-NEXT: vcvtph2ps %xmm0, %ymm2 -; AVX512F-NEXT: vcvtph2ps %xmm1, %ymm3 -; AVX512F-NEXT: vcmpltps %zmm3, %zmm2, %k1 +; AVX512F-NEXT: vcvtph2ps %xmm0, %xmm2 +; AVX512F-NEXT: vcvtph2ps %xmm1, %xmm3 +; AVX512F-NEXT: xorl %eax, %eax +; AVX512F-NEXT: vucomiss %xmm3, %xmm2 +; AVX512F-NEXT: sbbl %eax, %eax +; AVX512F-NEXT: kmovd %eax, %k1 ; AVX512F-NEXT: vmovdqu16 %zmm0, %zmm1 {%k1} ; AVX512F-NEXT: vmovdqa %xmm1, %xmm0 ; AVX512F-NEXT: vzeroupper diff --git a/llvm/test/CodeGen/X86/vector-replicaton-i1-mask.ll b/llvm/test/CodeGen/X86/vector-replicaton-i1-mask.ll index 358b2a503df2..a8df418143f3 100644 --- a/llvm/test/CodeGen/X86/vector-replicaton-i1-mask.ll +++ b/llvm/test/CodeGen/X86/vector-replicaton-i1-mask.ll @@ -256,12 +256,12 @@ define void @mask_replication_factor2_vf32(ptr %in.maskvec, ptr %in.vec, ptr %ou ; AVX512BW-ONLY-NEXT: vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,0,1,2,3,2,3] ; AVX512BW-ONLY-NEXT: vpshufb {{.*#+}} zmm0 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63] ; AVX512BW-ONLY-NEXT: vpmovb2m %zmm0, %k1 -; AVX512BW-ONLY-NEXT: kshiftrd $16, %k1, %k2 +; AVX512BW-ONLY-NEXT: kshiftrq $16, %k1, %k2 ; AVX512BW-ONLY-NEXT: vmovdqa32 64(%rsi), %zmm0 {%k2} {z} ; AVX512BW-ONLY-NEXT: vmovdqa32 (%rsi), %zmm1 {%k1} {z} -; AVX512BW-ONLY-NEXT: kshiftrq $32, %k1, %k1 -; AVX512BW-ONLY-NEXT: kshiftrd $16, %k1, %k2 +; AVX512BW-ONLY-NEXT: kshiftrq $48, %k1, %k2 ; AVX512BW-ONLY-NEXT: vmovdqa32 192(%rsi), %zmm2 {%k2} {z} +; AVX512BW-ONLY-NEXT: kshiftrq $32, %k1, %k1 ; AVX512BW-ONLY-NEXT: vmovdqa32 128(%rsi), %zmm3 {%k1} {z} ; AVX512BW-ONLY-NEXT: vmovdqa64 %zmm3, 128(%rdx) ; AVX512BW-ONLY-NEXT: vmovdqa64 %zmm2, 192(%rdx) @@ -277,12 +277,12 @@ define void @mask_replication_factor2_vf32(ptr %in.maskvec, ptr %in.vec, ptr %ou ; AVX512VBMI-ONLY-NEXT: vmovdqa64 {{.*#+}} zmm1 = [0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31] ; AVX512VBMI-ONLY-NEXT: vpermb %zmm0, %zmm1, %zmm0 ; AVX512VBMI-ONLY-NEXT: vpmovb2m %zmm0, %k1 -; AVX512VBMI-ONLY-NEXT: kshiftrd $16, %k1, %k2 +; AVX512VBMI-ONLY-NEXT: kshiftrq $16, %k1, %k2 ; AVX512VBMI-ONLY-NEXT: vmovdqa32 64(%rsi), %zmm0 {%k2} {z} ; AVX512VBMI-ONLY-NEXT: vmovdqa32 (%rsi), %zmm1 {%k1} {z} -; AVX512VBMI-ONLY-NEXT: kshiftrq $32, %k1, %k1 -; AVX512VBMI-ONLY-NEXT: kshiftrd $16, %k1, %k2 +; AVX512VBMI-ONLY-NEXT: kshiftrq $48, %k1, %k2 ; AVX512VBMI-ONLY-NEXT: vmovdqa32 192(%rsi), %zmm2 {%k2} {z} +; AVX512VBMI-ONLY-NEXT: kshiftrq $32, %k1, %k1 ; AVX512VBMI-ONLY-NEXT: vmovdqa32 128(%rsi), %zmm3 {%k1} {z} ; AVX512VBMI-ONLY-NEXT: vmovdqa64 %zmm3, 128(%rdx) ; AVX512VBMI-ONLY-NEXT: vmovdqa64 %zmm2, 192(%rdx) @@ -409,19 +409,19 @@ define void @mask_replication_factor2_vf64(ptr %in.maskvec, ptr %in.vec, ptr %ou ; AVX512BW-ONLY-NEXT: vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,0,1,2,3,2,3] ; AVX512BW-ONLY-NEXT: vpshufb %zmm2, %zmm0, %zmm0 ; AVX512BW-ONLY-NEXT: vpmovb2m %zmm0, %k2 -; AVX512BW-ONLY-NEXT: kshiftrd $16, %k2, %k3 +; AVX512BW-ONLY-NEXT: kshiftrq $16, %k2, %k3 ; AVX512BW-ONLY-NEXT: vmovdqa32 64(%rsi), %zmm0 {%k3} {z} ; AVX512BW-ONLY-NEXT: vmovdqa32 (%rsi), %zmm1 {%k2} {z} -; AVX512BW-ONLY-NEXT: kshiftrq $32, %k2, %k2 -; AVX512BW-ONLY-NEXT: kshiftrd $16, %k2, %k3 +; AVX512BW-ONLY-NEXT: kshiftrq $48, %k2, %k3 ; AVX512BW-ONLY-NEXT: vmovdqa32 192(%rsi), %zmm2 {%k3} {z} +; AVX512BW-ONLY-NEXT: kshiftrq $32, %k2, %k2 ; AVX512BW-ONLY-NEXT: vmovdqa32 128(%rsi), %zmm3 {%k2} {z} -; AVX512BW-ONLY-NEXT: kshiftrd $16, %k1, %k2 +; AVX512BW-ONLY-NEXT: kshiftrq $16, %k1, %k2 ; AVX512BW-ONLY-NEXT: vmovdqa32 320(%rsi), %zmm4 {%k2} {z} ; AVX512BW-ONLY-NEXT: vmovdqa32 256(%rsi), %zmm5 {%k1} {z} -; AVX512BW-ONLY-NEXT: kshiftrq $32, %k1, %k1 -; AVX512BW-ONLY-NEXT: kshiftrd $16, %k1, %k2 +; AVX512BW-ONLY-NEXT: kshiftrq $48, %k1, %k2 ; AVX512BW-ONLY-NEXT: vmovdqa32 448(%rsi), %zmm6 {%k2} {z} +; AVX512BW-ONLY-NEXT: kshiftrq $32, %k1, %k1 ; AVX512BW-ONLY-NEXT: vmovdqa32 384(%rsi), %zmm7 {%k1} {z} ; AVX512BW-ONLY-NEXT: vmovdqa64 %zmm7, 384(%rdx) ; AVX512BW-ONLY-NEXT: vmovdqa64 %zmm6, 448(%rdx) @@ -444,19 +444,19 @@ define void @mask_replication_factor2_vf64(ptr %in.maskvec, ptr %in.vec, ptr %ou ; AVX512VBMI-ONLY-NEXT: vmovdqa64 {{.*#+}} zmm1 = [0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31] ; AVX512VBMI-ONLY-NEXT: vpermb %zmm0, %zmm1, %zmm0 ; AVX512VBMI-ONLY-NEXT: vpmovb2m %zmm0, %k2 -; AVX512VBMI-ONLY-NEXT: kshiftrd $16, %k2, %k3 +; AVX512VBMI-ONLY-NEXT: kshiftrq $16, %k2, %k3 ; AVX512VBMI-ONLY-NEXT: vmovdqa32 64(%rsi), %zmm0 {%k3} {z} ; AVX512VBMI-ONLY-NEXT: vmovdqa32 (%rsi), %zmm1 {%k2} {z} -; AVX512VBMI-ONLY-NEXT: kshiftrq $32, %k2, %k2 -; AVX512VBMI-ONLY-NEXT: kshiftrd $16, %k2, %k3 +; AVX512VBMI-ONLY-NEXT: kshiftrq $48, %k2, %k3 ; AVX512VBMI-ONLY-NEXT: vmovdqa32 192(%rsi), %zmm2 {%k3} {z} +; AVX512VBMI-ONLY-NEXT: kshiftrq $32, %k2, %k2 ; AVX512VBMI-ONLY-NEXT: vmovdqa32 128(%rsi), %zmm3 {%k2} {z} -; AVX512VBMI-ONLY-NEXT: kshiftrd $16, %k1, %k2 +; AVX512VBMI-ONLY-NEXT: kshiftrq $16, %k1, %k2 ; AVX512VBMI-ONLY-NEXT: vmovdqa32 320(%rsi), %zmm4 {%k2} {z} ; AVX512VBMI-ONLY-NEXT: vmovdqa32 256(%rsi), %zmm5 {%k1} {z} -; AVX512VBMI-ONLY-NEXT: kshiftrq $32, %k1, %k1 -; AVX512VBMI-ONLY-NEXT: kshiftrd $16, %k1, %k2 +; AVX512VBMI-ONLY-NEXT: kshiftrq $48, %k1, %k2 ; AVX512VBMI-ONLY-NEXT: vmovdqa32 448(%rsi), %zmm6 {%k2} {z} +; AVX512VBMI-ONLY-NEXT: kshiftrq $32, %k1, %k1 ; AVX512VBMI-ONLY-NEXT: vmovdqa32 384(%rsi), %zmm7 {%k1} {z} ; AVX512VBMI-ONLY-NEXT: vmovdqa64 %zmm7, 384(%rdx) ; AVX512VBMI-ONLY-NEXT: vmovdqa64 %zmm6, 448(%rdx) @@ -2605,12 +2605,12 @@ define void @mask_replication_factor4_vf16(ptr %in.maskvec, ptr %in.vec, ptr %ou ; AVX512BW-ONLY-NEXT: vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,0,1,0,1,0,1] ; AVX512BW-ONLY-NEXT: vpshufb {{.*#+}} zmm0 = zmm0[0,0,0,0,1,1,1,1,2,2,2,2,3,3,3,3,20,20,20,20,21,21,21,21,22,22,22,22,23,23,23,23,40,40,40,40,41,41,41,41,42,42,42,42,43,43,43,43,60,60,60,60,61,61,61,61,62,62,62,62,63,63,63,63] ; AVX512BW-ONLY-NEXT: vpmovb2m %zmm0, %k1 -; AVX512BW-ONLY-NEXT: kshiftrd $16, %k1, %k2 +; AVX512BW-ONLY-NEXT: kshiftrq $16, %k1, %k2 ; AVX512BW-ONLY-NEXT: vmovdqa32 64(%rsi), %zmm0 {%k2} {z} ; AVX512BW-ONLY-NEXT: vmovdqa32 (%rsi), %zmm1 {%k1} {z} -; AVX512BW-ONLY-NEXT: kshiftrq $32, %k1, %k1 -; AVX512BW-ONLY-NEXT: kshiftrd $16, %k1, %k2 +; AVX512BW-ONLY-NEXT: kshiftrq $48, %k1, %k2 ; AVX512BW-ONLY-NEXT: vmovdqa32 192(%rsi), %zmm2 {%k2} {z} +; AVX512BW-ONLY-NEXT: kshiftrq $32, %k1, %k1 ; AVX512BW-ONLY-NEXT: vmovdqa32 128(%rsi), %zmm3 {%k1} {z} ; AVX512BW-ONLY-NEXT: vmovdqa64 %zmm3, 128(%rdx) ; AVX512BW-ONLY-NEXT: vmovdqa64 %zmm2, 192(%rdx) @@ -2626,12 +2626,12 @@ define void @mask_replication_factor4_vf16(ptr %in.maskvec, ptr %in.vec, ptr %ou ; AVX512VBMI-ONLY-NEXT: vmovdqa64 {{.*#+}} zmm1 = [0,0,0,0,1,1,1,1,2,2,2,2,3,3,3,3,4,4,4,4,5,5,5,5,6,6,6,6,7,7,7,7,8,8,8,8,9,9,9,9,10,10,10,10,11,11,11,11,12,12,12,12,13,13,13,13,14,14,14,14,15,15,15,15] ; AVX512VBMI-ONLY-NEXT: vpermb %zmm0, %zmm1, %zmm0 ; AVX512VBMI-ONLY-NEXT: vpmovb2m %zmm0, %k1 -; AVX512VBMI-ONLY-NEXT: kshiftrd $16, %k1, %k2 +; AVX512VBMI-ONLY-NEXT: kshiftrq $16, %k1, %k2 ; AVX512VBMI-ONLY-NEXT: vmovdqa32 64(%rsi), %zmm0 {%k2} {z} ; AVX512VBMI-ONLY-NEXT: vmovdqa32 (%rsi), %zmm1 {%k1} {z} -; AVX512VBMI-ONLY-NEXT: kshiftrq $32, %k1, %k1 -; AVX512VBMI-ONLY-NEXT: kshiftrd $16, %k1, %k2 +; AVX512VBMI-ONLY-NEXT: kshiftrq $48, %k1, %k2 ; AVX512VBMI-ONLY-NEXT: vmovdqa32 192(%rsi), %zmm2 {%k2} {z} +; AVX512VBMI-ONLY-NEXT: kshiftrq $32, %k1, %k1 ; AVX512VBMI-ONLY-NEXT: vmovdqa32 128(%rsi), %zmm3 {%k1} {z} ; AVX512VBMI-ONLY-NEXT: vmovdqa64 %zmm3, 128(%rdx) ; AVX512VBMI-ONLY-NEXT: vmovdqa64 %zmm2, 192(%rdx) @@ -2753,19 +2753,19 @@ define void @mask_replication_factor4_vf32(ptr %in.maskvec, ptr %in.vec, ptr %ou ; AVX512BW-ONLY-NEXT: vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,0,1,0,1,0,1] ; AVX512BW-ONLY-NEXT: vpshufb %zmm2, %zmm0, %zmm0 ; AVX512BW-ONLY-NEXT: vpmovb2m %zmm0, %k2 -; AVX512BW-ONLY-NEXT: kshiftrd $16, %k2, %k3 +; AVX512BW-ONLY-NEXT: kshiftrq $16, %k2, %k3 ; AVX512BW-ONLY-NEXT: vmovdqa32 64(%rsi), %zmm0 {%k3} {z} ; AVX512BW-ONLY-NEXT: vmovdqa32 (%rsi), %zmm1 {%k2} {z} -; AVX512BW-ONLY-NEXT: kshiftrq $32, %k2, %k2 -; AVX512BW-ONLY-NEXT: kshiftrd $16, %k2, %k3 +; AVX512BW-ONLY-NEXT: kshiftrq $48, %k2, %k3 ; AVX512BW-ONLY-NEXT: vmovdqa32 192(%rsi), %zmm2 {%k3} {z} +; AVX512BW-ONLY-NEXT: kshiftrq $32, %k2, %k2 ; AVX512BW-ONLY-NEXT: vmovdqa32 128(%rsi), %zmm3 {%k2} {z} -; AVX512BW-ONLY-NEXT: kshiftrd $16, %k1, %k2 +; AVX512BW-ONLY-NEXT: kshiftrq $16, %k1, %k2 ; AVX512BW-ONLY-NEXT: vmovdqa32 320(%rsi), %zmm4 {%k2} {z} ; AVX512BW-ONLY-NEXT: vmovdqa32 256(%rsi), %zmm5 {%k1} {z} -; AVX512BW-ONLY-NEXT: kshiftrq $32, %k1, %k1 -; AVX512BW-ONLY-NEXT: kshiftrd $16, %k1, %k2 +; AVX512BW-ONLY-NEXT: kshiftrq $48, %k1, %k2 ; AVX512BW-ONLY-NEXT: vmovdqa32 448(%rsi), %zmm6 {%k2} {z} +; AVX512BW-ONLY-NEXT: kshiftrq $32, %k1, %k1 ; AVX512BW-ONLY-NEXT: vmovdqa32 384(%rsi), %zmm7 {%k1} {z} ; AVX512BW-ONLY-NEXT: vmovdqa64 %zmm7, 384(%rdx) ; AVX512BW-ONLY-NEXT: vmovdqa64 %zmm6, 448(%rdx) @@ -2788,19 +2788,19 @@ define void @mask_replication_factor4_vf32(ptr %in.maskvec, ptr %in.vec, ptr %ou ; AVX512VBMI-ONLY-NEXT: vmovdqa64 {{.*#+}} zmm1 = [0,0,0,0,1,1,1,1,2,2,2,2,3,3,3,3,4,4,4,4,5,5,5,5,6,6,6,6,7,7,7,7,8,8,8,8,9,9,9,9,10,10,10,10,11,11,11,11,12,12,12,12,13,13,13,13,14,14,14,14,15,15,15,15] ; AVX512VBMI-ONLY-NEXT: vpermb %zmm0, %zmm1, %zmm0 ; AVX512VBMI-ONLY-NEXT: vpmovb2m %zmm0, %k2 -; AVX512VBMI-ONLY-NEXT: kshiftrd $16, %k2, %k3 +; AVX512VBMI-ONLY-NEXT: kshiftrq $16, %k2, %k3 ; AVX512VBMI-ONLY-NEXT: vmovdqa32 64(%rsi), %zmm0 {%k3} {z} ; AVX512VBMI-ONLY-NEXT: vmovdqa32 (%rsi), %zmm1 {%k2} {z} -; AVX512VBMI-ONLY-NEXT: kshiftrq $32, %k2, %k2 -; AVX512VBMI-ONLY-NEXT: kshiftrd $16, %k2, %k3 +; AVX512VBMI-ONLY-NEXT: kshiftrq $48, %k2, %k3 ; AVX512VBMI-ONLY-NEXT: vmovdqa32 192(%rsi), %zmm2 {%k3} {z} +; AVX512VBMI-ONLY-NEXT: kshiftrq $32, %k2, %k2 ; AVX512VBMI-ONLY-NEXT: vmovdqa32 128(%rsi), %zmm3 {%k2} {z} -; AVX512VBMI-ONLY-NEXT: kshiftrd $16, %k1, %k2 +; AVX512VBMI-ONLY-NEXT: kshiftrq $16, %k1, %k2 ; AVX512VBMI-ONLY-NEXT: vmovdqa32 320(%rsi), %zmm4 {%k2} {z} ; AVX512VBMI-ONLY-NEXT: vmovdqa32 256(%rsi), %zmm5 {%k1} {z} -; AVX512VBMI-ONLY-NEXT: kshiftrq $32, %k1, %k1 -; AVX512VBMI-ONLY-NEXT: kshiftrd $16, %k1, %k2 +; AVX512VBMI-ONLY-NEXT: kshiftrq $48, %k1, %k2 ; AVX512VBMI-ONLY-NEXT: vmovdqa32 448(%rsi), %zmm6 {%k2} {z} +; AVX512VBMI-ONLY-NEXT: kshiftrq $32, %k1, %k1 ; AVX512VBMI-ONLY-NEXT: vmovdqa32 384(%rsi), %zmm7 {%k1} {z} ; AVX512VBMI-ONLY-NEXT: vmovdqa64 %zmm7, 384(%rdx) ; AVX512VBMI-ONLY-NEXT: vmovdqa64 %zmm6, 448(%rdx) @@ -3000,33 +3000,33 @@ define void @mask_replication_factor4_vf64(ptr %in.maskvec, ptr %in.vec, ptr %ou ; AVX512BW-ONLY-NEXT: vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,0,1,0,1,0,1] ; AVX512BW-ONLY-NEXT: vpshufb %zmm2, %zmm0, %zmm0 ; AVX512BW-ONLY-NEXT: vpmovb2m %zmm0, %k4 -; AVX512BW-ONLY-NEXT: kshiftrd $16, %k4, %k5 +; AVX512BW-ONLY-NEXT: kshiftrq $16, %k4, %k5 ; AVX512BW-ONLY-NEXT: vmovdqa32 64(%rsi), %zmm0 {%k5} {z} ; AVX512BW-ONLY-NEXT: vmovdqa32 (%rsi), %zmm1 {%k4} {z} -; AVX512BW-ONLY-NEXT: kshiftrq $32, %k4, %k4 -; AVX512BW-ONLY-NEXT: kshiftrd $16, %k4, %k5 +; AVX512BW-ONLY-NEXT: kshiftrq $48, %k4, %k5 ; AVX512BW-ONLY-NEXT: vmovdqa32 192(%rsi), %zmm2 {%k5} {z} +; AVX512BW-ONLY-NEXT: kshiftrq $32, %k4, %k4 ; AVX512BW-ONLY-NEXT: vmovdqa32 128(%rsi), %zmm3 {%k4} {z} -; AVX512BW-ONLY-NEXT: kshiftrd $16, %k3, %k4 +; AVX512BW-ONLY-NEXT: kshiftrq $16, %k3, %k4 ; AVX512BW-ONLY-NEXT: vmovdqa32 320(%rsi), %zmm4 {%k4} {z} ; AVX512BW-ONLY-NEXT: vmovdqa32 256(%rsi), %zmm5 {%k3} {z} -; AVX512BW-ONLY-NEXT: kshiftrq $32, %k3, %k3 -; AVX512BW-ONLY-NEXT: kshiftrd $16, %k3, %k4 +; AVX512BW-ONLY-NEXT: kshiftrq $48, %k3, %k4 ; AVX512BW-ONLY-NEXT: vmovdqa32 448(%rsi), %zmm6 {%k4} {z} +; AVX512BW-ONLY-NEXT: kshiftrq $32, %k3, %k3 ; AVX512BW-ONLY-NEXT: vmovdqa32 384(%rsi), %zmm7 {%k3} {z} -; AVX512BW-ONLY-NEXT: kshiftrd $16, %k2, %k3 +; AVX512BW-ONLY-NEXT: kshiftrq $16, %k2, %k3 ; AVX512BW-ONLY-NEXT: vmovdqa32 576(%rsi), %zmm8 {%k3} {z} ; AVX512BW-ONLY-NEXT: vmovdqa32 512(%rsi), %zmm9 {%k2} {z} -; AVX512BW-ONLY-NEXT: kshiftrq $32, %k2, %k2 -; AVX512BW-ONLY-NEXT: kshiftrd $16, %k2, %k3 +; AVX512BW-ONLY-NEXT: kshiftrq $48, %k2, %k3 ; AVX512BW-ONLY-NEXT: vmovdqa32 704(%rsi), %zmm10 {%k3} {z} +; AVX512BW-ONLY-NEXT: kshiftrq $32, %k2, %k2 ; AVX512BW-ONLY-NEXT: vmovdqa32 640(%rsi), %zmm11 {%k2} {z} -; AVX512BW-ONLY-NEXT: kshiftrd $16, %k1, %k2 +; AVX512BW-ONLY-NEXT: kshiftrq $16, %k1, %k2 ; AVX512BW-ONLY-NEXT: vmovdqa32 832(%rsi), %zmm12 {%k2} {z} ; AVX512BW-ONLY-NEXT: vmovdqa32 768(%rsi), %zmm13 {%k1} {z} -; AVX512BW-ONLY-NEXT: kshiftrq $32, %k1, %k1 -; AVX512BW-ONLY-NEXT: kshiftrd $16, %k1, %k2 +; AVX512BW-ONLY-NEXT: kshiftrq $48, %k1, %k2 ; AVX512BW-ONLY-NEXT: vmovdqa32 960(%rsi), %zmm14 {%k2} {z} +; AVX512BW-ONLY-NEXT: kshiftrq $32, %k1, %k1 ; AVX512BW-ONLY-NEXT: vmovdqa32 896(%rsi), %zmm15 {%k1} {z} ; AVX512BW-ONLY-NEXT: vmovdqa64 %zmm15, 896(%rdx) ; AVX512BW-ONLY-NEXT: vmovdqa64 %zmm14, 960(%rdx) @@ -3063,33 +3063,33 @@ define void @mask_replication_factor4_vf64(ptr %in.maskvec, ptr %in.vec, ptr %ou ; AVX512VBMI-ONLY-NEXT: vmovdqa64 {{.*#+}} zmm1 = [0,0,0,0,1,1,1,1,2,2,2,2,3,3,3,3,4,4,4,4,5,5,5,5,6,6,6,6,7,7,7,7,8,8,8,8,9,9,9,9,10,10,10,10,11,11,11,11,12,12,12,12,13,13,13,13,14,14,14,14,15,15,15,15] ; AVX512VBMI-ONLY-NEXT: vpermb %zmm0, %zmm1, %zmm0 ; AVX512VBMI-ONLY-NEXT: vpmovb2m %zmm0, %k4 -; AVX512VBMI-ONLY-NEXT: kshiftrd $16, %k4, %k5 +; AVX512VBMI-ONLY-NEXT: kshiftrq $16, %k4, %k5 ; AVX512VBMI-ONLY-NEXT: vmovdqa32 64(%rsi), %zmm0 {%k5} {z} ; AVX512VBMI-ONLY-NEXT: vmovdqa32 (%rsi), %zmm1 {%k4} {z} -; AVX512VBMI-ONLY-NEXT: kshiftrq $32, %k4, %k4 -; AVX512VBMI-ONLY-NEXT: kshiftrd $16, %k4, %k5 +; AVX512VBMI-ONLY-NEXT: kshiftrq $48, %k4, %k5 ; AVX512VBMI-ONLY-NEXT: vmovdqa32 192(%rsi), %zmm2 {%k5} {z} +; AVX512VBMI-ONLY-NEXT: kshiftrq $32, %k4, %k4 ; AVX512VBMI-ONLY-NEXT: vmovdqa32 128(%rsi), %zmm3 {%k4} {z} -; AVX512VBMI-ONLY-NEXT: kshiftrd $16, %k3, %k4 +; AVX512VBMI-ONLY-NEXT: kshiftrq $16, %k3, %k4 ; AVX512VBMI-ONLY-NEXT: vmovdqa32 320(%rsi), %zmm4 {%k4} {z} ; AVX512VBMI-ONLY-NEXT: vmovdqa32 256(%rsi), %zmm5 {%k3} {z} -; AVX512VBMI-ONLY-NEXT: kshiftrq $32, %k3, %k3 -; AVX512VBMI-ONLY-NEXT: kshiftrd $16, %k3, %k4 +; AVX512VBMI-ONLY-NEXT: kshiftrq $48, %k3, %k4 ; AVX512VBMI-ONLY-NEXT: vmovdqa32 448(%rsi), %zmm6 {%k4} {z} +; AVX512VBMI-ONLY-NEXT: kshiftrq $32, %k3, %k3 ; AVX512VBMI-ONLY-NEXT: vmovdqa32 384(%rsi), %zmm7 {%k3} {z} -; AVX512VBMI-ONLY-NEXT: kshiftrd $16, %k2, %k3 +; AVX512VBMI-ONLY-NEXT: kshiftrq $16, %k2, %k3 ; AVX512VBMI-ONLY-NEXT: vmovdqa32 576(%rsi), %zmm8 {%k3} {z} ; AVX512VBMI-ONLY-NEXT: vmovdqa32 512(%rsi), %zmm9 {%k2} {z} -; AVX512VBMI-ONLY-NEXT: kshiftrq $32, %k2, %k2 -; AVX512VBMI-ONLY-NEXT: kshiftrd $16, %k2, %k3 +; AVX512VBMI-ONLY-NEXT: kshiftrq $48, %k2, %k3 ; AVX512VBMI-ONLY-NEXT: vmovdqa32 704(%rsi), %zmm10 {%k3} {z} +; AVX512VBMI-ONLY-NEXT: kshiftrq $32, %k2, %k2 ; AVX512VBMI-ONLY-NEXT: vmovdqa32 640(%rsi), %zmm11 {%k2} {z} -; AVX512VBMI-ONLY-NEXT: kshiftrd $16, %k1, %k2 +; AVX512VBMI-ONLY-NEXT: kshiftrq $16, %k1, %k2 ; AVX512VBMI-ONLY-NEXT: vmovdqa32 832(%rsi), %zmm12 {%k2} {z} ; AVX512VBMI-ONLY-NEXT: vmovdqa32 768(%rsi), %zmm13 {%k1} {z} -; AVX512VBMI-ONLY-NEXT: kshiftrq $32, %k1, %k1 -; AVX512VBMI-ONLY-NEXT: kshiftrd $16, %k1, %k2 +; AVX512VBMI-ONLY-NEXT: kshiftrq $48, %k1, %k2 ; AVX512VBMI-ONLY-NEXT: vmovdqa32 960(%rsi), %zmm14 {%k2} {z} +; AVX512VBMI-ONLY-NEXT: kshiftrq $32, %k1, %k1 ; AVX512VBMI-ONLY-NEXT: vmovdqa32 896(%rsi), %zmm15 {%k1} {z} ; AVX512VBMI-ONLY-NEXT: vmovdqa64 %zmm15, 896(%rdx) ; AVX512VBMI-ONLY-NEXT: vmovdqa64 %zmm14, 960(%rdx) @@ -3309,14 +3309,14 @@ define void @mask_replication_factor5_vf8(ptr %in.maskvec, ptr %in.vec, ptr %out ; AVX512BW-ONLY-NEXT: movabsq $1099511627775, %rax # imm = 0xFFFFFFFFFF ; AVX512BW-ONLY-NEXT: kmovq %rax, %k1 ; AVX512BW-ONLY-NEXT: vpcmpgtb %zmm0, %zmm1, %k1 {%k1} -; AVX512BW-ONLY-NEXT: vmovdqa32 (%rsi), %zmm0 {%k1} {z} ; AVX512BW-ONLY-NEXT: kshiftrq $32, %k1, %k2 -; AVX512BW-ONLY-NEXT: vmovdqa32 128(%rsi), %zmm1 {%k2} {z} -; AVX512BW-ONLY-NEXT: kshiftrd $16, %k1, %k1 +; AVX512BW-ONLY-NEXT: vmovdqa32 128(%rsi), %zmm0 {%k2} {z} +; AVX512BW-ONLY-NEXT: vmovdqa32 (%rsi), %zmm1 {%k1} {z} +; AVX512BW-ONLY-NEXT: kshiftrq $16, %k1, %k1 ; AVX512BW-ONLY-NEXT: vmovdqa32 64(%rsi), %zmm2 {%k1} {z} ; AVX512BW-ONLY-NEXT: vmovdqa64 %zmm2, 64(%rdx) -; AVX512BW-ONLY-NEXT: vmovdqa %ymm1, 128(%rdx) -; AVX512BW-ONLY-NEXT: vmovdqa64 %zmm0, (%rdx) +; AVX512BW-ONLY-NEXT: vmovdqa64 %zmm1, (%rdx) +; AVX512BW-ONLY-NEXT: vmovdqa %ymm0, 128(%rdx) ; AVX512BW-ONLY-NEXT: vzeroupper ; AVX512BW-ONLY-NEXT: retq ; @@ -3330,14 +3330,14 @@ define void @mask_replication_factor5_vf8(ptr %in.maskvec, ptr %in.vec, ptr %out ; AVX512VBMI-ONLY-NEXT: movabsq $1099511627775, %rax # imm = 0xFFFFFFFFFF ; AVX512VBMI-ONLY-NEXT: kmovq %rax, %k1 ; AVX512VBMI-ONLY-NEXT: vpcmpgtb %zmm0, %zmm1, %k1 {%k1} -; AVX512VBMI-ONLY-NEXT: vmovdqa32 (%rsi), %zmm0 {%k1} {z} ; AVX512VBMI-ONLY-NEXT: kshiftrq $32, %k1, %k2 -; AVX512VBMI-ONLY-NEXT: vmovdqa32 128(%rsi), %zmm1 {%k2} {z} -; AVX512VBMI-ONLY-NEXT: kshiftrd $16, %k1, %k1 +; AVX512VBMI-ONLY-NEXT: vmovdqa32 128(%rsi), %zmm0 {%k2} {z} +; AVX512VBMI-ONLY-NEXT: vmovdqa32 (%rsi), %zmm1 {%k1} {z} +; AVX512VBMI-ONLY-NEXT: kshiftrq $16, %k1, %k1 ; AVX512VBMI-ONLY-NEXT: vmovdqa32 64(%rsi), %zmm2 {%k1} {z} ; AVX512VBMI-ONLY-NEXT: vmovdqa64 %zmm2, 64(%rdx) -; AVX512VBMI-ONLY-NEXT: vmovdqa %ymm1, 128(%rdx) -; AVX512VBMI-ONLY-NEXT: vmovdqa64 %zmm0, (%rdx) +; AVX512VBMI-ONLY-NEXT: vmovdqa64 %zmm1, (%rdx) +; AVX512VBMI-ONLY-NEXT: vmovdqa %ymm0, 128(%rdx) ; AVX512VBMI-ONLY-NEXT: vzeroupper ; AVX512VBMI-ONLY-NEXT: retq %src.mask.padded = load <64 x i1>, ptr %in.maskvec, align 64 @@ -9338,12 +9338,12 @@ define void @mask_replication_factor7_vf8(ptr %in.maskvec, ptr %in.vec, ptr %out ; AVX512BW-ONLY-NEXT: movabsq $72057594037927935, %rax # imm = 0xFFFFFFFFFFFFFF ; AVX512BW-ONLY-NEXT: kmovq %rax, %k1 ; AVX512BW-ONLY-NEXT: vpcmpgtb %zmm0, %zmm1, %k1 {%k1} -; AVX512BW-ONLY-NEXT: kshiftrq $32, %k1, %k2 -; AVX512BW-ONLY-NEXT: kshiftrd $16, %k2, %k3 -; AVX512BW-ONLY-NEXT: vmovdqa32 192(%rsi), %zmm0 {%k3} {z} +; AVX512BW-ONLY-NEXT: kshiftrq $48, %k1, %k2 +; AVX512BW-ONLY-NEXT: vmovdqa32 192(%rsi), %zmm0 {%k2} {z} ; AVX512BW-ONLY-NEXT: vmovdqa32 (%rsi), %zmm1 {%k1} {z} +; AVX512BW-ONLY-NEXT: kshiftrq $32, %k1, %k2 ; AVX512BW-ONLY-NEXT: vmovdqa32 128(%rsi), %zmm2 {%k2} {z} -; AVX512BW-ONLY-NEXT: kshiftrd $16, %k1, %k1 +; AVX512BW-ONLY-NEXT: kshiftrq $16, %k1, %k1 ; AVX512BW-ONLY-NEXT: vmovdqa32 64(%rsi), %zmm3 {%k1} {z} ; AVX512BW-ONLY-NEXT: vmovdqa64 %zmm3, 64(%rdx) ; AVX512BW-ONLY-NEXT: vmovdqa64 %zmm2, 128(%rdx) @@ -9362,12 +9362,12 @@ define void @mask_replication_factor7_vf8(ptr %in.maskvec, ptr %in.vec, ptr %out ; AVX512VBMI-ONLY-NEXT: movabsq $72057594037927935, %rax # imm = 0xFFFFFFFFFFFFFF ; AVX512VBMI-ONLY-NEXT: kmovq %rax, %k1 ; AVX512VBMI-ONLY-NEXT: vpcmpgtb %zmm0, %zmm1, %k1 {%k1} -; AVX512VBMI-ONLY-NEXT: kshiftrq $32, %k1, %k2 -; AVX512VBMI-ONLY-NEXT: kshiftrd $16, %k2, %k3 -; AVX512VBMI-ONLY-NEXT: vmovdqa32 192(%rsi), %zmm0 {%k3} {z} +; AVX512VBMI-ONLY-NEXT: kshiftrq $48, %k1, %k2 +; AVX512VBMI-ONLY-NEXT: vmovdqa32 192(%rsi), %zmm0 {%k2} {z} ; AVX512VBMI-ONLY-NEXT: vmovdqa32 (%rsi), %zmm1 {%k1} {z} +; AVX512VBMI-ONLY-NEXT: kshiftrq $32, %k1, %k2 ; AVX512VBMI-ONLY-NEXT: vmovdqa32 128(%rsi), %zmm2 {%k2} {z} -; AVX512VBMI-ONLY-NEXT: kshiftrd $16, %k1, %k1 +; AVX512VBMI-ONLY-NEXT: kshiftrq $16, %k1, %k1 ; AVX512VBMI-ONLY-NEXT: vmovdqa32 64(%rsi), %zmm3 {%k1} {z} ; AVX512VBMI-ONLY-NEXT: vmovdqa64 %zmm3, 64(%rdx) ; AVX512VBMI-ONLY-NEXT: vmovdqa64 %zmm2, 128(%rdx) @@ -12938,12 +12938,12 @@ define void @mask_replication_factor8_vf8(ptr %in.maskvec, ptr %in.vec, ptr %out ; AVX512BW-ONLY-NEXT: vpbroadcastq %xmm0, %zmm0 ; AVX512BW-ONLY-NEXT: vpshufb {{.*#+}} zmm0 = zmm0[0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,18,18,18,18,18,18,18,18,19,19,19,19,19,19,19,19,36,36,36,36,36,36,36,36,37,37,37,37,37,37,37,37,54,54,54,54,54,54,54,54,55,55,55,55,55,55,55,55] ; AVX512BW-ONLY-NEXT: vpmovb2m %zmm0, %k1 -; AVX512BW-ONLY-NEXT: kshiftrd $16, %k1, %k2 +; AVX512BW-ONLY-NEXT: kshiftrq $16, %k1, %k2 ; AVX512BW-ONLY-NEXT: vmovdqa32 64(%rsi), %zmm0 {%k2} {z} ; AVX512BW-ONLY-NEXT: vmovdqa32 (%rsi), %zmm1 {%k1} {z} -; AVX512BW-ONLY-NEXT: kshiftrq $32, %k1, %k1 -; AVX512BW-ONLY-NEXT: kshiftrd $16, %k1, %k2 +; AVX512BW-ONLY-NEXT: kshiftrq $48, %k1, %k2 ; AVX512BW-ONLY-NEXT: vmovdqa32 192(%rsi), %zmm2 {%k2} {z} +; AVX512BW-ONLY-NEXT: kshiftrq $32, %k1, %k1 ; AVX512BW-ONLY-NEXT: vmovdqa32 128(%rsi), %zmm3 {%k1} {z} ; AVX512BW-ONLY-NEXT: vmovdqa64 %zmm3, 128(%rdx) ; AVX512BW-ONLY-NEXT: vmovdqa64 %zmm2, 192(%rdx) @@ -12959,12 +12959,12 @@ define void @mask_replication_factor8_vf8(ptr %in.maskvec, ptr %in.vec, ptr %out ; AVX512VBMI-ONLY-NEXT: vmovdqa64 {{.*#+}} zmm1 = [0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,2,2,2,2,2,2,2,2,3,3,3,3,3,3,3,3,4,4,4,4,4,4,4,4,5,5,5,5,5,5,5,5,6,6,6,6,6,6,6,6,7,7,7,7,7,7,7,7] ; AVX512VBMI-ONLY-NEXT: vpermb %zmm0, %zmm1, %zmm0 ; AVX512VBMI-ONLY-NEXT: vpmovb2m %zmm0, %k1 -; AVX512VBMI-ONLY-NEXT: kshiftrd $16, %k1, %k2 +; AVX512VBMI-ONLY-NEXT: kshiftrq $16, %k1, %k2 ; AVX512VBMI-ONLY-NEXT: vmovdqa32 64(%rsi), %zmm0 {%k2} {z} ; AVX512VBMI-ONLY-NEXT: vmovdqa32 (%rsi), %zmm1 {%k1} {z} -; AVX512VBMI-ONLY-NEXT: kshiftrq $32, %k1, %k1 -; AVX512VBMI-ONLY-NEXT: kshiftrd $16, %k1, %k2 +; AVX512VBMI-ONLY-NEXT: kshiftrq $48, %k1, %k2 ; AVX512VBMI-ONLY-NEXT: vmovdqa32 192(%rsi), %zmm2 {%k2} {z} +; AVX512VBMI-ONLY-NEXT: kshiftrq $32, %k1, %k1 ; AVX512VBMI-ONLY-NEXT: vmovdqa32 128(%rsi), %zmm3 {%k1} {z} ; AVX512VBMI-ONLY-NEXT: vmovdqa64 %zmm3, 128(%rdx) ; AVX512VBMI-ONLY-NEXT: vmovdqa64 %zmm2, 192(%rdx) @@ -13088,19 +13088,19 @@ define void @mask_replication_factor8_vf16(ptr %in.maskvec, ptr %in.vec, ptr %ou ; AVX512BW-NEXT: vpmovb2m %zmm1, %k1 ; AVX512BW-NEXT: vpshufb {{.*#+}} zmm0 = zmm0[0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,18,18,18,18,18,18,18,18,19,19,19,19,19,19,19,19,36,36,36,36,36,36,36,36,37,37,37,37,37,37,37,37,54,54,54,54,54,54,54,54,55,55,55,55,55,55,55,55] ; AVX512BW-NEXT: vpmovb2m %zmm0, %k2 -; AVX512BW-NEXT: kshiftrd $16, %k2, %k3 +; AVX512BW-NEXT: kshiftrq $16, %k2, %k3 ; AVX512BW-NEXT: vmovdqa32 64(%rsi), %zmm0 {%k3} {z} ; AVX512BW-NEXT: vmovdqa32 (%rsi), %zmm1 {%k2} {z} -; AVX512BW-NEXT: kshiftrq $32, %k2, %k2 -; AVX512BW-NEXT: kshiftrd $16, %k2, %k3 +; AVX512BW-NEXT: kshiftrq $48, %k2, %k3 ; AVX512BW-NEXT: vmovdqa32 192(%rsi), %zmm2 {%k3} {z} +; AVX512BW-NEXT: kshiftrq $32, %k2, %k2 ; AVX512BW-NEXT: vmovdqa32 128(%rsi), %zmm3 {%k2} {z} -; AVX512BW-NEXT: kshiftrd $16, %k1, %k2 +; AVX512BW-NEXT: kshiftrq $16, %k1, %k2 ; AVX512BW-NEXT: vmovdqa32 320(%rsi), %zmm4 {%k2} {z} ; AVX512BW-NEXT: vmovdqa32 256(%rsi), %zmm5 {%k1} {z} -; AVX512BW-NEXT: kshiftrq $32, %k1, %k1 -; AVX512BW-NEXT: kshiftrd $16, %k1, %k2 +; AVX512BW-NEXT: kshiftrq $48, %k1, %k2 ; AVX512BW-NEXT: vmovdqa32 448(%rsi), %zmm6 {%k2} {z} +; AVX512BW-NEXT: kshiftrq $32, %k1, %k1 ; AVX512BW-NEXT: vmovdqa32 384(%rsi), %zmm7 {%k1} {z} ; AVX512BW-NEXT: vmovdqa64 %zmm7, 384(%rdx) ; AVX512BW-NEXT: vmovdqa64 %zmm6, 448(%rdx) @@ -13299,33 +13299,33 @@ define void @mask_replication_factor8_vf32(ptr %in.maskvec, ptr %in.vec, ptr %ou ; AVX512BW-NEXT: vpmovb2m %zmm1, %k3 ; AVX512BW-NEXT: vpshufb %zmm3, %zmm0, %zmm0 ; AVX512BW-NEXT: vpmovb2m %zmm0, %k4 -; AVX512BW-NEXT: kshiftrd $16, %k4, %k5 +; AVX512BW-NEXT: kshiftrq $16, %k4, %k5 ; AVX512BW-NEXT: vmovdqa32 64(%rsi), %zmm0 {%k5} {z} ; AVX512BW-NEXT: vmovdqa32 (%rsi), %zmm1 {%k4} {z} -; AVX512BW-NEXT: kshiftrq $32, %k4, %k4 -; AVX512BW-NEXT: kshiftrd $16, %k4, %k5 +; AVX512BW-NEXT: kshiftrq $48, %k4, %k5 ; AVX512BW-NEXT: vmovdqa32 192(%rsi), %zmm2 {%k5} {z} +; AVX512BW-NEXT: kshiftrq $32, %k4, %k4 ; AVX512BW-NEXT: vmovdqa32 128(%rsi), %zmm3 {%k4} {z} -; AVX512BW-NEXT: kshiftrd $16, %k3, %k4 +; AVX512BW-NEXT: kshiftrq $16, %k3, %k4 ; AVX512BW-NEXT: vmovdqa32 320(%rsi), %zmm4 {%k4} {z} ; AVX512BW-NEXT: vmovdqa32 256(%rsi), %zmm5 {%k3} {z} -; AVX512BW-NEXT: kshiftrq $32, %k3, %k3 -; AVX512BW-NEXT: kshiftrd $16, %k3, %k4 +; AVX512BW-NEXT: kshiftrq $48, %k3, %k4 ; AVX512BW-NEXT: vmovdqa32 448(%rsi), %zmm6 {%k4} {z} +; AVX512BW-NEXT: kshiftrq $32, %k3, %k3 ; AVX512BW-NEXT: vmovdqa32 384(%rsi), %zmm7 {%k3} {z} -; AVX512BW-NEXT: kshiftrd $16, %k2, %k3 +; AVX512BW-NEXT: kshiftrq $16, %k2, %k3 ; AVX512BW-NEXT: vmovdqa32 576(%rsi), %zmm8 {%k3} {z} ; AVX512BW-NEXT: vmovdqa32 512(%rsi), %zmm9 {%k2} {z} -; AVX512BW-NEXT: kshiftrq $32, %k2, %k2 -; AVX512BW-NEXT: kshiftrd $16, %k2, %k3 +; AVX512BW-NEXT: kshiftrq $48, %k2, %k3 ; AVX512BW-NEXT: vmovdqa32 704(%rsi), %zmm10 {%k3} {z} +; AVX512BW-NEXT: kshiftrq $32, %k2, %k2 ; AVX512BW-NEXT: vmovdqa32 640(%rsi), %zmm11 {%k2} {z} -; AVX512BW-NEXT: kshiftrd $16, %k1, %k2 +; AVX512BW-NEXT: kshiftrq $16, %k1, %k2 ; AVX512BW-NEXT: vmovdqa32 832(%rsi), %zmm12 {%k2} {z} ; AVX512BW-NEXT: vmovdqa32 768(%rsi), %zmm13 {%k1} {z} -; AVX512BW-NEXT: kshiftrq $32, %k1, %k1 -; AVX512BW-NEXT: kshiftrd $16, %k1, %k2 +; AVX512BW-NEXT: kshiftrq $48, %k1, %k2 ; AVX512BW-NEXT: vmovdqa32 960(%rsi), %zmm14 {%k2} {z} +; AVX512BW-NEXT: kshiftrq $32, %k1, %k1 ; AVX512BW-NEXT: vmovdqa32 896(%rsi), %zmm15 {%k1} {z} ; AVX512BW-NEXT: vmovdqa64 %zmm15, 896(%rdx) ; AVX512BW-NEXT: vmovdqa64 %zmm14, 960(%rdx) @@ -13682,8 +13682,8 @@ define void @mask_replication_factor8_vf64(ptr %in.maskvec, ptr %in.vec, ptr %ou ; AVX512BW-NEXT: vmovdqa64 {{.*#+}} zmm3 = [0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,2,2,2,2,2,2,2,2,3,3,3,3,3,3,3,3,4,4,4,4,4,4,4,4,5,5,5,5,5,5,5,5,6,6,6,6,6,6,6,6,7,7,7,7,7,7,7,7] ; AVX512BW-NEXT: vpshufb %zmm3, %zmm1, %zmm12 ; AVX512BW-NEXT: vshufi64x2 {{.*#+}} zmm1 = zmm0[4,5,4,5,4,5,4,5] -; AVX512BW-NEXT: vpshufb %zmm2, %zmm1, %zmm16 -; AVX512BW-NEXT: vpshufb %zmm3, %zmm1, %zmm15 +; AVX512BW-NEXT: vpshufb %zmm2, %zmm1, %zmm15 +; AVX512BW-NEXT: vpshufb %zmm3, %zmm1, %zmm16 ; AVX512BW-NEXT: vshufi64x2 {{.*#+}} zmm1 = zmm0[2,3,2,3,2,3,2,3] ; AVX512BW-NEXT: vpshufb %zmm2, %zmm1, %zmm10 ; AVX512BW-NEXT: vpshufb %zmm3, %zmm1, %zmm5 @@ -13691,73 +13691,73 @@ define void @mask_replication_factor8_vf64(ptr %in.maskvec, ptr %in.vec, ptr %ou ; AVX512BW-NEXT: vpshufb %zmm2, %zmm0, %zmm1 ; AVX512BW-NEXT: vpshufb %zmm3, %zmm0, %zmm0 ; AVX512BW-NEXT: vpmovb2m %zmm0, %k2 -; AVX512BW-NEXT: kshiftrd $16, %k2, %k1 +; AVX512BW-NEXT: kshiftrq $16, %k2, %k1 ; AVX512BW-NEXT: vmovdqa32 64(%rsi), %zmm0 {%k1} {z} +; AVX512BW-NEXT: kshiftrq $32, %k2, %k1 +; AVX512BW-NEXT: vmovdqa32 128(%rsi), %zmm2 {%k1} {z} ; AVX512BW-NEXT: vpmovb2m %zmm1, %k1 ; AVX512BW-NEXT: vmovdqa32 (%rsi), %zmm1 {%k2} {z} -; AVX512BW-NEXT: kshiftrq $32, %k2, %k2 -; AVX512BW-NEXT: vmovdqa32 128(%rsi), %zmm2 {%k2} {z} -; AVX512BW-NEXT: kshiftrd $16, %k2, %k2 +; AVX512BW-NEXT: kshiftrq $48, %k2, %k2 ; AVX512BW-NEXT: vmovdqa32 192(%rsi), %zmm3 {%k2} {z} -; AVX512BW-NEXT: kshiftrd $16, %k1, %k2 +; AVX512BW-NEXT: kshiftrq $16, %k1, %k2 ; AVX512BW-NEXT: vmovdqa32 320(%rsi), %zmm4 {%k2} {z} +; AVX512BW-NEXT: kshiftrq $32, %k1, %k2 +; AVX512BW-NEXT: vmovdqa32 384(%rsi), %zmm6 {%k2} {z} ; AVX512BW-NEXT: vpmovb2m %zmm5, %k2 ; AVX512BW-NEXT: vmovdqa32 256(%rsi), %zmm5 {%k1} {z} -; AVX512BW-NEXT: kshiftrq $32, %k1, %k1 -; AVX512BW-NEXT: vmovdqa32 384(%rsi), %zmm6 {%k1} {z} -; AVX512BW-NEXT: kshiftrd $16, %k1, %k1 +; AVX512BW-NEXT: kshiftrq $48, %k1, %k1 ; AVX512BW-NEXT: vmovdqa32 448(%rsi), %zmm8 {%k1} {z} -; AVX512BW-NEXT: kshiftrd $16, %k2, %k1 +; AVX512BW-NEXT: kshiftrq $16, %k2, %k1 ; AVX512BW-NEXT: vmovdqa32 576(%rsi), %zmm9 {%k1} {z} +; AVX512BW-NEXT: kshiftrq $32, %k2, %k1 +; AVX512BW-NEXT: vmovdqa32 640(%rsi), %zmm11 {%k1} {z} ; AVX512BW-NEXT: vpmovb2m %zmm10, %k1 ; AVX512BW-NEXT: vmovdqa32 512(%rsi), %zmm10 {%k2} {z} -; AVX512BW-NEXT: kshiftrq $32, %k2, %k2 -; AVX512BW-NEXT: vmovdqa32 640(%rsi), %zmm11 {%k2} {z} -; AVX512BW-NEXT: kshiftrd $16, %k2, %k2 +; AVX512BW-NEXT: kshiftrq $48, %k2, %k2 ; AVX512BW-NEXT: vmovdqa32 704(%rsi), %zmm13 {%k2} {z} -; AVX512BW-NEXT: kshiftrd $16, %k1, %k2 +; AVX512BW-NEXT: kshiftrq $16, %k1, %k2 ; AVX512BW-NEXT: vmovdqa32 832(%rsi), %zmm14 {%k2} {z} -; AVX512BW-NEXT: vpmovb2m %zmm15, %k2 -; AVX512BW-NEXT: vmovdqa32 768(%rsi), %zmm15 {%k1} {z} -; AVX512BW-NEXT: kshiftrq $32, %k1, %k1 -; AVX512BW-NEXT: vmovdqa32 896(%rsi), %zmm17 {%k1} {z} -; AVX512BW-NEXT: kshiftrd $16, %k1, %k1 +; AVX512BW-NEXT: kshiftrq $32, %k1, %k2 +; AVX512BW-NEXT: vmovdqa32 896(%rsi), %zmm17 {%k2} {z} +; AVX512BW-NEXT: vpmovb2m %zmm16, %k2 +; AVX512BW-NEXT: vmovdqa32 768(%rsi), %zmm16 {%k1} {z} +; AVX512BW-NEXT: kshiftrq $48, %k1, %k1 ; AVX512BW-NEXT: vmovdqa32 960(%rsi), %zmm18 {%k1} {z} -; AVX512BW-NEXT: kshiftrd $16, %k2, %k1 +; AVX512BW-NEXT: kshiftrq $16, %k2, %k1 ; AVX512BW-NEXT: vmovdqa32 1088(%rsi), %zmm19 {%k1} {z} -; AVX512BW-NEXT: vpmovb2m %zmm16, %k1 -; AVX512BW-NEXT: vmovdqa32 1024(%rsi), %zmm16 {%k2} {z} -; AVX512BW-NEXT: kshiftrq $32, %k2, %k2 -; AVX512BW-NEXT: vmovdqa32 1152(%rsi), %zmm20 {%k2} {z} -; AVX512BW-NEXT: kshiftrd $16, %k2, %k2 +; AVX512BW-NEXT: kshiftrq $32, %k2, %k1 +; AVX512BW-NEXT: vmovdqa32 1152(%rsi), %zmm20 {%k1} {z} +; AVX512BW-NEXT: vpmovb2m %zmm15, %k1 +; AVX512BW-NEXT: vmovdqa32 1024(%rsi), %zmm15 {%k2} {z} +; AVX512BW-NEXT: kshiftrq $48, %k2, %k2 ; AVX512BW-NEXT: vmovdqa32 1216(%rsi), %zmm21 {%k2} {z} -; AVX512BW-NEXT: kshiftrd $16, %k1, %k2 +; AVX512BW-NEXT: kshiftrq $16, %k1, %k2 ; AVX512BW-NEXT: vmovdqa32 1344(%rsi), %zmm22 {%k2} {z} +; AVX512BW-NEXT: kshiftrq $32, %k1, %k2 +; AVX512BW-NEXT: vmovdqa32 1408(%rsi), %zmm23 {%k2} {z} ; AVX512BW-NEXT: vpmovb2m %zmm12, %k2 ; AVX512BW-NEXT: vmovdqa32 1280(%rsi), %zmm12 {%k1} {z} -; AVX512BW-NEXT: kshiftrq $32, %k1, %k1 -; AVX512BW-NEXT: vmovdqa32 1408(%rsi), %zmm23 {%k1} {z} -; AVX512BW-NEXT: kshiftrd $16, %k1, %k1 +; AVX512BW-NEXT: kshiftrq $48, %k1, %k1 ; AVX512BW-NEXT: vmovdqa32 1472(%rsi), %zmm24 {%k1} {z} -; AVX512BW-NEXT: kshiftrd $16, %k2, %k1 +; AVX512BW-NEXT: kshiftrq $16, %k2, %k1 ; AVX512BW-NEXT: vmovdqa32 1600(%rsi), %zmm25 {%k1} {z} +; AVX512BW-NEXT: kshiftrq $32, %k2, %k1 +; AVX512BW-NEXT: vmovdqa32 1664(%rsi), %zmm26 {%k1} {z} ; AVX512BW-NEXT: vpmovb2m %zmm7, %k1 ; AVX512BW-NEXT: vmovdqa32 1536(%rsi), %zmm7 {%k2} {z} -; AVX512BW-NEXT: kshiftrq $32, %k2, %k2 -; AVX512BW-NEXT: vmovdqa32 1664(%rsi), %zmm26 {%k2} {z} -; AVX512BW-NEXT: kshiftrd $16, %k2, %k2 +; AVX512BW-NEXT: kshiftrq $48, %k2, %k2 ; AVX512BW-NEXT: vmovdqa32 1728(%rsi), %zmm27 {%k2} {z} -; AVX512BW-NEXT: kshiftrd $16, %k1, %k2 +; AVX512BW-NEXT: kshiftrq $16, %k1, %k2 ; AVX512BW-NEXT: vmovdqa32 1856(%rsi), %zmm28 {%k2} {z} -; AVX512BW-NEXT: vmovdqa32 1792(%rsi), %zmm29 {%k1} {z} -; AVX512BW-NEXT: kshiftrq $32, %k1, %k1 -; AVX512BW-NEXT: vmovdqa32 1920(%rsi), %zmm30 {%k1} {z} -; AVX512BW-NEXT: kshiftrd $16, %k1, %k1 +; AVX512BW-NEXT: kshiftrq $32, %k1, %k2 +; AVX512BW-NEXT: vmovdqa32 1920(%rsi), %zmm29 {%k2} {z} +; AVX512BW-NEXT: vmovdqa32 1792(%rsi), %zmm30 {%k1} {z} +; AVX512BW-NEXT: kshiftrq $48, %k1, %k1 ; AVX512BW-NEXT: vmovdqa32 1984(%rsi), %zmm31 {%k1} {z} ; AVX512BW-NEXT: vmovdqa64 %zmm31, 1984(%rdx) -; AVX512BW-NEXT: vmovdqa64 %zmm30, 1920(%rdx) +; AVX512BW-NEXT: vmovdqa64 %zmm29, 1920(%rdx) ; AVX512BW-NEXT: vmovdqa64 %zmm28, 1856(%rdx) -; AVX512BW-NEXT: vmovdqa64 %zmm29, 1792(%rdx) +; AVX512BW-NEXT: vmovdqa64 %zmm30, 1792(%rdx) ; AVX512BW-NEXT: vmovdqa64 %zmm27, 1728(%rdx) ; AVX512BW-NEXT: vmovdqa64 %zmm26, 1664(%rdx) ; AVX512BW-NEXT: vmovdqa64 %zmm25, 1600(%rdx) @@ -13769,11 +13769,11 @@ define void @mask_replication_factor8_vf64(ptr %in.maskvec, ptr %in.vec, ptr %ou ; AVX512BW-NEXT: vmovdqa64 %zmm21, 1216(%rdx) ; AVX512BW-NEXT: vmovdqa64 %zmm20, 1152(%rdx) ; AVX512BW-NEXT: vmovdqa64 %zmm19, 1088(%rdx) -; AVX512BW-NEXT: vmovdqa64 %zmm16, 1024(%rdx) +; AVX512BW-NEXT: vmovdqa64 %zmm15, 1024(%rdx) ; AVX512BW-NEXT: vmovdqa64 %zmm18, 960(%rdx) ; AVX512BW-NEXT: vmovdqa64 %zmm17, 896(%rdx) ; AVX512BW-NEXT: vmovdqa64 %zmm14, 832(%rdx) -; AVX512BW-NEXT: vmovdqa64 %zmm15, 768(%rdx) +; AVX512BW-NEXT: vmovdqa64 %zmm16, 768(%rdx) ; AVX512BW-NEXT: vmovdqa64 %zmm13, 704(%rdx) ; AVX512BW-NEXT: vmovdqa64 %zmm11, 640(%rdx) ; AVX512BW-NEXT: vmovdqa64 %zmm9, 576(%rdx) diff --git a/llvm/test/CodeGen/X86/vector-shift-ashr-128.ll b/llvm/test/CodeGen/X86/vector-shift-ashr-128.ll index b5571edfa522..54056461bff8 100644 --- a/llvm/test/CodeGen/X86/vector-shift-ashr-128.ll +++ b/llvm/test/CodeGen/X86/vector-shift-ashr-128.ll @@ -1553,7 +1553,7 @@ define <8 x i16> @constant_shift_v8i16_pairs(<8 x i16> %a) nounwind { ; AVX512DQVL: # %bb.0: ; AVX512DQVL-NEXT: vpsrlvd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 ; AVX512DQVL-NEXT: vmovdqa {{.*#+}} xmm1 = [256,256,16384,16384,4096,4096,512,512] -; AVX512DQVL-NEXT: vpternlogq $108, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm0 +; AVX512DQVL-NEXT: vpternlogq {{.*#+}} xmm0 = xmm1 ^ (xmm0 & mem) ; AVX512DQVL-NEXT: vpsubw %xmm1, %xmm0, %xmm0 ; AVX512DQVL-NEXT: retq ; @@ -1734,7 +1734,7 @@ define <16 x i8> @constant_shift_v16i8_pairs(<16 x i8> %a) nounwind { ; AVX512DQVL-NEXT: vpmulhuw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm1 # [32768,4096,512,8192,16384,u,2048,1024] ; AVX512DQVL-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0,1,2,3,4],xmm0[5],xmm1[6,7] ; AVX512DQVL-NEXT: vmovdqa {{.*#+}} xmm1 = [64,64,8,8,1,1,16,16,32,32,128,128,4,4,2,2] -; AVX512DQVL-NEXT: vpternlogq $108, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm0 +; AVX512DQVL-NEXT: vpternlogq {{.*#+}} xmm0 = xmm1 ^ (xmm0 & mem) ; AVX512DQVL-NEXT: vpsubb %xmm1, %xmm0, %xmm0 ; AVX512DQVL-NEXT: retq ; @@ -1742,7 +1742,7 @@ define <16 x i8> @constant_shift_v16i8_pairs(<16 x i8> %a) nounwind { ; AVX512BWVL: # %bb.0: ; AVX512BWVL-NEXT: vpsrlvw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 ; AVX512BWVL-NEXT: vmovdqa {{.*#+}} xmm1 = [64,64,8,8,1,1,16,16,32,32,128,128,4,4,2,2] -; AVX512BWVL-NEXT: vpternlogq $108, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm0 +; AVX512BWVL-NEXT: vpternlogq {{.*#+}} xmm0 = xmm1 ^ (xmm0 & mem) ; AVX512BWVL-NEXT: vpsubb %xmm1, %xmm0, %xmm0 ; AVX512BWVL-NEXT: retq ; @@ -1822,7 +1822,7 @@ define <16 x i8> @constant_shift_v16i8_quads(<16 x i8> %a) nounwind { ; AVX512VL: # %bb.0: ; AVX512VL-NEXT: vpsrlvd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 ; AVX512VL-NEXT: vmovdqa {{.*#+}} xmm1 = [128,128,128,128,64,64,64,64,16,16,16,16,32,32,32,32] -; AVX512VL-NEXT: vpternlogq $108, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm0 +; AVX512VL-NEXT: vpternlogq {{.*#+}} xmm0 = xmm1 ^ (xmm0 & mem) ; AVX512VL-NEXT: vpsubb %xmm1, %xmm0, %xmm0 ; AVX512VL-NEXT: retq ; @@ -2021,7 +2021,7 @@ define <16 x i8> @splatconstant_shift_v16i8(<16 x i8> %a) nounwind { ; AVX512DQVL: # %bb.0: ; AVX512DQVL-NEXT: vpsrlw $3, %xmm0, %xmm0 ; AVX512DQVL-NEXT: vpbroadcastd {{.*#+}} xmm1 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16] -; AVX512DQVL-NEXT: vpternlogd $108, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm1, %xmm0 +; AVX512DQVL-NEXT: vpternlogd {{.*#+}} xmm0 = xmm1 ^ (xmm0 & mem) ; AVX512DQVL-NEXT: vpsubb %xmm1, %xmm0, %xmm0 ; AVX512DQVL-NEXT: retq ; @@ -2029,7 +2029,7 @@ define <16 x i8> @splatconstant_shift_v16i8(<16 x i8> %a) nounwind { ; AVX512BWVL: # %bb.0: ; AVX512BWVL-NEXT: vpsrlw $3, %xmm0, %xmm0 ; AVX512BWVL-NEXT: vpbroadcastb {{.*#+}} xmm1 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16] -; AVX512BWVL-NEXT: vpternlogd $108, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm1, %xmm0 +; AVX512BWVL-NEXT: vpternlogd {{.*#+}} xmm0 = xmm1 ^ (xmm0 & mem) ; AVX512BWVL-NEXT: vpsubb %xmm1, %xmm0, %xmm0 ; AVX512BWVL-NEXT: retq ; diff --git a/llvm/test/CodeGen/X86/vector-shift-ashr-256.ll b/llvm/test/CodeGen/X86/vector-shift-ashr-256.ll index cc64b4375e45..abd81a0e9f99 100644 --- a/llvm/test/CodeGen/X86/vector-shift-ashr-256.ll +++ b/llvm/test/CodeGen/X86/vector-shift-ashr-256.ll @@ -952,7 +952,7 @@ define <32 x i8> @splatvar_shift_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind { ; AVX512DQVL-NEXT: vpsrlw %xmm1, %xmm3, %xmm1 ; AVX512DQVL-NEXT: vpsrlw $8, %xmm1, %xmm1 ; AVX512DQVL-NEXT: vpbroadcastb %xmm1, %ymm1 -; AVX512DQVL-NEXT: vpternlogq $108, %ymm0, %ymm2, %ymm1 +; AVX512DQVL-NEXT: vpternlogq {{.*#+}} ymm1 = ymm2 ^ (ymm1 & ymm0) ; AVX512DQVL-NEXT: vpsubb %ymm2, %ymm1, %ymm0 ; AVX512DQVL-NEXT: retq ; @@ -1321,7 +1321,7 @@ define <32 x i8> @splatvar_modulo_shift_v32i8(<32 x i8> %a, <32 x i8> %b) nounwi ; AVX512DQVL-NEXT: vpsrlw %xmm1, %xmm3, %xmm1 ; AVX512DQVL-NEXT: vpsrlw $8, %xmm1, %xmm1 ; AVX512DQVL-NEXT: vpbroadcastb %xmm1, %ymm1 -; AVX512DQVL-NEXT: vpternlogq $108, %ymm0, %ymm2, %ymm1 +; AVX512DQVL-NEXT: vpternlogq {{.*#+}} ymm1 = ymm2 ^ (ymm1 & ymm0) ; AVX512DQVL-NEXT: vpsubb %ymm2, %ymm1, %ymm0 ; AVX512DQVL-NEXT: retq ; @@ -1681,7 +1681,7 @@ define <16 x i16> @constant_shift_v16i16_pairs(<16 x i16> %a) nounwind { ; AVX512DQVL: # %bb.0: ; AVX512DQVL-NEXT: vpsrlvd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0 ; AVX512DQVL-NEXT: vmovdqa {{.*#+}} ymm1 = [32768,32768,16384,16384,4096,4096,8192,8192,512,512,256,256,1024,1024,2048,2048] -; AVX512DQVL-NEXT: vpternlogq $108, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm0 +; AVX512DQVL-NEXT: vpternlogq {{.*#+}} ymm0 = ymm1 ^ (ymm0 & mem) ; AVX512DQVL-NEXT: vpsubw %ymm1, %ymm0, %ymm0 ; AVX512DQVL-NEXT: retq ; @@ -1926,7 +1926,7 @@ define <32 x i8> @constant_shift_v32i8_pairs(<32 x i8> %a) nounwind { ; AVX512DQVL-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0,1,2,3,4,5,6],xmm0[7] ; AVX512DQVL-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7] ; AVX512DQVL-NEXT: vmovdqa {{.*#+}} ymm1 = [4,4,2,2,4,4,8,8,32,32,2,2,32,32,128,128,1,1,32,32,8,8,2,2,16,16,4,4,8,8,64,64] -; AVX512DQVL-NEXT: vpternlogq $108, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm0 +; AVX512DQVL-NEXT: vpternlogq {{.*#+}} ymm0 = ymm1 ^ (ymm0 & mem) ; AVX512DQVL-NEXT: vpsubb %ymm1, %ymm0, %ymm0 ; AVX512DQVL-NEXT: retq ; @@ -1934,7 +1934,7 @@ define <32 x i8> @constant_shift_v32i8_pairs(<32 x i8> %a) nounwind { ; AVX512BWVL: # %bb.0: ; AVX512BWVL-NEXT: vpsrlvw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0 ; AVX512BWVL-NEXT: vmovdqa {{.*#+}} ymm1 = [4,4,2,2,4,4,8,8,32,32,2,2,32,32,128,128,1,1,32,32,8,8,2,2,16,16,4,4,8,8,64,64] -; AVX512BWVL-NEXT: vpternlogq $108, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm0 +; AVX512BWVL-NEXT: vpternlogq {{.*#+}} ymm0 = ymm1 ^ (ymm0 & mem) ; AVX512BWVL-NEXT: vpsubb %ymm1, %ymm0, %ymm0 ; AVX512BWVL-NEXT: retq ; @@ -2025,7 +2025,7 @@ define <32 x i8> @constant_shift_v32i8_quads(<32 x i8> %a) nounwind { ; AVX512VL: # %bb.0: ; AVX512VL-NEXT: vpsrlvd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0 ; AVX512VL-NEXT: vmovdqa {{.*#+}} ymm1 = [128,128,128,128,64,64,64,64,16,16,16,16,32,32,32,32,2,2,2,2,1,1,1,1,4,4,4,4,8,8,8,8] -; AVX512VL-NEXT: vpternlogq $108, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm0 +; AVX512VL-NEXT: vpternlogq {{.*#+}} ymm0 = ymm1 ^ (ymm0 & mem) ; AVX512VL-NEXT: vpsubb %ymm1, %ymm0, %ymm0 ; AVX512VL-NEXT: retq ; @@ -2295,7 +2295,7 @@ define <32 x i8> @splatconstant_shift_v32i8(<32 x i8> %a) nounwind { ; AVX512DQVL: # %bb.0: ; AVX512DQVL-NEXT: vpsrlw $3, %ymm0, %ymm0 ; AVX512DQVL-NEXT: vpbroadcastd {{.*#+}} ymm1 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16] -; AVX512DQVL-NEXT: vpternlogd $108, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm1, %ymm0 +; AVX512DQVL-NEXT: vpternlogd {{.*#+}} ymm0 = ymm1 ^ (ymm0 & mem) ; AVX512DQVL-NEXT: vpsubb %ymm1, %ymm0, %ymm0 ; AVX512DQVL-NEXT: retq ; @@ -2303,7 +2303,7 @@ define <32 x i8> @splatconstant_shift_v32i8(<32 x i8> %a) nounwind { ; AVX512BWVL: # %bb.0: ; AVX512BWVL-NEXT: vpsrlw $3, %ymm0, %ymm0 ; AVX512BWVL-NEXT: vpbroadcastb {{.*#+}} ymm1 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16] -; AVX512BWVL-NEXT: vpternlogd $108, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm1, %ymm0 +; AVX512BWVL-NEXT: vpternlogd {{.*#+}} ymm0 = ymm1 ^ (ymm0 & mem) ; AVX512BWVL-NEXT: vpsubb %ymm1, %ymm0, %ymm0 ; AVX512BWVL-NEXT: retq ; diff --git a/llvm/test/CodeGen/X86/vector-shift-ashr-512.ll b/llvm/test/CodeGen/X86/vector-shift-ashr-512.ll index 738092a7825e..74dbee5e5d2c 100644 --- a/llvm/test/CodeGen/X86/vector-shift-ashr-512.ll +++ b/llvm/test/CodeGen/X86/vector-shift-ashr-512.ll @@ -218,7 +218,7 @@ define <64 x i8> @splatvar_shift_v64i8(<64 x i8> %a, <64 x i8> %b) nounwind { ; AVX512BW-NEXT: vpsrlw %xmm1, %xmm3, %xmm1 ; AVX512BW-NEXT: vpsrlw $8, %xmm1, %xmm1 ; AVX512BW-NEXT: vpbroadcastb %xmm1, %zmm1 -; AVX512BW-NEXT: vpternlogq $108, %zmm0, %zmm2, %zmm1 +; AVX512BW-NEXT: vpternlogq {{.*#+}} zmm1 = zmm2 ^ (zmm1 & zmm0) ; AVX512BW-NEXT: vpsubb %zmm2, %zmm1, %zmm0 ; AVX512BW-NEXT: retq %splat = shufflevector <64 x i8> %b, <64 x i8> poison, <64 x i32> zeroinitializer @@ -307,7 +307,7 @@ define <64 x i8> @splatvar_modulo_shift_v64i8(<64 x i8> %a, <64 x i8> %b) nounwi ; AVX512BW-NEXT: vpsrlw %xmm1, %xmm3, %xmm1 ; AVX512BW-NEXT: vpsrlw $8, %xmm1, %xmm1 ; AVX512BW-NEXT: vpbroadcastb %xmm1, %zmm1 -; AVX512BW-NEXT: vpternlogq $108, %zmm0, %zmm2, %zmm1 +; AVX512BW-NEXT: vpternlogq {{.*#+}} zmm1 = zmm2 ^ (zmm1 & zmm0) ; AVX512BW-NEXT: vpsubb %zmm2, %zmm1, %zmm0 ; AVX512BW-NEXT: retq %mod = and <64 x i8> %b, <i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7> @@ -365,7 +365,7 @@ define <32 x i16> @constant_shift_v32i16_pairs(<32 x i16> %a) nounwind { ; AVX512DQ: # %bb.0: ; AVX512DQ-NEXT: vpsrlvd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0 ; AVX512DQ-NEXT: vmovdqa64 {{.*#+}} zmm1 = [128,128,128,128,64,64,64,64,32,32,32,32,16,16,16,16,8,8,8,8,4,4,4,4,2,2,2,2,1,1,1,1] -; AVX512DQ-NEXT: vpternlogq $108, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm0 +; AVX512DQ-NEXT: vpternlogq {{.*#+}} zmm0 = zmm1 ^ (zmm0 & mem) ; AVX512DQ-NEXT: vpsubw %ymm1, %ymm0, %ymm1 ; AVX512DQ-NEXT: vextracti64x4 $1, %zmm0, %ymm0 ; AVX512DQ-NEXT: vpsubw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0 @@ -450,7 +450,7 @@ define <64 x i8> @constant_shift_v64i8_pairs(<64 x i8> %a) nounwind { ; AVX512BW: # %bb.0: ; AVX512BW-NEXT: vpsrlvw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0 ; AVX512BW-NEXT: vmovdqa64 {{.*#+}} zmm1 = [2,2,2,2,32,32,32,32,2,2,8,8,8,8,4,4,2,2,64,64,16,16,32,32,8,8,1,1,4,4,128,128,1,1,64,64,128,128,1,1,8,8,128,128,64,64,16,16,64,64,8,8,8,8,16,16,2,2,2,2,4,4,2,2] -; AVX512BW-NEXT: vpternlogq $108, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm0 +; AVX512BW-NEXT: vpternlogq {{.*#+}} zmm0 = zmm1 ^ (zmm0 & mem) ; AVX512BW-NEXT: vpsubb %zmm1, %zmm0, %zmm0 ; AVX512BW-NEXT: retq %shift = ashr <64 x i8> %a, <i8 6, i8 6, i8 6, i8 6, i8 2, i8 2, i8 2, i8 2, i8 6, i8 6, i8 4, i8 4, i8 4, i8 4, i8 5, i8 5, i8 6, i8 6, i8 1, i8 1, i8 3, i8 3, i8 2, i8 2, i8 4, i8 4, i8 7, i8 7, i8 5, i8 5, i8 0, i8 0, i8 7, i8 7, i8 1, i8 1, i8 0, i8 0, i8 7, i8 7, i8 4, i8 4, i8 0, i8 0, i8 1, i8 1, i8 3, i8 3, i8 1, i8 1, i8 4, i8 4, i8 4, i8 4, i8 3, i8 3, i8 6, i8 6, i8 6, i8 6, i8 5, i8 5, i8 6, i8 6> @@ -462,7 +462,7 @@ define <64 x i8> @constant_shift_v64i8_quads(<64 x i8> %a) nounwind { ; AVX512DQ: # %bb.0: ; AVX512DQ-NEXT: vpsrlvd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0 ; AVX512DQ-NEXT: vmovdqa64 {{.*#+}} zmm1 = [16,16,16,16,4,4,4,4,32,32,32,32,1,1,1,1,1,1,1,1,4,4,4,4,1,1,1,1,4,4,4,4,8,8,8,8,16,16,16,16,16,16,16,16,2,2,2,2,64,64,64,64,4,4,4,4,32,32,32,32,128,128,128,128] -; AVX512DQ-NEXT: vpternlogq $108, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm0 +; AVX512DQ-NEXT: vpternlogq {{.*#+}} zmm0 = zmm1 ^ (zmm0 & mem) ; AVX512DQ-NEXT: vpsubb %ymm1, %ymm0, %ymm1 ; AVX512DQ-NEXT: vextracti64x4 $1, %zmm0, %ymm0 ; AVX512DQ-NEXT: vpsubb {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0 @@ -473,7 +473,7 @@ define <64 x i8> @constant_shift_v64i8_quads(<64 x i8> %a) nounwind { ; AVX512BW: # %bb.0: ; AVX512BW-NEXT: vpsrlvd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0 ; AVX512BW-NEXT: vmovdqa64 {{.*#+}} zmm1 = [16,16,16,16,4,4,4,4,32,32,32,32,1,1,1,1,1,1,1,1,4,4,4,4,1,1,1,1,4,4,4,4,8,8,8,8,16,16,16,16,16,16,16,16,2,2,2,2,64,64,64,64,4,4,4,4,32,32,32,32,128,128,128,128] -; AVX512BW-NEXT: vpternlogq $108, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm0 +; AVX512BW-NEXT: vpternlogq {{.*#+}} zmm0 = zmm1 ^ (zmm0 & mem) ; AVX512BW-NEXT: vpsubb %zmm1, %zmm0, %zmm0 ; AVX512BW-NEXT: retq %shift = ashr <64 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 5, i8 5, i8 5, i8 5, i8 2, i8 2, i8 2, i8 2, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 5, i8 5, i8 5, i8 5, i8 7, i8 7, i8 7, i8 7, i8 5, i8 5, i8 5, i8 5, i8 4, i8 4, i8 4, i8 4, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 6, i8 6, i8 6, i8 6, i8 1, i8 1, i8 1, i8 1, i8 5, i8 5, i8 5, i8 5, i8 2, i8 2, i8 2, i8 2, i8 0, i8 0, i8 0, i8 0> @@ -540,7 +540,7 @@ define <64 x i8> @splatconstant_shift_v64i8(<64 x i8> %a) nounwind { ; AVX512BW: # %bb.0: ; AVX512BW-NEXT: vpsrlw $3, %zmm0, %zmm0 ; AVX512BW-NEXT: vpbroadcastb {{.*#+}} zmm1 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16] -; AVX512BW-NEXT: vpternlogd $108, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm0 +; AVX512BW-NEXT: vpternlogd {{.*#+}} zmm0 = zmm1 ^ (zmm0 & mem) ; AVX512BW-NEXT: vpsubb %zmm1, %zmm0, %zmm0 ; AVX512BW-NEXT: retq %shift = ashr <64 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3> diff --git a/llvm/test/CodeGen/X86/vector-shift-ashr-sub128.ll b/llvm/test/CodeGen/X86/vector-shift-ashr-sub128.ll index 95d8c604aace..e577388d88c2 100644 --- a/llvm/test/CodeGen/X86/vector-shift-ashr-sub128.ll +++ b/llvm/test/CodeGen/X86/vector-shift-ashr-sub128.ll @@ -2344,7 +2344,7 @@ define <8 x i8> @splatconstant_shift_v8i8(<8 x i8> %a) nounwind { ; AVX512DQVL: # %bb.0: ; AVX512DQVL-NEXT: vpsrlw $3, %xmm0, %xmm0 ; AVX512DQVL-NEXT: vpbroadcastd {{.*#+}} xmm1 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16] -; AVX512DQVL-NEXT: vpternlogd $108, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm1, %xmm0 +; AVX512DQVL-NEXT: vpternlogd {{.*#+}} xmm0 = xmm1 ^ (xmm0 & mem) ; AVX512DQVL-NEXT: vpsubb %xmm1, %xmm0, %xmm0 ; AVX512DQVL-NEXT: retq ; @@ -2352,7 +2352,7 @@ define <8 x i8> @splatconstant_shift_v8i8(<8 x i8> %a) nounwind { ; AVX512BWVL: # %bb.0: ; AVX512BWVL-NEXT: vpsrlw $3, %xmm0, %xmm0 ; AVX512BWVL-NEXT: vpbroadcastb {{.*#+}} xmm1 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16] -; AVX512BWVL-NEXT: vpternlogd $108, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm1, %xmm0 +; AVX512BWVL-NEXT: vpternlogd {{.*#+}} xmm0 = xmm1 ^ (xmm0 & mem) ; AVX512BWVL-NEXT: vpsubb %xmm1, %xmm0, %xmm0 ; AVX512BWVL-NEXT: retq ; @@ -2414,7 +2414,7 @@ define <4 x i8> @splatconstant_shift_v4i8(<4 x i8> %a) nounwind { ; AVX512DQVL: # %bb.0: ; AVX512DQVL-NEXT: vpsrlw $3, %xmm0, %xmm0 ; AVX512DQVL-NEXT: vpbroadcastd {{.*#+}} xmm1 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16] -; AVX512DQVL-NEXT: vpternlogd $108, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm1, %xmm0 +; AVX512DQVL-NEXT: vpternlogd {{.*#+}} xmm0 = xmm1 ^ (xmm0 & mem) ; AVX512DQVL-NEXT: vpsubb %xmm1, %xmm0, %xmm0 ; AVX512DQVL-NEXT: retq ; @@ -2422,7 +2422,7 @@ define <4 x i8> @splatconstant_shift_v4i8(<4 x i8> %a) nounwind { ; AVX512BWVL: # %bb.0: ; AVX512BWVL-NEXT: vpsrlw $3, %xmm0, %xmm0 ; AVX512BWVL-NEXT: vpbroadcastb {{.*#+}} xmm1 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16] -; AVX512BWVL-NEXT: vpternlogd $108, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm1, %xmm0 +; AVX512BWVL-NEXT: vpternlogd {{.*#+}} xmm0 = xmm1 ^ (xmm0 & mem) ; AVX512BWVL-NEXT: vpsubb %xmm1, %xmm0, %xmm0 ; AVX512BWVL-NEXT: retq ; @@ -2484,7 +2484,7 @@ define <2 x i8> @splatconstant_shift_v2i8(<2 x i8> %a) nounwind { ; AVX512DQVL: # %bb.0: ; AVX512DQVL-NEXT: vpsrlw $3, %xmm0, %xmm0 ; AVX512DQVL-NEXT: vpbroadcastd {{.*#+}} xmm1 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16] -; AVX512DQVL-NEXT: vpternlogd $108, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm1, %xmm0 +; AVX512DQVL-NEXT: vpternlogd {{.*#+}} xmm0 = xmm1 ^ (xmm0 & mem) ; AVX512DQVL-NEXT: vpsubb %xmm1, %xmm0, %xmm0 ; AVX512DQVL-NEXT: retq ; @@ -2492,7 +2492,7 @@ define <2 x i8> @splatconstant_shift_v2i8(<2 x i8> %a) nounwind { ; AVX512BWVL: # %bb.0: ; AVX512BWVL-NEXT: vpsrlw $3, %xmm0, %xmm0 ; AVX512BWVL-NEXT: vpbroadcastb {{.*#+}} xmm1 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16] -; AVX512BWVL-NEXT: vpternlogd $108, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm1, %xmm0 +; AVX512BWVL-NEXT: vpternlogd {{.*#+}} xmm0 = xmm1 ^ (xmm0 & mem) ; AVX512BWVL-NEXT: vpsubb %xmm1, %xmm0, %xmm0 ; AVX512BWVL-NEXT: retq ; diff --git a/llvm/test/CodeGen/X86/vector-shift-shl-256.ll b/llvm/test/CodeGen/X86/vector-shift-shl-256.ll index 3c6aa642c778..c80f24ad5777 100644 --- a/llvm/test/CodeGen/X86/vector-shift-shl-256.ll +++ b/llvm/test/CodeGen/X86/vector-shift-shl-256.ll @@ -1371,7 +1371,7 @@ define <32 x i8> @constant_shift_v32i8(<32 x i8> %a) nounwind { ; AVX512DQVL-NEXT: vpmaddubsw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm1 # [1,0,4,0,16,0,64,0,128,0,32,0,8,0,2,0,1,0,4,0,16,0,64,0,128,0,32,0,8,0,2,0] ; AVX512DQVL-NEXT: vpmaddubsw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0 # [0,2,0,8,0,32,0,128,0,64,0,16,0,4,0,1,0,2,0,8,0,32,0,128,0,64,0,16,0,4,0,1] ; AVX512DQVL-NEXT: vpsllw $8, %ymm0, %ymm0 -; AVX512DQVL-NEXT: vpternlogd $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm1, %ymm0 +; AVX512DQVL-NEXT: vpternlogd {{.*#+}} ymm0 = ymm0 | (ymm1 & mem) ; AVX512DQVL-NEXT: retq ; ; AVX512BWVL-LABEL: constant_shift_v32i8: diff --git a/llvm/test/CodeGen/X86/vector-shift-shl-512.ll b/llvm/test/CodeGen/X86/vector-shift-shl-512.ll index a7fb05fbc5ba..a42056be895e 100644 --- a/llvm/test/CodeGen/X86/vector-shift-shl-512.ll +++ b/llvm/test/CodeGen/X86/vector-shift-shl-512.ll @@ -324,7 +324,7 @@ define <64 x i8> @constant_shift_v64i8(<64 x i8> %a) nounwind { ; AVX512DQ-NEXT: vpmaddubsw %ymm3, %ymm1, %ymm1 ; AVX512DQ-NEXT: vpsllw $8, %ymm1, %ymm1 ; AVX512DQ-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0 -; AVX512DQ-NEXT: vpternlogd $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm2, %zmm0 +; AVX512DQ-NEXT: vpternlogd {{.*#+}} zmm0 = zmm0 | (zmm2 & mem) ; AVX512DQ-NEXT: retq ; ; AVX512BW-LABEL: constant_shift_v64i8: @@ -332,7 +332,7 @@ define <64 x i8> @constant_shift_v64i8(<64 x i8> %a) nounwind { ; AVX512BW-NEXT: vpmaddubsw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm1 # [1,0,4,0,16,0,64,0,128,0,32,0,8,0,2,0,1,0,4,0,16,0,64,0,128,0,32,0,8,0,2,0,1,0,4,0,16,0,64,0,128,0,32,0,8,0,2,0,1,0,4,0,16,0,64,0,128,0,32,0,8,0,2,0] ; AVX512BW-NEXT: vpmaddubsw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0 # [0,2,0,8,0,32,0,128,0,64,0,16,0,4,0,1,0,2,0,8,0,32,0,128,0,64,0,16,0,4,0,1,0,2,0,8,0,32,0,128,0,64,0,16,0,4,0,1,0,2,0,8,0,32,0,128,0,64,0,16,0,4,0,1] ; AVX512BW-NEXT: vpsllw $8, %zmm0, %zmm0 -; AVX512BW-NEXT: vpternlogd $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm0 +; AVX512BW-NEXT: vpternlogd {{.*#+}} zmm0 = zmm0 | (zmm1 & mem) ; AVX512BW-NEXT: retq %shift = shl <64 x i8> %a, <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0> ret <64 x i8> %shift diff --git a/llvm/test/CodeGen/X86/vector-shuffle-512-v64.ll b/llvm/test/CodeGen/X86/vector-shuffle-512-v64.ll index 64ee78e4620e..62327fb2618c 100644 --- a/llvm/test/CodeGen/X86/vector-shuffle-512-v64.ll +++ b/llvm/test/CodeGen/X86/vector-shuffle-512-v64.ll @@ -1154,8 +1154,8 @@ define <64 x i8> @shuffle_v64i8_62_63_64_65_66_67_68_69_70_71_72_73_74_75_76_77_ ; ; AVX512BW-LABEL: shuffle_v64i8_62_63_64_65_66_67_68_69_70_71_72_73_74_75_76_77_78_79_80_81_82_83_84_85_86_87_88_89_90_91_92_93_94_95_96_97_98_99_100_101_102_103_104_105_106_107_108_109_110_111_112_113_114_115_116_117_118_119_120_121_122_123_124_125: ; AVX512BW: # %bb.0: -; AVX512BW-NEXT: vpmovsxbw {{.*#+}} zmm2 = [63,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30] -; AVX512BW-NEXT: vpermt2w %zmm1, %zmm2, %zmm0 +; AVX512BW-NEXT: valignq {{.*#+}} zmm1 = zmm1[6,7],zmm0[0,1,2,3,4,5] +; AVX512BW-NEXT: vpalignr {{.*#+}} zmm0 = zmm1[14,15],zmm0[0,1,2,3,4,5,6,7,8,9,10,11,12,13],zmm1[30,31],zmm0[16,17,18,19,20,21,22,23,24,25,26,27,28,29],zmm1[46,47],zmm0[32,33,34,35,36,37,38,39,40,41,42,43,44,45],zmm1[62,63],zmm0[48,49,50,51,52,53,54,55,56,57,58,59,60,61] ; AVX512BW-NEXT: retq ; ; AVX512DQ-LABEL: shuffle_v64i8_62_63_64_65_66_67_68_69_70_71_72_73_74_75_76_77_78_79_80_81_82_83_84_85_86_87_88_89_90_91_92_93_94_95_96_97_98_99_100_101_102_103_104_105_106_107_108_109_110_111_112_113_114_115_116_117_118_119_120_121_122_123_124_125: @@ -1171,8 +1171,8 @@ define <64 x i8> @shuffle_v64i8_62_63_64_65_66_67_68_69_70_71_72_73_74_75_76_77_ ; ; AVX512VBMI-LABEL: shuffle_v64i8_62_63_64_65_66_67_68_69_70_71_72_73_74_75_76_77_78_79_80_81_82_83_84_85_86_87_88_89_90_91_92_93_94_95_96_97_98_99_100_101_102_103_104_105_106_107_108_109_110_111_112_113_114_115_116_117_118_119_120_121_122_123_124_125: ; AVX512VBMI: # %bb.0: -; AVX512VBMI-NEXT: vpmovsxbw {{.*#+}} zmm2 = [63,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30] -; AVX512VBMI-NEXT: vpermt2w %zmm1, %zmm2, %zmm0 +; AVX512VBMI-NEXT: valignq {{.*#+}} zmm1 = zmm1[6,7],zmm0[0,1,2,3,4,5] +; AVX512VBMI-NEXT: vpalignr {{.*#+}} zmm0 = zmm1[14,15],zmm0[0,1,2,3,4,5,6,7,8,9,10,11,12,13],zmm1[30,31],zmm0[16,17,18,19,20,21,22,23,24,25,26,27,28,29],zmm1[46,47],zmm0[32,33,34,35,36,37,38,39,40,41,42,43,44,45],zmm1[62,63],zmm0[48,49,50,51,52,53,54,55,56,57,58,59,60,61] ; AVX512VBMI-NEXT: retq %r = shufflevector <64 x i8> %a1, <64 x i8> %a0, <64 x i32> <i32 62, i32 63, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 104, i32 105, i32 106, i32 107, i32 108, i32 109, i32 110, i32 111, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122, i32 123, i32 124, i32 125> ret <64 x i8> %r diff --git a/llvm/test/CodeGen/X86/vector-shuffle-combining-avx512bw.ll b/llvm/test/CodeGen/X86/vector-shuffle-combining-avx512bw.ll index b4375cfb343b..376fe37ef1fa 100644 --- a/llvm/test/CodeGen/X86/vector-shuffle-combining-avx512bw.ll +++ b/llvm/test/CodeGen/X86/vector-shuffle-combining-avx512bw.ll @@ -24,8 +24,7 @@ define <32 x i16> @combine_vpermt2var_32i16_identity_mask(<32 x i16> %x0, <32 x ; X86-NEXT: vpmovsxbw {{.*#+}} zmm1 = [31,30,29,28,27,26,25,24,23,22,21,20,19,18,17,16,15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0] ; X86-NEXT: kmovd {{[0-9]+}}(%esp), %k1 ; X86-NEXT: vpermt2w %zmm0, %zmm1, %zmm0 {%k1} {z} -; X86-NEXT: vpmovsxbw {{.*#+}} zmm1 = [63,30,61,28,59,26,57,24,55,22,53,20,51,18,49,16,47,46,13,44,11,42,9,40,7,38,5,36,3,34,1,32] -; X86-NEXT: vpermt2w %zmm0, %zmm1, %zmm0 {%k1} {z} +; X86-NEXT: vpermw %zmm0, %zmm1, %zmm0 {%k1} {z} ; X86-NEXT: retl ; ; X64-LABEL: combine_vpermt2var_32i16_identity_mask: @@ -33,8 +32,7 @@ define <32 x i16> @combine_vpermt2var_32i16_identity_mask(<32 x i16> %x0, <32 x ; X64-NEXT: vpmovsxbw {{.*#+}} zmm1 = [31,30,29,28,27,26,25,24,23,22,21,20,19,18,17,16,15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0] ; X64-NEXT: kmovd %edi, %k1 ; X64-NEXT: vpermt2w %zmm0, %zmm1, %zmm0 {%k1} {z} -; X64-NEXT: vpmovsxbw {{.*#+}} zmm1 = [63,30,61,28,59,26,57,24,55,22,53,20,51,18,49,16,47,46,13,44,11,42,9,40,7,38,5,36,3,34,1,32] -; X64-NEXT: vpermt2w %zmm0, %zmm1, %zmm0 {%k1} {z} +; X64-NEXT: vpermw %zmm0, %zmm1, %zmm0 {%k1} {z} ; X64-NEXT: retq %res0 = call <32 x i16> @llvm.x86.avx512.maskz.vpermt2var.hi.512(<32 x i16> <i16 31, i16 30, i16 29, i16 28, i16 27, i16 26, i16 25, i16 24, i16 23, i16 22, i16 21, i16 20, i16 19, i16 18, i16 17, i16 16, i16 15, i16 14, i16 13, i16 12, i16 11, i16 10, i16 9, i16 8, i16 7, i16 6, i16 5, i16 4, i16 3, i16 2, i16 1, i16 0>, <32 x i16> %x0, <32 x i16> %x1, i32 %m) %res1 = call <32 x i16> @llvm.x86.avx512.maskz.vpermt2var.hi.512(<32 x i16> <i16 63, i16 30, i16 61, i16 28, i16 59, i16 26, i16 57, i16 24, i16 55, i16 22, i16 53, i16 20, i16 51, i16 18, i16 49, i16 16, i16 47, i16 46, i16 13, i16 44, i16 11, i16 42, i16 9, i16 40, i16 7, i16 38, i16 5, i16 36, i16 3, i16 34, i16 1, i16 32>, <32 x i16> %res0, <32 x i16> %res0, i32 %m) diff --git a/llvm/test/CodeGen/X86/vector-shuffle-combining-avx512bwvl.ll b/llvm/test/CodeGen/X86/vector-shuffle-combining-avx512bwvl.ll index 388511ce0741..f0b70ae26b1f 100644 --- a/llvm/test/CodeGen/X86/vector-shuffle-combining-avx512bwvl.ll +++ b/llvm/test/CodeGen/X86/vector-shuffle-combining-avx512bwvl.ll @@ -19,8 +19,7 @@ define <16 x i16> @combine_vpermt2var_16i16_identity_mask(<16 x i16> %x0, <16 x ; X86-NEXT: vpmovsxbw {{.*#+}} ymm1 = [15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0] ; X86-NEXT: kmovw {{[0-9]+}}(%esp), %k1 ; X86-NEXT: vpermt2w %ymm0, %ymm1, %ymm0 {%k1} {z} -; X86-NEXT: vpmovsxbw {{.*#+}} ymm1 = [15,30,13,28,11,26,9,24,7,22,5,20,3,18,1,16] -; X86-NEXT: vpermt2w %ymm0, %ymm1, %ymm0 {%k1} {z} +; X86-NEXT: vpermw %ymm0, %ymm1, %ymm0 {%k1} {z} ; X86-NEXT: retl ; ; X64-LABEL: combine_vpermt2var_16i16_identity_mask: @@ -28,8 +27,7 @@ define <16 x i16> @combine_vpermt2var_16i16_identity_mask(<16 x i16> %x0, <16 x ; X64-NEXT: vpmovsxbw {{.*#+}} ymm1 = [15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0] ; X64-NEXT: kmovd %edi, %k1 ; X64-NEXT: vpermt2w %ymm0, %ymm1, %ymm0 {%k1} {z} -; X64-NEXT: vpmovsxbw {{.*#+}} ymm1 = [15,30,13,28,11,26,9,24,7,22,5,20,3,18,1,16] -; X64-NEXT: vpermt2w %ymm0, %ymm1, %ymm0 {%k1} {z} +; X64-NEXT: vpermw %ymm0, %ymm1, %ymm0 {%k1} {z} ; X64-NEXT: retq %res0 = call <16 x i16> @llvm.x86.avx512.maskz.vpermt2var.hi.256(<16 x i16> <i16 15, i16 14, i16 13, i16 12, i16 11, i16 10, i16 9, i16 8, i16 7, i16 6, i16 5, i16 4, i16 3, i16 2, i16 1, i16 0>, <16 x i16> %x0, <16 x i16> %x1, i16 %m) %res1 = call <16 x i16> @llvm.x86.avx512.maskz.vpermt2var.hi.256(<16 x i16> <i16 15, i16 30, i16 13, i16 28, i16 11, i16 26, i16 9, i16 24, i16 7, i16 22, i16 5, i16 20, i16 3, i16 18, i16 1, i16 16>, <16 x i16> %res0, <16 x i16> %res0, i16 %m) diff --git a/llvm/test/CodeGen/X86/vector-shuffle-combining-avx512f.ll b/llvm/test/CodeGen/X86/vector-shuffle-combining-avx512f.ll index e87e810971e1..29806cd25fe3 100644 --- a/llvm/test/CodeGen/X86/vector-shuffle-combining-avx512f.ll +++ b/llvm/test/CodeGen/X86/vector-shuffle-combining-avx512f.ll @@ -155,8 +155,7 @@ define <8 x double> @combine_vpermt2var_8f64_identity_mask(<8 x double> %x0, <8 ; X86-AVX512F-NEXT: movzbl {{[0-9]+}}(%esp), %eax ; X86-AVX512F-NEXT: kmovw %eax, %k1 ; X86-AVX512F-NEXT: vpermt2pd %zmm0, %zmm1, %zmm0 {%k1} {z} -; X86-AVX512F-NEXT: vmovapd {{.*#+}} zmm1 = [7,0,14,0,5,0,12,0,3,0,10,0,1,0,8,0] -; X86-AVX512F-NEXT: vpermt2pd %zmm0, %zmm1, %zmm0 {%k1} {z} +; X86-AVX512F-NEXT: vpermpd %zmm0, %zmm1, %zmm0 {%k1} {z} ; X86-AVX512F-NEXT: retl ; ; X86-AVX512BW-LABEL: combine_vpermt2var_8f64_identity_mask: @@ -165,8 +164,7 @@ define <8 x double> @combine_vpermt2var_8f64_identity_mask(<8 x double> %x0, <8 ; X86-AVX512BW-NEXT: movzbl {{[0-9]+}}(%esp), %eax ; X86-AVX512BW-NEXT: kmovd %eax, %k1 ; X86-AVX512BW-NEXT: vpermt2pd %zmm0, %zmm1, %zmm0 {%k1} {z} -; X86-AVX512BW-NEXT: vmovapd {{.*#+}} zmm1 = [7,0,14,0,5,0,12,0,3,0,10,0,1,0,8,0] -; X86-AVX512BW-NEXT: vpermt2pd %zmm0, %zmm1, %zmm0 {%k1} {z} +; X86-AVX512BW-NEXT: vpermpd %zmm0, %zmm1, %zmm0 {%k1} {z} ; X86-AVX512BW-NEXT: retl ; ; X64-AVX512F-LABEL: combine_vpermt2var_8f64_identity_mask: @@ -174,8 +172,7 @@ define <8 x double> @combine_vpermt2var_8f64_identity_mask(<8 x double> %x0, <8 ; X64-AVX512F-NEXT: vmovapd {{.*#+}} zmm1 = [7,6,5,4,3,2,1,0] ; X64-AVX512F-NEXT: kmovw %edi, %k1 ; X64-AVX512F-NEXT: vpermt2pd %zmm0, %zmm1, %zmm0 {%k1} {z} -; X64-AVX512F-NEXT: vmovapd {{.*#+}} zmm1 = [7,14,5,12,3,10,1,8] -; X64-AVX512F-NEXT: vpermt2pd %zmm0, %zmm1, %zmm0 {%k1} {z} +; X64-AVX512F-NEXT: vpermpd %zmm0, %zmm1, %zmm0 {%k1} {z} ; X64-AVX512F-NEXT: retq ; ; X64-AVX512BW-LABEL: combine_vpermt2var_8f64_identity_mask: @@ -183,8 +180,7 @@ define <8 x double> @combine_vpermt2var_8f64_identity_mask(<8 x double> %x0, <8 ; X64-AVX512BW-NEXT: vmovapd {{.*#+}} zmm1 = [7,6,5,4,3,2,1,0] ; X64-AVX512BW-NEXT: kmovd %edi, %k1 ; X64-AVX512BW-NEXT: vpermt2pd %zmm0, %zmm1, %zmm0 {%k1} {z} -; X64-AVX512BW-NEXT: vmovapd {{.*#+}} zmm1 = [7,14,5,12,3,10,1,8] -; X64-AVX512BW-NEXT: vpermt2pd %zmm0, %zmm1, %zmm0 {%k1} {z} +; X64-AVX512BW-NEXT: vpermpd %zmm0, %zmm1, %zmm0 {%k1} {z} ; X64-AVX512BW-NEXT: retq %res0 = call <8 x double> @llvm.x86.avx512.maskz.vpermt2var.pd.512(<8 x i64> <i64 7, i64 6, i64 5, i64 4, i64 3, i64 2, i64 1, i64 0>, <8 x double> %x0, <8 x double> %x1, i8 %m) %res1 = call <8 x double> @llvm.x86.avx512.maskz.vpermt2var.pd.512(<8 x i64> <i64 7, i64 14, i64 5, i64 12, i64 3, i64 10, i64 1, i64 8>, <8 x double> %res0, <8 x double> %res0, i8 %m) @@ -259,8 +255,7 @@ define <8 x i64> @combine_vpermt2var_8i64_identity_mask(<8 x i64> %x0, <8 x i64> ; X86-AVX512F-NEXT: movzbl {{[0-9]+}}(%esp), %eax ; X86-AVX512F-NEXT: kmovw %eax, %k1 ; X86-AVX512F-NEXT: vpermt2q %zmm0, %zmm1, %zmm0 {%k1} {z} -; X86-AVX512F-NEXT: vpmovsxbq {{.*#+}} zmm1 = [7,14,5,12,3,10,1,8] -; X86-AVX512F-NEXT: vpermt2q %zmm0, %zmm1, %zmm0 {%k1} {z} +; X86-AVX512F-NEXT: vpermq %zmm0, %zmm1, %zmm0 {%k1} {z} ; X86-AVX512F-NEXT: retl ; ; X86-AVX512BW-LABEL: combine_vpermt2var_8i64_identity_mask: @@ -269,8 +264,7 @@ define <8 x i64> @combine_vpermt2var_8i64_identity_mask(<8 x i64> %x0, <8 x i64> ; X86-AVX512BW-NEXT: movzbl {{[0-9]+}}(%esp), %eax ; X86-AVX512BW-NEXT: kmovd %eax, %k1 ; X86-AVX512BW-NEXT: vpermt2q %zmm0, %zmm1, %zmm0 {%k1} {z} -; X86-AVX512BW-NEXT: vpmovsxbq {{.*#+}} zmm1 = [7,14,5,12,3,10,1,8] -; X86-AVX512BW-NEXT: vpermt2q %zmm0, %zmm1, %zmm0 {%k1} {z} +; X86-AVX512BW-NEXT: vpermq %zmm0, %zmm1, %zmm0 {%k1} {z} ; X86-AVX512BW-NEXT: retl ; ; X64-AVX512F-LABEL: combine_vpermt2var_8i64_identity_mask: @@ -278,8 +272,7 @@ define <8 x i64> @combine_vpermt2var_8i64_identity_mask(<8 x i64> %x0, <8 x i64> ; X64-AVX512F-NEXT: vpmovsxbq {{.*#+}} zmm1 = [7,6,5,4,3,2,1,0] ; X64-AVX512F-NEXT: kmovw %edi, %k1 ; X64-AVX512F-NEXT: vpermt2q %zmm0, %zmm1, %zmm0 {%k1} {z} -; X64-AVX512F-NEXT: vpmovsxbq {{.*#+}} zmm1 = [7,14,5,12,3,10,1,8] -; X64-AVX512F-NEXT: vpermt2q %zmm0, %zmm1, %zmm0 {%k1} {z} +; X64-AVX512F-NEXT: vpermq %zmm0, %zmm1, %zmm0 {%k1} {z} ; X64-AVX512F-NEXT: retq ; ; X64-AVX512BW-LABEL: combine_vpermt2var_8i64_identity_mask: @@ -287,8 +280,7 @@ define <8 x i64> @combine_vpermt2var_8i64_identity_mask(<8 x i64> %x0, <8 x i64> ; X64-AVX512BW-NEXT: vpmovsxbq {{.*#+}} zmm1 = [7,6,5,4,3,2,1,0] ; X64-AVX512BW-NEXT: kmovd %edi, %k1 ; X64-AVX512BW-NEXT: vpermt2q %zmm0, %zmm1, %zmm0 {%k1} {z} -; X64-AVX512BW-NEXT: vpmovsxbq {{.*#+}} zmm1 = [7,14,5,12,3,10,1,8] -; X64-AVX512BW-NEXT: vpermt2q %zmm0, %zmm1, %zmm0 {%k1} {z} +; X64-AVX512BW-NEXT: vpermq %zmm0, %zmm1, %zmm0 {%k1} {z} ; X64-AVX512BW-NEXT: retq %res0 = call <8 x i64> @llvm.x86.avx512.maskz.vpermt2var.q.512(<8 x i64> <i64 7, i64 6, i64 5, i64 4, i64 3, i64 2, i64 1, i64 0>, <8 x i64> %x0, <8 x i64> %x1, i8 %m) %res1 = call <8 x i64> @llvm.x86.avx512.maskz.vpermt2var.q.512(<8 x i64> <i64 7, i64 14, i64 5, i64 12, i64 3, i64 10, i64 1, i64 8>, <8 x i64> %res0, <8 x i64> %res0, i8 %m) @@ -309,8 +301,7 @@ define <16 x float> @combine_vpermt2var_16f32_identity_mask(<16 x float> %x0, <1 ; X86-NEXT: vmovaps {{.*#+}} zmm1 = [15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0] ; X86-NEXT: kmovw {{[0-9]+}}(%esp), %k1 ; X86-NEXT: vpermt2ps %zmm0, %zmm1, %zmm0 {%k1} {z} -; X86-NEXT: vmovaps {{.*#+}} zmm1 = [15,30,13,28,11,26,9,24,7,22,5,20,3,18,1,16] -; X86-NEXT: vpermt2ps %zmm0, %zmm1, %zmm0 {%k1} {z} +; X86-NEXT: vpermps %zmm0, %zmm1, %zmm0 {%k1} {z} ; X86-NEXT: retl ; ; X64-AVX512F-LABEL: combine_vpermt2var_16f32_identity_mask: @@ -318,8 +309,7 @@ define <16 x float> @combine_vpermt2var_16f32_identity_mask(<16 x float> %x0, <1 ; X64-AVX512F-NEXT: vmovaps {{.*#+}} zmm1 = [15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0] ; X64-AVX512F-NEXT: kmovw %edi, %k1 ; X64-AVX512F-NEXT: vpermt2ps %zmm0, %zmm1, %zmm0 {%k1} {z} -; X64-AVX512F-NEXT: vmovaps {{.*#+}} zmm1 = [15,30,13,28,11,26,9,24,7,22,5,20,3,18,1,16] -; X64-AVX512F-NEXT: vpermt2ps %zmm0, %zmm1, %zmm0 {%k1} {z} +; X64-AVX512F-NEXT: vpermps %zmm0, %zmm1, %zmm0 {%k1} {z} ; X64-AVX512F-NEXT: retq ; ; X64-AVX512BW-LABEL: combine_vpermt2var_16f32_identity_mask: @@ -327,8 +317,7 @@ define <16 x float> @combine_vpermt2var_16f32_identity_mask(<16 x float> %x0, <1 ; X64-AVX512BW-NEXT: vmovaps {{.*#+}} zmm1 = [15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0] ; X64-AVX512BW-NEXT: kmovd %edi, %k1 ; X64-AVX512BW-NEXT: vpermt2ps %zmm0, %zmm1, %zmm0 {%k1} {z} -; X64-AVX512BW-NEXT: vmovaps {{.*#+}} zmm1 = [15,30,13,28,11,26,9,24,7,22,5,20,3,18,1,16] -; X64-AVX512BW-NEXT: vpermt2ps %zmm0, %zmm1, %zmm0 {%k1} {z} +; X64-AVX512BW-NEXT: vpermps %zmm0, %zmm1, %zmm0 {%k1} {z} ; X64-AVX512BW-NEXT: retq %res0 = call <16 x float> @llvm.x86.avx512.maskz.vpermt2var.ps.512(<16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>, <16 x float> %x0, <16 x float> %x1, i16 %m) %res1 = call <16 x float> @llvm.x86.avx512.maskz.vpermt2var.ps.512(<16 x i32> <i32 15, i32 30, i32 13, i32 28, i32 11, i32 26, i32 9, i32 24, i32 7, i32 22, i32 5, i32 20, i32 3, i32 18, i32 1, i32 16>, <16 x float> %res0, <16 x float> %res0, i16 %m) @@ -598,8 +587,7 @@ define <16 x i32> @combine_vpermt2var_16i32_identity_mask(<16 x i32> %x0, <16 x ; X86-NEXT: vpmovsxbd {{.*#+}} zmm1 = [15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0] ; X86-NEXT: kmovw {{[0-9]+}}(%esp), %k1 ; X86-NEXT: vpermt2d %zmm0, %zmm1, %zmm0 {%k1} {z} -; X86-NEXT: vpmovsxbd {{.*#+}} zmm1 = [15,30,13,28,11,26,9,24,7,22,5,20,3,18,1,16] -; X86-NEXT: vpermt2d %zmm0, %zmm1, %zmm0 {%k1} {z} +; X86-NEXT: vpermd %zmm0, %zmm1, %zmm0 {%k1} {z} ; X86-NEXT: retl ; ; X64-AVX512F-LABEL: combine_vpermt2var_16i32_identity_mask: @@ -607,8 +595,7 @@ define <16 x i32> @combine_vpermt2var_16i32_identity_mask(<16 x i32> %x0, <16 x ; X64-AVX512F-NEXT: vpmovsxbd {{.*#+}} zmm1 = [15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0] ; X64-AVX512F-NEXT: kmovw %edi, %k1 ; X64-AVX512F-NEXT: vpermt2d %zmm0, %zmm1, %zmm0 {%k1} {z} -; X64-AVX512F-NEXT: vpmovsxbd {{.*#+}} zmm1 = [15,30,13,28,11,26,9,24,7,22,5,20,3,18,1,16] -; X64-AVX512F-NEXT: vpermt2d %zmm0, %zmm1, %zmm0 {%k1} {z} +; X64-AVX512F-NEXT: vpermd %zmm0, %zmm1, %zmm0 {%k1} {z} ; X64-AVX512F-NEXT: retq ; ; X64-AVX512BW-LABEL: combine_vpermt2var_16i32_identity_mask: @@ -616,8 +603,7 @@ define <16 x i32> @combine_vpermt2var_16i32_identity_mask(<16 x i32> %x0, <16 x ; X64-AVX512BW-NEXT: vpmovsxbd {{.*#+}} zmm1 = [15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0] ; X64-AVX512BW-NEXT: kmovd %edi, %k1 ; X64-AVX512BW-NEXT: vpermt2d %zmm0, %zmm1, %zmm0 {%k1} {z} -; X64-AVX512BW-NEXT: vpmovsxbd {{.*#+}} zmm1 = [15,30,13,28,11,26,9,24,7,22,5,20,3,18,1,16] -; X64-AVX512BW-NEXT: vpermt2d %zmm0, %zmm1, %zmm0 {%k1} {z} +; X64-AVX512BW-NEXT: vpermd %zmm0, %zmm1, %zmm0 {%k1} {z} ; X64-AVX512BW-NEXT: retq %res0 = call <16 x i32> @llvm.x86.avx512.maskz.vpermt2var.d.512(<16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>, <16 x i32> %x0, <16 x i32> %x1, i16 %m) %res1 = call <16 x i32> @llvm.x86.avx512.maskz.vpermt2var.d.512(<16 x i32> <i32 15, i32 30, i32 13, i32 28, i32 11, i32 26, i32 9, i32 24, i32 7, i32 22, i32 5, i32 20, i32 3, i32 18, i32 1, i32 16>, <16 x i32> %res0, <16 x i32> %res0, i16 %m) diff --git a/llvm/test/CodeGen/X86/vector-shuffle-combining-avx512vbmi.ll b/llvm/test/CodeGen/X86/vector-shuffle-combining-avx512vbmi.ll index 04cfa3cedd58..ccc78823623d 100644 --- a/llvm/test/CodeGen/X86/vector-shuffle-combining-avx512vbmi.ll +++ b/llvm/test/CodeGen/X86/vector-shuffle-combining-avx512vbmi.ll @@ -32,8 +32,7 @@ define <16 x i8> @combine_vpermt2var_16i8_identity_mask(<16 x i8> %x0, <16 x i8> ; X86-NEXT: vmovdqa {{.*#+}} xmm1 = [15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0] ; X86-NEXT: kmovw {{[0-9]+}}(%esp), %k1 ; X86-NEXT: vpermt2b %xmm0, %xmm1, %xmm0 {%k1} {z} -; X86-NEXT: vmovdqa {{.*#+}} xmm1 = [15,30,13,28,11,26,9,24,7,22,5,20,3,18,1,16] -; X86-NEXT: vpermt2b %xmm0, %xmm1, %xmm0 {%k1} {z} +; X86-NEXT: vpermb %xmm0, %xmm1, %xmm0 {%k1} {z} ; X86-NEXT: retl ; ; X64-LABEL: combine_vpermt2var_16i8_identity_mask: @@ -41,8 +40,7 @@ define <16 x i8> @combine_vpermt2var_16i8_identity_mask(<16 x i8> %x0, <16 x i8> ; X64-NEXT: vmovdqa {{.*#+}} xmm1 = [15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0] ; X64-NEXT: kmovd %edi, %k1 ; X64-NEXT: vpermt2b %xmm0, %xmm1, %xmm0 {%k1} {z} -; X64-NEXT: vmovdqa {{.*#+}} xmm1 = [15,30,13,28,11,26,9,24,7,22,5,20,3,18,1,16] -; X64-NEXT: vpermt2b %xmm0, %xmm1, %xmm0 {%k1} {z} +; X64-NEXT: vpermb %xmm0, %xmm1, %xmm0 {%k1} {z} ; X64-NEXT: retq %res0 = call <16 x i8> @llvm.x86.avx512.maskz.vpermt2var.qi.128(<16 x i8> <i8 15, i8 14, i8 13, i8 12, i8 11, i8 10, i8 9, i8 8, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0>, <16 x i8> %x0, <16 x i8> %x1, i16 %m) %res1 = call <16 x i8> @llvm.x86.avx512.maskz.vpermt2var.qi.128(<16 x i8> <i8 15, i8 30, i8 13, i8 28, i8 11, i8 26, i8 9, i8 24, i8 7, i8 22, i8 5, i8 20, i8 3, i8 18, i8 1, i8 16>, <16 x i8> %res0, <16 x i8> %res0, i16 %m) diff --git a/llvm/test/CodeGen/X86/vpdpwssd.ll b/llvm/test/CodeGen/X86/vpdpwssd.ll index 3c1eb92e9e3c..2ac2b48af4ce 100644 --- a/llvm/test/CodeGen/X86/vpdpwssd.ll +++ b/llvm/test/CodeGen/X86/vpdpwssd.ll @@ -1,7 +1,8 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=znver4 | FileCheck %s -; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=znver5 | FileCheck %s -; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512vnni,+fast-dpwssd | FileCheck %s +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=znver4 | FileCheck %s --check-prefixes=CHECK,AVX512VL-VNNI +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=znver5 | FileCheck %s --check-prefixes=CHECK,AVX-VNNI +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512vnni,+fast-dpwssd | FileCheck %s --check-prefixes=CHECK,AVX512-VNNI +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512vnni,+avx512vl,+fast-dpwssd | FileCheck %s --check-prefixes=CHECK,AVX512VL-VNNI define <16 x i32> @vpdpwssd_test(<16 x i32> %0, <16 x i32> %1, <16 x i32> %2) { ; CHECK-LABEL: vpdpwssd_test: @@ -11,3 +12,75 @@ define <16 x i32> @vpdpwssd_test(<16 x i32> %0, <16 x i32> %1, <16 x i32> %2) { %4 = tail call <16 x i32> @llvm.x86.avx512.vpdpwssd.512(<16 x i32> %0, <16 x i32> %1, <16 x i32> %2) ret <16 x i32> %4 } + +define <16 x i32> @vpdpwssd_v16i32_accumulate(<32 x i16> %a0, <32 x i16> %a1, <16 x i32> %a2) { +; CHECK-LABEL: vpdpwssd_v16i32_accumulate: +; CHECK: # %bb.0: +; CHECK-NEXT: vpdpwssd %zmm1, %zmm0, %zmm2 +; CHECK-NEXT: vmovdqa64 %zmm2, %zmm0 +; CHECK-NEXT: retq + %x0 = sext <32 x i16> %a0 to <32 x i32> + %x1 = sext <32 x i16> %a1 to <32 x i32> + %m = mul nsw <32 x i32> %x0, %x1 + %lo = shufflevector <32 x i32> %m, <32 x i32> poison, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30> + %hi = shufflevector <32 x i32> %m, <32 x i32> poison, <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31> + %r0 = add <16 x i32> %lo, %a2 + %r1 = add <16 x i32> %r0, %hi + ret <16 x i32> %r1 +} + +define <8 x i32> @vpdpwssd_v8i32_accumulate(<16 x i16> %a0, <16 x i16> %a1, <8 x i32> %a2) { +; AVX512VL-VNNI-LABEL: vpdpwssd_v8i32_accumulate: +; AVX512VL-VNNI: # %bb.0: +; AVX512VL-VNNI-NEXT: vpdpwssd %ymm1, %ymm0, %ymm2 +; AVX512VL-VNNI-NEXT: vmovdqa %ymm2, %ymm0 +; AVX512VL-VNNI-NEXT: retq +; +; AVX-VNNI-LABEL: vpdpwssd_v8i32_accumulate: +; AVX-VNNI: # %bb.0: +; AVX-VNNI-NEXT: {vex} vpdpwssd %ymm1, %ymm0, %ymm2 +; AVX-VNNI-NEXT: vmovdqa %ymm2, %ymm0 +; AVX-VNNI-NEXT: retq +; +; AVX512-VNNI-LABEL: vpdpwssd_v8i32_accumulate: +; AVX512-VNNI: # %bb.0: +; AVX512-VNNI-NEXT: vpmaddwd %ymm1, %ymm0, %ymm0 +; AVX512-VNNI-NEXT: vpaddd %ymm2, %ymm0, %ymm0 +; AVX512-VNNI-NEXT: retq + %x0 = sext <16 x i16> %a0 to <16 x i32> + %x1 = sext <16 x i16> %a1 to <16 x i32> + %m = mul nsw <16 x i32> %x0, %x1 + %lo = shufflevector <16 x i32> %m, <16 x i32> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14> + %hi = shufflevector <16 x i32> %m, <16 x i32> poison, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15> + %r0 = add <8 x i32> %hi, %a2 + %r1 = add <8 x i32> %lo, %r0 + ret <8 x i32> %r1 +} + +define <4 x i32> @vpdpwssd_v4i32_accumulate(<8 x i16> %a0, <8 x i16> %a1, <4 x i32> %a2) { +; AVX512VL-VNNI-LABEL: vpdpwssd_v4i32_accumulate: +; AVX512VL-VNNI: # %bb.0: +; AVX512VL-VNNI-NEXT: vpdpwssd %xmm1, %xmm0, %xmm2 +; AVX512VL-VNNI-NEXT: vmovdqa %xmm2, %xmm0 +; AVX512VL-VNNI-NEXT: retq +; +; AVX-VNNI-LABEL: vpdpwssd_v4i32_accumulate: +; AVX-VNNI: # %bb.0: +; AVX-VNNI-NEXT: {vex} vpdpwssd %xmm1, %xmm0, %xmm2 +; AVX-VNNI-NEXT: vmovdqa %xmm2, %xmm0 +; AVX-VNNI-NEXT: retq +; +; AVX512-VNNI-LABEL: vpdpwssd_v4i32_accumulate: +; AVX512-VNNI: # %bb.0: +; AVX512-VNNI-NEXT: vpmaddwd %xmm1, %xmm0, %xmm0 +; AVX512-VNNI-NEXT: vpaddd %xmm2, %xmm0, %xmm0 +; AVX512-VNNI-NEXT: retq + %x0 = sext <8 x i16> %a0 to <8 x i32> + %x1 = sext <8 x i16> %a1 to <8 x i32> + %m = mul nsw <8 x i32> %x0, %x1 + %lo = shufflevector <8 x i32> %m, <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6> + %hi = shufflevector <8 x i32> %m, <8 x i32> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7> + %r0 = add <4 x i32> %lo, %a2 + %r1 = add <4 x i32> %hi, %r0 + ret <4 x i32> %r1 +} diff --git a/llvm/test/CodeGen/X86/vselect.ll b/llvm/test/CodeGen/X86/vselect.ll index 9acd995d612c..be6ee8f68995 100644 --- a/llvm/test/CodeGen/X86/vselect.ll +++ b/llvm/test/CodeGen/X86/vselect.ll @@ -796,3 +796,29 @@ define i64 @vselect_any_extend_vector_inreg_crash(ptr %x) { ret i64 %4 } +; Tests the scalarizeBinOp code in DAGCombiner +define void @scalarize_binop(<1 x i1> %a) { +; SSE-LABEL: scalarize_binop: +; SSE: # %bb.0: # %bb0 +; SSE-NEXT: .p2align 4 +; SSE-NEXT: .LBB35_1: # %bb1 +; SSE-NEXT: # =>This Inner Loop Header: Depth=1 +; SSE-NEXT: jmp .LBB35_1 +; +; AVX-LABEL: scalarize_binop: +; AVX: # %bb.0: # %bb0 +; AVX-NEXT: .p2align 4 +; AVX-NEXT: .LBB35_1: # %bb1 +; AVX-NEXT: # =>This Inner Loop Header: Depth=1 +; AVX-NEXT: jmp .LBB35_1 +bb0: + br label %bb1 + +bb1: + %b = select <1 x i1> %a, <1 x i1> zeroinitializer, <1 x i1> splat (i1 true) + br label %bb2 + +bb2: + %c = extractelement <1 x i1> %b, i32 0 + br label %bb1 +} diff --git a/llvm/test/CodeGen/X86/widen_arith-4.ll b/llvm/test/CodeGen/X86/widen_arith-4.ll index c49882ffe0b3..ea6bf66fd292 100644 --- a/llvm/test/CodeGen/X86/widen_arith-4.ll +++ b/llvm/test/CodeGen/X86/widen_arith-4.ll @@ -65,7 +65,7 @@ define void @update(ptr %dst, ptr %src, i32 %n) nounwind { ; SSE41-NEXT: psubw %xmm0, %xmm1 ; SSE41-NEXT: movdqa %xmm1, %xmm2 ; SSE41-NEXT: psllw $2, %xmm2 -; SSE41-NEXT: psllw $1, %xmm1 +; SSE41-NEXT: paddw %xmm1, %xmm1 ; SSE41-NEXT: pblendw {{.*#+}} xmm2 = xmm1[0],xmm2[1],xmm1[2,3,4,5,6,7] ; SSE41-NEXT: pextrw $4, %xmm1, 8(%rcx,%rax) ; SSE41-NEXT: movq %xmm2, (%rcx,%rax) |