[flang] Implement COMPLEX(10) passing and return ABI for X86-64 linux (#74094)

COMPLEX(10) passing by value and returning follows C complex
passing/returning ABI.

Cover the COMPLEX(10) case (X87 / __Complex long double on X86-64).

Implements System V ABI for AMD64 version 1.0.

The LLVM signatures match the one generated by clang for the __Complex
long double case.

Note that a FIXME is added for the COMPLEX(8) case that is incorrect in
a corner case. This will be fixed when dealing with passing derived type
by value in BIND(C) context.

1 files changed, 16 insertions, 0 deletions

diff --git a/flang/lib/Optimizer/CodeGen/Target.cpp b/flang/lib/Optimizer/CodeGen/Target.cpp
index b136c8f7a9dd..112f56e268c3 100644
--- a/flang/lib/Optimizer/CodeGen/Target.cpp
+++ b/flang/lib/Optimizer/CodeGen/Target.cpp
@@ -261,9 +261,20 @@ struct TargetX86_64 : public GenericTarget<TargetX86_64> {
       // <2 x t>   vector of 2 eleTy
       marshal.emplace_back(fir::VectorType::get(2, eleTy), AT{});
     } else if (sem == &llvm::APFloat::IEEEdouble()) {
+      // FIXME: In case of SSE register exhaustion, the ABI here may be
+      // incorrect since LLVM may pass the real via register and the imaginary
+      // part via the stack while the ABI it should be all in register or all
+      // in memory. Register occupancy must be analyzed here.
       // two distinct double arguments
       marshal.emplace_back(eleTy, AT{});
       marshal.emplace_back(eleTy, AT{});
+    } else if (sem == &llvm::APFloat::x87DoubleExtended()) {
+      // Use a type that will be translated into LLVM as:
+      // { x86_fp80, x86_fp80 }  struct of 2 fp128, byval, align 16
+      marshal.emplace_back(
+          fir::ReferenceType::get(mlir::TupleType::get(
+              eleTy.getContext(), mlir::TypeRange{eleTy, eleTy})),
+          AT{/*align=*/16, /*byval=*/true});
     } else if (sem == &llvm::APFloat::IEEEquad()) {
       // Use a type that will be translated into LLVM as:
       // { fp128, fp128 }   struct of 2 fp128, byval, align 16
@@ -290,6 +301,11 @@ struct TargetX86_64 : public GenericTarget<TargetX86_64> {
       marshal.emplace_back(mlir::TupleType::get(eleTy.getContext(),
                                                 mlir::TypeRange{eleTy, eleTy}),
                            AT{});
+    } else if (sem == &llvm::APFloat::x87DoubleExtended()) {
+      // { x86_fp80, x86_fp80 }
+      marshal.emplace_back(mlir::TupleType::get(eleTy.getContext(),
+                                                mlir::TypeRange{eleTy, eleTy}),
+                           AT{});
     } else if (sem == &llvm::APFloat::IEEEquad()) {
       // Use a type that will be translated into LLVM as:
       // { fp128, fp128 }   struct of 2 fp128, sret, align 16