From bb72f073808a42e35ddc52cb0810470ffeb3cce3 Mon Sep 17 00:00:00 2001 From: Lang Hames Date: Sat, 11 Sep 2021 19:07:16 +1000 Subject: Re-apply bb27e456435 and 5629afea910 with fixes. This reapplies bb27e4564355243e479cab40885d6e0f7f640572 (SimpleRemoteEPC support) and 2269a941a450a0d395161cfb792be58870b2875b (#include fix) with further fixes to support building with LLVM_ENABLE_THREADS=Off. --- .../Orc/TargetProcess/SimpleRemoteEPCServer.cpp | 404 +++++++++++++++++++++ 1 file changed, 404 insertions(+) create mode 100644 llvm/lib/ExecutionEngine/Orc/TargetProcess/SimpleRemoteEPCServer.cpp (limited to 'llvm/lib/ExecutionEngine/Orc/TargetProcess/SimpleRemoteEPCServer.cpp') diff --git a/llvm/lib/ExecutionEngine/Orc/TargetProcess/SimpleRemoteEPCServer.cpp b/llvm/lib/ExecutionEngine/Orc/TargetProcess/SimpleRemoteEPCServer.cpp new file mode 100644 index 000000000000..d3180ccf1ae3 --- /dev/null +++ b/llvm/lib/ExecutionEngine/Orc/TargetProcess/SimpleRemoteEPCServer.cpp @@ -0,0 +1,404 @@ +//===------- SimpleEPCServer.cpp - EPC over simple abstract channel -------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/TargetProcess/SimpleRemoteEPCServer.h" + +#include "llvm/ExecutionEngine/Orc/Shared/TargetProcessControlTypes.h" +#include "llvm/ExecutionEngine/Orc/TargetProcess/TargetExecutionUtils.h" +#include "llvm/Support/FormatVariadic.h" +#include "llvm/Support/Host.h" +#include "llvm/Support/Process.h" + +#define DEBUG_TYPE "orc" + +using namespace llvm::orc::shared; + +namespace llvm { +namespace orc { + +static llvm::orc::shared::detail::CWrapperFunctionResult +reserveWrapper(const char *ArgData, size_t ArgSize) { + return WrapperFunction::handle( + ArgData, ArgSize, + [](uint64_t Size) -> Expected { + std::error_code EC; + auto MB = sys::Memory::allocateMappedMemory( + Size, 0, sys::Memory::MF_READ | sys::Memory::MF_WRITE, EC); + if (EC) + return errorCodeToError(EC); + return ExecutorAddress::fromPtr(MB.base()); + }) + .release(); +} + +static llvm::orc::shared::detail::CWrapperFunctionResult +finalizeWrapper(const char *ArgData, size_t ArgSize) { + return WrapperFunction::handle( + ArgData, ArgSize, + [](const tpctypes::FinalizeRequest &FR) -> Error { + for (auto &Seg : FR) { + char *Mem = Seg.Addr.toPtr(); + memcpy(Mem, Seg.Content.data(), Seg.Content.size()); + memset(Mem + Seg.Content.size(), 0, + Seg.Size - Seg.Content.size()); + assert(Seg.Size <= std::numeric_limits::max()); + if (auto EC = sys::Memory::protectMappedMemory( + {Mem, static_cast(Seg.Size)}, + tpctypes::fromWireProtectionFlags(Seg.Prot))) + return errorCodeToError(EC); + if (Seg.Prot & tpctypes::WPF_Exec) + sys::Memory::InvalidateInstructionCache(Mem, Seg.Size); + } + return Error::success(); + }) + .release(); +} + +static llvm::orc::shared::detail::CWrapperFunctionResult +deallocateWrapper(const char *ArgData, size_t ArgSize) { + return WrapperFunction::handle( + ArgData, ArgSize, + [](ExecutorAddress Base, uint64_t Size) -> Error { + sys::MemoryBlock MB(Base.toPtr(), Size); + if (auto EC = sys::Memory::releaseMappedMemory(MB)) + return errorCodeToError(EC); + return Error::success(); + }) + .release(); +} + +template +static llvm::orc::shared::detail::CWrapperFunctionResult +writeUIntsWrapper(const char *ArgData, size_t ArgSize) { + return WrapperFunction)>::handle( + ArgData, ArgSize, + [](std::vector Ws) { + for (auto &W : Ws) + *jitTargetAddressToPointer(W.Address) = + W.Value; + }) + .release(); +} + +static llvm::orc::shared::detail::CWrapperFunctionResult +writeBuffersWrapper(const char *ArgData, size_t ArgSize) { + return WrapperFunction)>::handle( + ArgData, ArgSize, + [](std::vector Ws) { + for (auto &W : Ws) + memcpy(jitTargetAddressToPointer(W.Address), + W.Buffer.data(), W.Buffer.size()); + }) + .release(); +} + +static llvm::orc::shared::detail::CWrapperFunctionResult +runAsMainWrapper(const char *ArgData, size_t ArgSize) { + return WrapperFunction::handle( + ArgData, ArgSize, + [](ExecutorAddress MainAddr, + std::vector Args) -> int64_t { + return runAsMain(MainAddr.toPtr(), Args); + }) + .release(); +} + +SimpleRemoteEPCServer::Dispatcher::~Dispatcher() {} + +#if LLVM_ENABLE_THREADS +void SimpleRemoteEPCServer::ThreadDispatcher::dispatch( + unique_function Work) { + { + std::lock_guard Lock(DispatchMutex); + if (!Running) + return; + ++Outstanding; + } + + std::thread([this, Work = std::move(Work)]() mutable { + Work(); + std::lock_guard Lock(DispatchMutex); + --Outstanding; + OutstandingCV.notify_all(); + }).detach(); +} + +void SimpleRemoteEPCServer::ThreadDispatcher::shutdown() { + std::unique_lock Lock(DispatchMutex); + Running = false; + OutstandingCV.wait(Lock, [this]() { return Outstanding == 0; }); +} +#endif + +StringMap SimpleRemoteEPCServer::defaultBootstrapSymbols() { + StringMap DBS; + + DBS["__llvm_orc_memory_reserve"] = ExecutorAddress::fromPtr(&reserveWrapper); + DBS["__llvm_orc_memory_finalize"] = + ExecutorAddress::fromPtr(&finalizeWrapper); + DBS["__llvm_orc_memory_deallocate"] = + ExecutorAddress::fromPtr(&deallocateWrapper); + DBS["__llvm_orc_memory_write_uint8s"] = ExecutorAddress::fromPtr( + &writeUIntsWrapper); + DBS["__llvm_orc_memory_write_uint16s"] = ExecutorAddress::fromPtr( + &writeUIntsWrapper); + DBS["__llvm_orc_memory_write_uint32s"] = ExecutorAddress::fromPtr( + &writeUIntsWrapper); + DBS["__llvm_orc_memory_write_uint64s"] = ExecutorAddress::fromPtr( + &writeUIntsWrapper); + DBS["__llvm_orc_memory_write_buffers"] = + ExecutorAddress::fromPtr(&writeBuffersWrapper); + DBS["__llvm_orc_run_as_main"] = ExecutorAddress::fromPtr(&runAsMainWrapper); + DBS["__llvm_orc_load_dylib"] = ExecutorAddress::fromPtr(&loadDylibWrapper); + DBS["__llvm_orc_lookup_symbols"] = + ExecutorAddress::fromPtr(&lookupSymbolsWrapper); + return DBS; +} + +Expected +SimpleRemoteEPCServer::handleMessage(SimpleRemoteEPCOpcode OpC, uint64_t SeqNo, + ExecutorAddress TagAddr, + SimpleRemoteEPCArgBytesVector ArgBytes) { + using UT = std::underlying_type_t; + if (static_cast(OpC) < static_cast(SimpleRemoteEPCOpcode::FirstOpC) || + static_cast(OpC) > static_cast(SimpleRemoteEPCOpcode::LastOpC)) + return make_error("Unexpected opcode", + inconvertibleErrorCode()); + + // TODO: Clean detach message? + switch (OpC) { + case SimpleRemoteEPCOpcode::Setup: + return make_error("Unexpected Setup opcode", + inconvertibleErrorCode()); + case SimpleRemoteEPCOpcode::Hangup: + return SimpleRemoteEPCTransportClient::EndSession; + case SimpleRemoteEPCOpcode::Result: + if (auto Err = handleResult(SeqNo, TagAddr, std::move(ArgBytes))) + return std::move(Err); + break; + case SimpleRemoteEPCOpcode::CallWrapper: + handleCallWrapper(SeqNo, TagAddr, std::move(ArgBytes)); + break; + } + return ContinueSession; +} + +Error SimpleRemoteEPCServer::waitForDisconnect() { + std::unique_lock Lock(ServerStateMutex); + ShutdownCV.wait(Lock, [this]() { return RunState == ServerShutDown; }); + return std::move(ShutdownErr); +} + +void SimpleRemoteEPCServer::handleDisconnect(Error Err) { + PendingJITDispatchResultsMap TmpPending; + + { + std::lock_guard Lock(ServerStateMutex); + std::swap(TmpPending, PendingJITDispatchResults); + RunState = ServerShuttingDown; + } + + // Send out-of-band errors to any waiting threads. + for (auto &KV : TmpPending) + KV.second->set_value( + shared::WrapperFunctionResult::createOutOfBandError("disconnecting")); + + // TODO: Free attached resources. + // 1. Close libraries in DylibHandles. + + // Wait for dispatcher to clear. + D->shutdown(); + + std::lock_guard Lock(ServerStateMutex); + ShutdownErr = joinErrors(std::move(ShutdownErr), std::move(Err)); + RunState = ServerShutDown; + ShutdownCV.notify_all(); +} + +Error SimpleRemoteEPCServer::sendSetupMessage( + StringMap BootstrapSymbols) { + + using namespace SimpleRemoteEPCDefaultBootstrapSymbolNames; + + std::vector SetupPacket; + SimpleRemoteEPCExecutorInfo EI; + EI.TargetTriple = sys::getProcessTriple(); + if (auto PageSize = sys::Process::getPageSize()) + EI.PageSize = *PageSize; + else + return PageSize.takeError(); + EI.BootstrapSymbols = std::move(BootstrapSymbols); + + assert(!EI.BootstrapSymbols.count(ExecutorSessionObjectName) && + "Dispatch context name should not be set"); + assert(!EI.BootstrapSymbols.count(DispatchFnName) && + "Dispatch function name should not be set"); + EI.BootstrapSymbols[ExecutorSessionObjectName] = + ExecutorAddress::fromPtr(this); + EI.BootstrapSymbols[DispatchFnName] = + ExecutorAddress::fromPtr(jitDispatchEntry); + + using SPSSerialize = + shared::SPSArgList; + auto SetupPacketBytes = + shared::WrapperFunctionResult::allocate(SPSSerialize::size(EI)); + shared::SPSOutputBuffer OB(SetupPacketBytes.data(), SetupPacketBytes.size()); + if (!SPSSerialize::serialize(OB, EI)) + return make_error("Could not send setup packet", + inconvertibleErrorCode()); + + return T->sendMessage(SimpleRemoteEPCOpcode::Setup, 0, ExecutorAddress(), + {SetupPacketBytes.data(), SetupPacketBytes.size()}); +} + +Error SimpleRemoteEPCServer::handleResult( + uint64_t SeqNo, ExecutorAddress TagAddr, + SimpleRemoteEPCArgBytesVector ArgBytes) { + std::promise *P = nullptr; + { + std::lock_guard Lock(ServerStateMutex); + auto I = PendingJITDispatchResults.find(SeqNo); + if (I == PendingJITDispatchResults.end()) + return make_error("No call for sequence number " + + Twine(SeqNo), + inconvertibleErrorCode()); + P = I->second; + PendingJITDispatchResults.erase(I); + releaseSeqNo(SeqNo); + } + auto R = shared::WrapperFunctionResult::allocate(ArgBytes.size()); + memcpy(R.data(), ArgBytes.data(), ArgBytes.size()); + P->set_value(std::move(R)); + return Error::success(); +} + +void SimpleRemoteEPCServer::handleCallWrapper( + uint64_t RemoteSeqNo, ExecutorAddress TagAddr, + SimpleRemoteEPCArgBytesVector ArgBytes) { + D->dispatch([this, RemoteSeqNo, TagAddr, ArgBytes = std::move(ArgBytes)]() { + using WrapperFnTy = + shared::detail::CWrapperFunctionResult (*)(const char *, size_t); + auto *Fn = TagAddr.toPtr(); + shared::WrapperFunctionResult ResultBytes( + Fn(ArgBytes.data(), ArgBytes.size())); + if (auto Err = T->sendMessage(SimpleRemoteEPCOpcode::Result, RemoteSeqNo, + ExecutorAddress(), + {ResultBytes.data(), ResultBytes.size()})) + ReportError(std::move(Err)); + }); +} + +shared::detail::CWrapperFunctionResult +SimpleRemoteEPCServer::loadDylibWrapper(const char *ArgData, size_t ArgSize) { + return shared::WrapperFunction::handle( + ArgData, ArgSize, + [](ExecutorAddress ExecutorSessionObj, std::string Path, + uint64_t Flags) -> Expected { + return ExecutorSessionObj.toPtr() + ->loadDylib(Path, Flags); + }) + .release(); +} + +shared::detail::CWrapperFunctionResult +SimpleRemoteEPCServer::lookupSymbolsWrapper(const char *ArgData, + size_t ArgSize) { + return shared::WrapperFunction::handle( + ArgData, ArgSize, + [](ExecutorAddress ExecutorSessionObj, + std::vector Lookup) { + return ExecutorSessionObj.toPtr() + ->lookupSymbols(Lookup); + }) + .release(); +} + +Expected +SimpleRemoteEPCServer::loadDylib(const std::string &Path, uint64_t Mode) { + std::string ErrMsg; + const char *P = Path.empty() ? nullptr : Path.c_str(); + auto DL = sys::DynamicLibrary::getPermanentLibrary(P, &ErrMsg); + if (!DL.isValid()) + return make_error(std::move(ErrMsg), inconvertibleErrorCode()); + std::lock_guard Lock(ServerStateMutex); + uint64_t Id = Dylibs.size(); + Dylibs.push_back(std::move(DL)); + return Id; +} + +Expected>> +SimpleRemoteEPCServer::lookupSymbols(const std::vector &L) { + std::vector> Result; + + for (const auto &E : L) { + if (E.H >= Dylibs.size()) + return make_error("Unrecognized handle", + inconvertibleErrorCode()); + auto &DL = Dylibs[E.H]; + Result.push_back({}); + + for (const auto &Sym : E.Symbols) { + + const char *DemangledSymName = Sym.Name.c_str(); +#ifdef __APPLE__ + if (*DemangledSymName == '_') + ++DemangledSymName; +#endif + + void *Addr = DL.getAddressOfSymbol(DemangledSymName); + if (!Addr && Sym.Required) + return make_error(Twine("Missing definition for ") + + DemangledSymName, + inconvertibleErrorCode()); + + Result.back().push_back(ExecutorAddress::fromPtr(Addr)); + } + } + + return std::move(Result); +} + +shared::WrapperFunctionResult +SimpleRemoteEPCServer::doJITDispatch(const void *FnTag, const char *ArgData, + size_t ArgSize) { + uint64_t SeqNo; + std::promise ResultP; + auto ResultF = ResultP.get_future(); + { + std::lock_guard Lock(ServerStateMutex); + if (RunState != ServerRunning) + return shared::WrapperFunctionResult::createOutOfBandError( + "jit_dispatch not available (EPC server shut down)"); + + SeqNo = getNextSeqNo(); + assert(!PendingJITDispatchResults.count(SeqNo) && "SeqNo already in use"); + PendingJITDispatchResults[SeqNo] = &ResultP; + } + + if (auto Err = + T->sendMessage(SimpleRemoteEPCOpcode::CallWrapper, SeqNo, + ExecutorAddress::fromPtr(FnTag), {ArgData, ArgSize})) + ReportError(std::move(Err)); + + return ResultF.get(); +} + +shared::detail::CWrapperFunctionResult +SimpleRemoteEPCServer::jitDispatchEntry(void *DispatchCtx, const void *FnTag, + const char *ArgData, size_t ArgSize) { + return reinterpret_cast(DispatchCtx) + ->doJITDispatch(FnTag, ArgData, ArgSize) + .release(); +} + +} // end namespace orc +} // end namespace llvm -- cgit v1.2.3