Lang Hames
6e5c97d417
Hoist runFinalizeActions and runDeallocActions into AllocAction.h and turn them into function templates with a ReportErrorFn argument that enables error reporting. This is used to report errors from dealloc actions. Update SimpleNativeMemoryMap to direct such errors to the Session via ReportErrorsViaSession. Add unit tests covering the new error-reporting paths.
350 lines
10 KiB
C++
350 lines
10 KiB
C++
//===- SimpleNativeMemoryMap.cpp ------------------------------------------===//
|
|
//
|
|
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
|
// See https://llvm.org/LICENSE.txt for license information.
|
|
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
//
|
|
// SimpleNativeMemoryMap and related APIs.
|
|
//
|
|
// TODO: We don't reset / uncommit pages on deinitialize, or on failure during
|
|
// initialize. We should do that to reduce memory pressure.
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
#include "orc-rt/SimpleNativeMemoryMap.h"
|
|
#include "orc-rt/Session.h"
|
|
|
|
#include <optional>
|
|
#include <sstream>
|
|
|
|
#if defined(__APPLE__) || defined(__linux__)
|
|
#include "Unix/NativeMemoryAPIs.inc"
|
|
#else
|
|
#error "Target OS memory APIs unsupported"
|
|
#endif
|
|
|
|
namespace orc_rt {
|
|
|
|
Expected<std::unique_ptr<SimpleNativeMemoryMap>>
|
|
SimpleNativeMemoryMap::Create(Session &S, SimpleSymbolTable &ST,
|
|
const char *InstanceName,
|
|
SimpleSymbolTable::MutatorFn AddInterface) {
|
|
|
|
std::unique_ptr<SimpleNativeMemoryMap> Instance(new SimpleNativeMemoryMap(S));
|
|
|
|
SimpleSymbolTable SNMMST;
|
|
if (auto Err = AddInterface(SNMMST))
|
|
return Err;
|
|
std::pair<const char *, const void *> InstanceSym[] = {
|
|
{InstanceName, static_cast<const void *>(Instance.get())}};
|
|
if (auto Err = SNMMST.addUnique(InstanceSym))
|
|
return std::move(Err);
|
|
|
|
if (auto Err = ST.addUnique(SNMMST))
|
|
return std::move(Err);
|
|
|
|
return std::move(Instance);
|
|
}
|
|
|
|
void SimpleNativeMemoryMap::reserve(OnReserveCompleteFn &&OnComplete,
|
|
size_t Size) {
|
|
if (Size % S.processInfo().pageSize()) {
|
|
return OnComplete(make_error<StringError>(
|
|
(std::ostringstream()
|
|
<< "SimpleNativeMemoryMap error: reserved size " << std::hex << Size
|
|
<< " is not a page-size multiple")
|
|
.str()));
|
|
}
|
|
|
|
auto Addr = hostOSMemoryReserve(Size);
|
|
if (!Addr)
|
|
return OnComplete(Addr.takeError());
|
|
|
|
{
|
|
std::scoped_lock<std::mutex> Lock(M);
|
|
assert(!Slabs.count(*Addr) &&
|
|
"hostOSMemoryReserve returned duplicate addresses");
|
|
Slabs.emplace(std::make_pair(*Addr, SlabInfo(Size)));
|
|
}
|
|
|
|
OnComplete(*Addr);
|
|
}
|
|
|
|
void SimpleNativeMemoryMap::release(OnReleaseCompleteFn &&OnComplete,
|
|
void *Addr) {
|
|
std::optional<SlabInfo> SI;
|
|
{
|
|
std::scoped_lock<std::mutex> Lock(M);
|
|
auto I = Slabs.find(Addr);
|
|
if (I != Slabs.end()) {
|
|
SI = std::move(I->second);
|
|
Slabs.erase(I);
|
|
}
|
|
}
|
|
|
|
if (!SI) {
|
|
std::ostringstream ErrMsg;
|
|
ErrMsg << "SimpleNativeMemoryMap error: release called on unrecognized "
|
|
"address "
|
|
<< Addr;
|
|
return OnComplete(make_error<StringError>(ErrMsg.str()));
|
|
}
|
|
|
|
for (auto &[Addr, DAAs] : SI->DeallocActions)
|
|
runDeallocActions(std::move(DAAs), ReportErrorsViaSession(S));
|
|
|
|
OnComplete(hostOSMemoryRelease(Addr, SI->Size));
|
|
}
|
|
|
|
void SimpleNativeMemoryMap::releaseMultiple(OnReleaseCompleteFn &&OnComplete,
|
|
std::vector<void *> Addrs) {
|
|
releaseNext(std::move(OnComplete), std::move(Addrs), false, Error::success());
|
|
}
|
|
|
|
void SimpleNativeMemoryMap::initialize(OnInitializeCompleteFn &&OnComplete,
|
|
InitializeRequest IR) {
|
|
|
|
void *Base = nullptr;
|
|
|
|
// TODO: Record initialize segments for release.
|
|
// std::vector<std::pair<void*, size_t>> InitializeSegments;
|
|
|
|
// Check segment validity before proceeding.
|
|
for (auto &S : IR.Segments) {
|
|
|
|
if (S.Content.size() > S.Size) {
|
|
return OnComplete(make_error<StringError>(
|
|
(std::ostringstream()
|
|
<< "For segment [" << (void *)S.Address << ".."
|
|
<< (void *)(S.Address + S.Size) << "), "
|
|
<< " content size (" << std::hex << S.Content.size()
|
|
<< ") exceeds segment size (" << S.Size << ")")
|
|
.str()));
|
|
}
|
|
|
|
// Copy any requested content.
|
|
if (!S.Content.empty())
|
|
memcpy(S.Address, S.Content.data(), S.Content.size());
|
|
|
|
// Zero-fill the rest of the section.
|
|
if (size_t ZeroFillSize = S.Size - S.Content.size())
|
|
memset(S.Address + S.Content.size(), 0, ZeroFillSize);
|
|
|
|
if (auto Err = hostOSMemoryProtect(S.Address, S.Size, S.AG.getMemProt()))
|
|
return OnComplete(std::move(Err));
|
|
|
|
switch (S.AG.getMemLifetime()) {
|
|
case MemLifetime::Standard:
|
|
if (!Base || S.Address < Base)
|
|
Base = S.Address;
|
|
break;
|
|
case MemLifetime::Finalize:
|
|
// TODO: Record finalize segment for release.
|
|
// FinalizeSegments.push_back({S.Address, S.Size});
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (!Base)
|
|
return OnComplete(
|
|
make_error<StringError>("SimpleNativeMemoryMap initialize error: "
|
|
"finalization requires at least "
|
|
"one standard-lifetime segment"));
|
|
|
|
auto DeallocActions =
|
|
runFinalizeActions(std::move(IR.AAPs), ReportErrorsViaSession(S));
|
|
if (!DeallocActions)
|
|
return OnComplete(DeallocActions.takeError());
|
|
|
|
if (auto Err = recordDeallocActions(Base, std::move(*DeallocActions))) {
|
|
runDeallocActions(std::move(*DeallocActions), ReportErrorsViaSession(S));
|
|
return OnComplete(std::move(Err));
|
|
}
|
|
|
|
OnComplete(Base);
|
|
}
|
|
|
|
void SimpleNativeMemoryMap::deinitialize(OnDeinitializeCompleteFn &&OnComplete,
|
|
void *Base) {
|
|
std::vector<AllocAction> DAAs;
|
|
|
|
{
|
|
std::unique_lock<std::mutex> Lock(M);
|
|
auto *SI = findSlabInfoFor(Base);
|
|
if (!SI) {
|
|
Lock.unlock();
|
|
return OnComplete(makeBadSlabError(Base, "deinitialize"));
|
|
}
|
|
|
|
auto I = SI->DeallocActions.find(Base);
|
|
if (I == SI->DeallocActions.end()) {
|
|
Lock.unlock();
|
|
std::ostringstream ErrMsg;
|
|
ErrMsg
|
|
<< "SimpleNativeMemoryMap deinitialize error: no deallocate actions "
|
|
"registered for segment base address "
|
|
<< Base;
|
|
return OnComplete(make_error<StringError>(ErrMsg.str()));
|
|
}
|
|
|
|
DAAs = std::move(I->second);
|
|
SI->DeallocActions.erase(I);
|
|
}
|
|
|
|
runDeallocActions(std::move(DAAs), ReportErrorsViaSession(S));
|
|
OnComplete(Error::success());
|
|
}
|
|
|
|
void SimpleNativeMemoryMap::deinitializeMultiple(
|
|
OnDeinitializeCompleteFn &&OnComplete, std::vector<void *> Bases) {
|
|
deinitializeNext(std::move(OnComplete), std::move(Bases), false,
|
|
Error::success());
|
|
}
|
|
|
|
void SimpleNativeMemoryMap::onDetach(Service::OnCompleteFn OnComplete,
|
|
bool ShutdownRequested) {
|
|
// Detach is a noop for now: we just retain all actions to run at shutdown
|
|
// time.
|
|
OnComplete();
|
|
}
|
|
|
|
void SimpleNativeMemoryMap::onShutdown(Service::OnCompleteFn OnComplete) {
|
|
// TODO: Establish a clear order to run deallocate actions across slabs,
|
|
// object boundaries.
|
|
|
|
// Collect slab base addresses for removal.
|
|
std::vector<void *> Bases;
|
|
{
|
|
std::scoped_lock<std::mutex> Lock(M);
|
|
for (auto &[Base, _] : Slabs)
|
|
Bases.push_back(Base);
|
|
}
|
|
|
|
shutdownNext(std::move(OnComplete), std::move(Bases));
|
|
}
|
|
|
|
void SimpleNativeMemoryMap::releaseNext(OnReleaseCompleteFn &&OnComplete,
|
|
std::vector<void *> Addrs,
|
|
bool AnyError, Error LastErr) {
|
|
if (LastErr) {
|
|
S.reportError(std::move(LastErr));
|
|
AnyError |= true;
|
|
}
|
|
|
|
if (Addrs.empty()) {
|
|
if (!AnyError)
|
|
return OnComplete(Error::success());
|
|
|
|
return OnComplete(
|
|
make_error<StringError>("Failed to release some addresses"));
|
|
}
|
|
|
|
void *NextAddr = Addrs.back();
|
|
Addrs.pop_back();
|
|
|
|
release(
|
|
[this, OnComplete = std::move(OnComplete), AnyError = AnyError,
|
|
Addrs = std::move(Addrs)](Error Err) mutable {
|
|
releaseNext(std::move(OnComplete), std::move(Addrs), AnyError,
|
|
std::move(Err));
|
|
},
|
|
NextAddr);
|
|
}
|
|
|
|
void SimpleNativeMemoryMap::deinitializeNext(
|
|
OnDeinitializeCompleteFn &&OnComplete, std::vector<void *> Addrs,
|
|
bool AnyError, Error LastErr) {
|
|
if (LastErr) {
|
|
S.reportError(std::move(LastErr));
|
|
AnyError |= true;
|
|
}
|
|
|
|
if (Addrs.empty()) {
|
|
if (!AnyError)
|
|
return OnComplete(Error::success());
|
|
|
|
return OnComplete(
|
|
make_error<StringError>("Failed to deinitialize some addresses"));
|
|
}
|
|
|
|
void *NextAddr = Addrs.back();
|
|
Addrs.pop_back();
|
|
|
|
deinitialize(
|
|
[this, OnComplete = std::move(OnComplete), AnyError = AnyError,
|
|
Addrs = std::move(Addrs)](Error Err) mutable {
|
|
deinitializeNext(std::move(OnComplete), std::move(Addrs), AnyError,
|
|
std::move(Err));
|
|
},
|
|
NextAddr);
|
|
}
|
|
|
|
void SimpleNativeMemoryMap::shutdownNext(Service::OnCompleteFn OnComplete,
|
|
std::vector<void *> Bases) {
|
|
if (Bases.empty())
|
|
return OnComplete();
|
|
|
|
auto *Base = Bases.back();
|
|
Bases.pop_back();
|
|
|
|
release(
|
|
[this, Bases = std::move(Bases),
|
|
OnComplete = std::move(OnComplete)](Error Err) mutable {
|
|
if (Err) {
|
|
// TODO: Log release error?
|
|
consumeError(std::move(Err));
|
|
}
|
|
shutdownNext(std::move(OnComplete), std::move(Bases));
|
|
},
|
|
Base);
|
|
}
|
|
|
|
Error SimpleNativeMemoryMap::makeBadSlabError(void *Base, const char *Op) {
|
|
std::ostringstream ErrMsg;
|
|
ErrMsg << "SimpleNativeMemoryMap " << Op << " error: segment base address "
|
|
<< Base << " does not fall within an allocated slab";
|
|
return make_error<StringError>(ErrMsg.str());
|
|
}
|
|
|
|
SimpleNativeMemoryMap::SlabInfo *
|
|
SimpleNativeMemoryMap::findSlabInfoFor(void *Base) {
|
|
// NOTE: We assume that the caller is holding a lock for M.
|
|
auto I = Slabs.upper_bound(Base);
|
|
if (I == Slabs.begin())
|
|
return nullptr;
|
|
|
|
--I;
|
|
if (reinterpret_cast<char *>(I->first) + I->second.Size <=
|
|
reinterpret_cast<char *>(Base))
|
|
return nullptr;
|
|
|
|
return &I->second;
|
|
}
|
|
|
|
Error SimpleNativeMemoryMap::recordDeallocActions(
|
|
void *Base, std::vector<AllocAction> DeallocActions) {
|
|
|
|
std::unique_lock<std::mutex> Lock(M);
|
|
auto *SI = findSlabInfoFor(Base);
|
|
if (!SI) {
|
|
Lock.unlock();
|
|
return makeBadSlabError(Base, "deinitialize");
|
|
}
|
|
|
|
auto I = SI->DeallocActions.find(Base);
|
|
if (I != SI->DeallocActions.end()) {
|
|
Lock.unlock();
|
|
std::ostringstream ErrMsg;
|
|
ErrMsg << "SimpleNativeMemoryMap initialize error: segment base address "
|
|
"reused in subsequent initialize call";
|
|
return make_error<StringError>(ErrMsg.str());
|
|
}
|
|
|
|
SI->DeallocActions[Base] = std::move(DeallocActions);
|
|
return Error::success();
|
|
}
|
|
|
|
} // namespace orc_rt
|