Prevent OOM

This commit is contained in:
2026-06-18 05:55:31 +10:00
parent ebba70b9d9
commit b3436a61ee
44 changed files with 1797 additions and 66 deletions
+1 -1
View File
@@ -540,7 +540,7 @@ namespace ANSCENTER {
") returned null — the DLL may be corrupt.");
s_ort_api = api;
Ort::Global<void>::api_ = api;
Ort::InitApi(api);
std::cout << "[EPLoader] ORT loaded successfully." << std::endl;
std::cout << "[EPLoader] ORT DLL version : " << base->GetVersionString() << std::endl;
+11 -3
View File
@@ -141,7 +141,7 @@ namespace ANSCENTER {
{
try {
// Make sure the Ort API pointer is initialised in THIS DLL.
if (Ort::Global<void>::api_ == nullptr) {
if (Ort::detail::Global::Api() == nullptr) {
Ort::InitApi(static_cast<const OrtApi*>(EPLoader::GetOrtApiRaw()));
}
@@ -574,10 +574,10 @@ namespace ANSCENTER {
const auto& epInfo = EPLoader::Current();
ANS_DBG("OrtHandler", "initialize_handler: EPLoader type=%d dir=%s",
static_cast<int>(epInfo.type), epInfo.libraryDir.c_str());
if (Ort::Global<void>::api_ == nullptr)
if (Ort::detail::Global::Api() == nullptr)
Ort::InitApi(static_cast<const OrtApi*>(EPLoader::GetOrtApiRaw()));
std::cout << "[ORT] api_ = " << (void*)Ort::Global<void>::api_ << std::endl;
std::cout << "[ORT] api_ = " << (void*)Ort::detail::Global::Api() << std::endl;
EngineType engine = (static_cast<int>(m_engineType) == -1)
? epInfo.type : m_engineType;
@@ -1900,4 +1900,12 @@ namespace ANSCENTER {
return std::fabs(width() * height());
}
// Explicit instantiations of every BoundingBoxType member must live
// AFTER the template definitions above so the compiler actually emits
// symbols (the directives in ONNXEngine.h sit before the definitions
// and ClangCL, unlike MSVC, refuses to emit symbols from there).
template struct ANSCENTER::types::BoundingBoxType<int, float>;
template struct ANSCENTER::types::BoundingBoxType<float, float>;
template struct ANSCENTER::types::BoundingBoxType<double, double>;
} // namespace ANSCENTER
+4 -3
View File
@@ -73,9 +73,10 @@ namespace ANSCENTER {
}
};
template class BoundingBoxType<int, float>;
template class BoundingBoxType<float, float>;
template class BoundingBoxType<double, double>;
// Explicit instantiations now live at the end of ONNXEngine.cpp
// (after the template member definitions) so the compiler actually
// emits symbols. ClangCL refuses to emit them from a directive that
// precedes the definitions, whereas MSVC silently tolerated it.
typedef BoundingBoxType<int, float> Boxi;
typedef BoundingBoxType<float, float> Boxf;
+1 -1
View File
@@ -331,7 +331,7 @@ namespace ANSCENTER
{
// Initialize ORT API
const auto& epInfo = EPLoader::Current();
if (Ort::Global<void>::api_ == nullptr)
if (Ort::detail::Global::Api() == nullptr)
Ort::InitApi(static_cast<const OrtApi*>(EPLoader::GetOrtApiRaw()));
m_env = new Ort::Env(ORT_LOGGING_LEVEL_ERROR, "ONNXSAM3");
@@ -4,6 +4,7 @@
#include <sstream>
#include "Utility.h"
#include "TRTCompat.h"
#include "GpuMemoryGuard.h"
// ============================================================================
// Crash-safe wrappers for TensorRT operations that can crash the process.
@@ -805,13 +806,22 @@ bool Engine<T>::loadNetwork(std::string trtModelPath, const std::array<float, 3>
// (cudaMalloc failure during inference) when too many tasks are loaded.
{
size_t freeVRAM = 0, totalVRAM = 0;
cudaError_t memErr = cudaMemGetInfo(&freeVRAM, &totalVRAM);
constexpr size_t kMinFreeBytes = 256ULL * 1024 * 1024; // 256 MiB minimum
if (memErr != cudaSuccess) {
// cudaMemGetInfo failed — CUDA context may not be initialized on this thread.
// Don't reject: let TRT try to deserialize (it may succeed).
} else if (freeVRAM < kMinFreeBytes) {
m_lastLoadFailedVRAM = true; // signal to buildLoadNetwork: engine file is NOT corrupt
bool haveMem = ANSCENTER::GpuMemoryGuard::Query(freeVRAM, totalVRAM);
// (1) Admission control: refuse to load this engine on the GPU if it
// does not fit while keeping the GpuMemoryGuard safety margin free.
// Prevents VRAM over-subscription across concurrent engines (FR/OD/LPR/
// liveness) that would otherwise OOM and permanently poison the CUDA
// context. The engine file is valid — m_lastLoadFailedVRAM tells the
// caller to preserve it (no rebuild) and just skip GPU load for now.
if (haveMem && !ANSCENTER::GpuMemoryGuard::CanLoad(buffer.size())) {
m_lastLoadFailedVRAM = true;
std::cerr << "[GpuMemGuard] REFUSING engine load on GPU "
<< m_options.deviceIndex << ": need ~"
<< (buffer.size() / (1024 * 1024)) << " MiB + "
<< (ANSCENTER::GpuMemoryGuard::Margin() / (1024 * 1024))
<< " MiB margin, only " << (freeVRAM / (1024 * 1024))
<< " MiB free. Reduce concurrent engines/cameras or batch size."
<< std::endl;
return false;
}
if (m_verbose) {