Prevent OOM
This commit is contained in:
@@ -540,7 +540,7 @@ namespace ANSCENTER {
|
||||
") returned null — the DLL may be corrupt.");
|
||||
|
||||
s_ort_api = api;
|
||||
Ort::Global<void>::api_ = api;
|
||||
Ort::InitApi(api);
|
||||
|
||||
std::cout << "[EPLoader] ORT loaded successfully." << std::endl;
|
||||
std::cout << "[EPLoader] ORT DLL version : " << base->GetVersionString() << std::endl;
|
||||
|
||||
@@ -141,7 +141,7 @@ namespace ANSCENTER {
|
||||
{
|
||||
try {
|
||||
// Make sure the Ort API pointer is initialised in THIS DLL.
|
||||
if (Ort::Global<void>::api_ == nullptr) {
|
||||
if (Ort::detail::Global::Api() == nullptr) {
|
||||
Ort::InitApi(static_cast<const OrtApi*>(EPLoader::GetOrtApiRaw()));
|
||||
}
|
||||
|
||||
@@ -574,10 +574,10 @@ namespace ANSCENTER {
|
||||
const auto& epInfo = EPLoader::Current();
|
||||
ANS_DBG("OrtHandler", "initialize_handler: EPLoader type=%d dir=%s",
|
||||
static_cast<int>(epInfo.type), epInfo.libraryDir.c_str());
|
||||
if (Ort::Global<void>::api_ == nullptr)
|
||||
if (Ort::detail::Global::Api() == nullptr)
|
||||
Ort::InitApi(static_cast<const OrtApi*>(EPLoader::GetOrtApiRaw()));
|
||||
|
||||
std::cout << "[ORT] api_ = " << (void*)Ort::Global<void>::api_ << std::endl;
|
||||
std::cout << "[ORT] api_ = " << (void*)Ort::detail::Global::Api() << std::endl;
|
||||
|
||||
EngineType engine = (static_cast<int>(m_engineType) == -1)
|
||||
? epInfo.type : m_engineType;
|
||||
@@ -1900,4 +1900,12 @@ namespace ANSCENTER {
|
||||
return std::fabs(width() * height());
|
||||
}
|
||||
|
||||
// Explicit instantiations of every BoundingBoxType member must live
|
||||
// AFTER the template definitions above so the compiler actually emits
|
||||
// symbols (the directives in ONNXEngine.h sit before the definitions
|
||||
// and ClangCL, unlike MSVC, refuses to emit symbols from there).
|
||||
template struct ANSCENTER::types::BoundingBoxType<int, float>;
|
||||
template struct ANSCENTER::types::BoundingBoxType<float, float>;
|
||||
template struct ANSCENTER::types::BoundingBoxType<double, double>;
|
||||
|
||||
} // namespace ANSCENTER
|
||||
@@ -73,9 +73,10 @@ namespace ANSCENTER {
|
||||
}
|
||||
};
|
||||
|
||||
template class BoundingBoxType<int, float>;
|
||||
template class BoundingBoxType<float, float>;
|
||||
template class BoundingBoxType<double, double>;
|
||||
// Explicit instantiations now live at the end of ONNXEngine.cpp
|
||||
// (after the template member definitions) so the compiler actually
|
||||
// emits symbols. ClangCL refuses to emit them from a directive that
|
||||
// precedes the definitions, whereas MSVC silently tolerated it.
|
||||
|
||||
typedef BoundingBoxType<int, float> Boxi;
|
||||
typedef BoundingBoxType<float, float> Boxf;
|
||||
|
||||
@@ -331,7 +331,7 @@ namespace ANSCENTER
|
||||
{
|
||||
// Initialize ORT API
|
||||
const auto& epInfo = EPLoader::Current();
|
||||
if (Ort::Global<void>::api_ == nullptr)
|
||||
if (Ort::detail::Global::Api() == nullptr)
|
||||
Ort::InitApi(static_cast<const OrtApi*>(EPLoader::GetOrtApiRaw()));
|
||||
|
||||
m_env = new Ort::Env(ORT_LOGGING_LEVEL_ERROR, "ONNXSAM3");
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
#include <sstream>
|
||||
#include "Utility.h"
|
||||
#include "TRTCompat.h"
|
||||
#include "GpuMemoryGuard.h"
|
||||
|
||||
// ============================================================================
|
||||
// Crash-safe wrappers for TensorRT operations that can crash the process.
|
||||
@@ -805,13 +806,22 @@ bool Engine<T>::loadNetwork(std::string trtModelPath, const std::array<float, 3>
|
||||
// (cudaMalloc failure during inference) when too many tasks are loaded.
|
||||
{
|
||||
size_t freeVRAM = 0, totalVRAM = 0;
|
||||
cudaError_t memErr = cudaMemGetInfo(&freeVRAM, &totalVRAM);
|
||||
constexpr size_t kMinFreeBytes = 256ULL * 1024 * 1024; // 256 MiB minimum
|
||||
if (memErr != cudaSuccess) {
|
||||
// cudaMemGetInfo failed — CUDA context may not be initialized on this thread.
|
||||
// Don't reject: let TRT try to deserialize (it may succeed).
|
||||
} else if (freeVRAM < kMinFreeBytes) {
|
||||
m_lastLoadFailedVRAM = true; // signal to buildLoadNetwork: engine file is NOT corrupt
|
||||
bool haveMem = ANSCENTER::GpuMemoryGuard::Query(freeVRAM, totalVRAM);
|
||||
// (1) Admission control: refuse to load this engine on the GPU if it
|
||||
// does not fit while keeping the GpuMemoryGuard safety margin free.
|
||||
// Prevents VRAM over-subscription across concurrent engines (FR/OD/LPR/
|
||||
// liveness) that would otherwise OOM and permanently poison the CUDA
|
||||
// context. The engine file is valid — m_lastLoadFailedVRAM tells the
|
||||
// caller to preserve it (no rebuild) and just skip GPU load for now.
|
||||
if (haveMem && !ANSCENTER::GpuMemoryGuard::CanLoad(buffer.size())) {
|
||||
m_lastLoadFailedVRAM = true;
|
||||
std::cerr << "[GpuMemGuard] REFUSING engine load on GPU "
|
||||
<< m_options.deviceIndex << ": need ~"
|
||||
<< (buffer.size() / (1024 * 1024)) << " MiB + "
|
||||
<< (ANSCENTER::GpuMemoryGuard::Margin() / (1024 * 1024))
|
||||
<< " MiB margin, only " << (freeVRAM / (1024 * 1024))
|
||||
<< " MiB free. Reduce concurrent engines/cameras or batch size."
|
||||
<< std::endl;
|
||||
return false;
|
||||
}
|
||||
if (m_verbose) {
|
||||
|
||||
Reference in New Issue
Block a user