From cd99fcf6d9b5da4af3a4075557de0b9ddee2918b Mon Sep 17 00:00:00 2001 From: Tuan Nghia Nguyen Date: Sun, 13 Sep 2026 09:38:45 +1000 Subject: [PATCH] feat(ANSODEngine): run a sealed model on TensorRT without unsealing it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A .ansmodel reaching the GPU path went to ModelOptimizer, which knows one way to open a model — unzip it — and failed on the first byte. Even past that, the TensorRT engine only ever parsed an ONNX file, so the one thing the format exists to prevent, a decrypted model on disk, was the only way it could have run. The builder now takes its bytes from memory when it has them. The path it is given still names the plan that gets cached beside it, because a plan is a built artefact and carries no model: the second run loads it and never needs the ONNX at all. For a package the plan lands in the folder the zip flow used, so a model converted from a zip reuses the plan that zip already built. OptimizeModelStr recognises a package and skips the unzip-and-optimise leg entirely: the handle below it opens the package in memory and builds the plan from the decrypted bytes, which is the same work by a shorter road. Still on the file path: the other TensorRT handlers (segmentation, pose, classification). Only the YOLO detector reads a package today. Co-Authored-By: Claude Opus 5 --- .../include/engine/EngineBuildLoadNetwork.inl | 42 +++---- .../include/engine/EnginePoolManager.h | 24 +++- .../include/engine/EngineUtilities.inl | 42 +++++++ modules/ANSODEngine/ANSRTYOLO.cpp | 113 +++++++++++++----- modules/ANSODEngine/ANSRTYOLO.h | 5 + modules/ANSODEngine/dllmain.cpp | 15 ++- modules/ANSODEngine/engine.h | 17 +++ 7 files changed, 195 insertions(+), 63 deletions(-) diff --git a/engines/TensorRTAPI/include/engine/EngineBuildLoadNetwork.inl b/engines/TensorRTAPI/include/engine/EngineBuildLoadNetwork.inl index f65379c..5a39932 100644 --- a/engines/TensorRTAPI/include/engine/EngineBuildLoadNetwork.inl +++ b/engines/TensorRTAPI/include/engine/EngineBuildLoadNetwork.inl @@ -316,7 +316,7 @@ bool Engine::buildLoadNetwork(std::string onnxModelPath, const std::array::buildLoadNetwork(std::string onnxModelPath, const std::array(TRT_CREATE_NETWORK(tempBuilder)); auto tempParser = std::unique_ptr(nvonnxparser::createParser(*tempNetwork, m_logger)); - std::ifstream onnxFile(onnxModelPath, std::ios::binary | std::ios::ate); - std::streamsize onnxSize = onnxFile.tellg(); - onnxFile.seekg(0, std::ios::beg); - std::vector onnxBuffer(onnxSize); - if (!onnxFile.read(onnxBuffer.data(), onnxSize)) { - std::cout << "Error: Failed to read ONNX file" << std::endl; + std::vector onnxBuffer; + if (!readOnnxBytes(onnxModelPath, onnxBuffer)) { + std::cout << "Error: Failed to read ONNX model" << std::endl; return false; } @@ -1465,7 +1462,7 @@ bool Engine::build(std::string onnxModelPath, const std::array& sub std::cout << "Engine file already exists: " << engineName << std::endl; return true; } - if (!FileExist(onnxModelPath)) { + if (!hasOnnxBytes() && !FileExist(onnxModelPath)) { std::cout << "Error: ONNX model file does not exist: " << onnxModelPath << std::endl; return false; } @@ -1502,16 +1499,14 @@ bool Engine::build(std::string onnxModelPath, const std::array& sub return false; } - // Read the onnx file into memory - std::ifstream file(onnxModelPath, std::ios::binary | std::ios::ate); - std::streamsize size = file.tellg(); - file.seekg(0, std::ios::beg); - - std::vector buffer(size); - if (!file.read(buffer.data(), size)) { - std::cout << "Error: Unable to read ONNX file" << std::endl; + // Get the onnx bytes: from the in-memory buffer of a sealed package, or + // from the file on disk. + std::vector buffer; + if (!readOnnxBytes(onnxModelPath, buffer)) { + std::cout << "Error: Unable to read ONNX model" << std::endl; return false; } + const std::streamsize size = static_cast(buffer.size()); std::cout << "ONNX model size: " << size / (1024 * 1024) << " MiB" << std::endl; @@ -2471,16 +2466,14 @@ bool Engine::buildWithRetry(std::string onnxModelPath, auto tempParser = std::unique_ptr( nvonnxparser::createParser(*tempNetwork, m_logger)); - std::ifstream onnxFile(onnxModelPath, std::ios::binary | std::ios::ate); - if (!onnxFile.good()) { + std::vector onnxBuffer; + const bool onnxReadable = readOnnxBytes(onnxModelPath, onnxBuffer); + if (!onnxReadable) { ANS_DBG("TRT_Build", "buildWithRetry WARN cannot open ONNX for pre-parse path=%s", onnxModelPath.c_str()); } - if (onnxFile.good()) { - std::streamsize onnxSize = onnxFile.tellg(); - onnxFile.seekg(0, std::ios::beg); - std::vector onnxBuffer(onnxSize); - if (onnxFile.read(onnxBuffer.data(), onnxSize)) { + if (onnxReadable) { + { unsigned long sehRetryParse = 0; bool retryParsed = parseOnnxModelSafe(tempParser.get(), onnxBuffer.data(), onnxBuffer.size(), &sehRetryParse); @@ -2508,9 +2501,6 @@ bool Engine::buildWithRetry(std::string onnxModelPath, (int)retryParsed, retryParsed ? tempNetwork->getNbInputs() : -1); } - } else { - ANS_DBG("TRT_Build", "buildWithRetry WARN ONNX read failed bytes=%zd", - (long long)onnxSize); } } } else { diff --git a/engines/TensorRTAPI/include/engine/EnginePoolManager.h b/engines/TensorRTAPI/include/engine/EnginePoolManager.h index c26e1fc..ce9260e 100644 --- a/engines/TensorRTAPI/include/engine/EnginePoolManager.h +++ b/engines/TensorRTAPI/include/engine/EnginePoolManager.h @@ -88,8 +88,20 @@ public: const std::array& subVals, const std::array& divVals, bool normalize, - int maxSlotsPerGpu) + int maxSlotsPerGpu, + const void* onnxData = nullptr, + size_t onnxSize = 0) { + // A sealed .ansmodel is decrypted in memory and never written out, so + // the caller hands the bytes down here: every Engine this call builds + // parses them instead of opening modelPath, which only names the + // TensorRT plan that gets cached. + auto newEngine = [&]() { + auto e = std::make_shared>(options); + if (onnxData != nullptr && onnxSize > 0) e->setOnnxBytes(onnxData, onnxSize); + return e; + }; + // Optimizer / temporary engines: maxSlotsPerGpu==0 means the caller // only needs a lightweight, non-shared engine (e.g., OptimizeModelStr). // Bypass the pool cache entirely: @@ -99,7 +111,7 @@ public: // Note: maxSlotsPerGpu==1 is now the normal "1 slot per GPU" multi-GPU // round-robin mode, so it goes through the pool path below. if (maxSlotsPerGpu == 0) { - auto engine = std::make_shared>(options); + auto engine = newEngine(); bool ok = engine->buildLoadNetwork(modelPath, subVals, divVals, normalize); return ok ? engine : nullptr; } @@ -141,13 +153,13 @@ public: } // Cache miss — create new Engine pool - auto engine = std::make_shared>(options); + auto engine = newEngine(); bool ok = engine->buildLoadNetwork(modelPath, subVals, divVals, normalize, maxSlotsPerGpu); if (!ok) { // Step 1: Force-evict all pools with refcount=0 to reclaim VRAM int evicted = forceEvictPending(); if (evicted > 0) { - engine = std::make_shared>(options); + engine = newEngine(); ok = engine->buildLoadNetwork(modelPath, subVals, divVals, normalize, maxSlotsPerGpu); } @@ -156,7 +168,7 @@ public: // consumes ~300-500 MB vs ~50-100 MB for a simple loadNetwork. // Lightweight mode: tasks queue for a single shared slot — slower but works. if (!ok) { - engine = std::make_shared>(options); + engine = newEngine(); ok = engine->buildLoadNetwork(modelPath, subVals, divVals, normalize); } @@ -183,7 +195,7 @@ public: } // Final retry — try lightweight again after delay - engine = std::make_shared>(options); + engine = newEngine(); ok = engine->buildLoadNetwork(modelPath, subVals, divVals, normalize); } diff --git a/engines/TensorRTAPI/include/engine/EngineUtilities.inl b/engines/TensorRTAPI/include/engine/EngineUtilities.inl index 15b6f6d..8c0abb3 100644 --- a/engines/TensorRTAPI/include/engine/EngineUtilities.inl +++ b/engines/TensorRTAPI/include/engine/EngineUtilities.inl @@ -4,6 +4,48 @@ #include // also defines TRT version macros #include // CUDNN_MAJOR/MINOR/PATCHLEVEL #include // cudaRuntimeGetVersion +#include // memset for wiping decrypted model bytes +#include + +// ============================================================================ +// ONNX bytes in memory +// +// A sealed .ansmodel is decrypted into RAM and stays there: handing the +// parser a buffer is what keeps the plaintext model off the disk. The build +// path still takes a path, because the path names the TensorRT plan that is +// cached beside it — a plan is a built artefact and carries no model. +// ============================================================================ +template +void Engine::setOnnxBytes(const void* data, size_t size) { + clearOnnxBytes(); + if (data == nullptr || size == 0) return; + const char* p = static_cast(data); + m_onnxBytes.assign(p, p + size); +} + +template +void Engine::clearOnnxBytes() { + if (m_onnxBytes.empty()) return; + std::memset(m_onnxBytes.data(), 0, m_onnxBytes.size()); + m_onnxBytes.clear(); + m_onnxBytes.shrink_to_fit(); +} + +template +bool Engine::readOnnxBytes(const std::string& onnxModelPath, + std::vector& buffer) const { + if (!m_onnxBytes.empty()) { + buffer = m_onnxBytes; + return true; + } + std::ifstream file(onnxModelPath, std::ios::binary | std::ios::ate); + if (!file.good()) return false; + const std::streamsize size = file.tellg(); + if (size <= 0) return false; + file.seekg(0, std::ios::beg); + buffer.resize(static_cast(size)); + return static_cast(file.read(buffer.data(), size)); +} template void Engine::transformOutput(std::vector>> &input, std::vector> &output) { diff --git a/modules/ANSODEngine/ANSRTYOLO.cpp b/modules/ANSODEngine/ANSRTYOLO.cpp index 7ccb552..fcba79e 100644 --- a/modules/ANSODEngine/ANSRTYOLO.cpp +++ b/modules/ANSODEngine/ANSRTYOLO.cpp @@ -11,12 +11,37 @@ namespace ANSCENTER { + // ==================================================================== + // Sealed packages (.ansmodel) + // + // TensorRT still writes a plan to disk and still reads it back on the + // next run — that is what makes the second start fast, and a plan is a + // built artefact that carries no model. What must never be written is + // the ONNX itself, so the builder parses the decrypted bytes from + // memory and the path below names nothing that is ever created: it + // only tells the plan where to live and what to be called. The folder + // is the one the zip flow used, so a package converted from a zip + // reuses the plan that zip already built. + // ==================================================================== + std::string ANSRTYOLO::PackagePlanBasePath(const std::string& packagePath) { + const std::string folder = GenerateOutputFolder( + packagePath, GetFileNameWithoutExtension(packagePath), false); + return CreateFilePath(folder, "train_last.onnx"); + } + // ==================================================================== // ANSODBase interface — OptimizeModel // ==================================================================== bool ANSRTYOLO::OptimizeModel(bool fp16, std::string& optimizedModelFolder) { std::lock_guard lock(_mutex); if (!ANSODBase::OptimizeModel(fp16, optimizedModelFolder)) return false; + if (_fromPackage) { + // The plan is built from the decrypted bytes when the engine + // loads; there is no ONNX on disk to pre-optimise. Report the + // folder the plan is cached in so the caller sees a real path. + optimizedModelFolder = GetParentFolder(_modelFilePath); + return true; + } if (!FileExist(_modelFilePath)) { _logger.LogFatal("ANSRTYOLO::OptimizeModel", "Raw model file path does not exist", __FILE__, __LINE__); return false; @@ -96,21 +121,33 @@ namespace ANSCENTER { m_options.engineFileDir = _modelFolder; m_options.precision = (_fp16 ? Precision::FP16 : Precision::FP32); - _modelFilePath = CreateFilePath(_modelFolder, "train_last.onnx"); - if (FileExist(_modelConfigFile)) { - ModelType modelType; - std::vector inputShape; - _classes = ANSUtilityHelper::GetConfigFileContent(_modelConfigFile, modelType, inputShape); - if (inputShape.size() == 2) { - if (inputShape[0] > 0) _modelConfig.inpHeight = inputShape[0]; - if (inputShape[1] > 0) _modelConfig.inpWidth = inputShape[1]; - } + if (_fromPackage) { + // The package already carries the classes and the geometry, + // and there is no folder to look in. All that is needed here + // is where the built plan lives. + _modelFilePath = PackagePlanBasePath(modelZipFilePath); + m_options.engineFileDir = GetParentFolder(_modelFilePath); + // The defaults above have just overwritten the input size the + // package asked for; put it back. + ApplyPackageInputSize(); } else { - _classFilePath = CreateFilePath(_modelFolder, "classes.names"); - std::ifstream isValid(_classFilePath); - if (!isValid) LoadClassesFromString(); - else LoadClassesFromFile(); + _modelFilePath = CreateFilePath(_modelFolder, "train_last.onnx"); + if (FileExist(_modelConfigFile)) { + ModelType modelType; + std::vector inputShape; + _classes = ANSUtilityHelper::GetConfigFileContent(_modelConfigFile, modelType, inputShape); + if (inputShape.size() == 2) { + if (inputShape[0] > 0) _modelConfig.inpHeight = inputShape[0]; + if (inputShape[1] > 0) _modelConfig.inpWidth = inputShape[1]; + } + } + else { + _classFilePath = CreateFilePath(_modelFolder, "classes.names"); + std::ifstream isValid(_classFilePath); + if (!isValid) LoadClassesFromString(); + else LoadClassesFromFile(); + } } if (this->_loadEngineOnCreation) { @@ -119,7 +156,9 @@ namespace ANSCENTER { static_cast(m_options.precision), m_options.maxBatchSize }; m_trtEngine = EnginePoolManager::instance().acquire( m_poolKey, m_options, _modelFilePath, - SUB_VALS, DIV_VALS, NORMALIZE, m_maxSlotsPerGpu); + SUB_VALS, DIV_VALS, NORMALIZE, m_maxSlotsPerGpu, + _fromPackage ? _packageModel.data() : nullptr, + _fromPackage ? _packageModel.size() : 0); m_usingSharedPool = (m_trtEngine != nullptr); } if (!m_trtEngine) { @@ -221,7 +260,9 @@ namespace ANSCENTER { static_cast(m_options.precision), m_options.maxBatchSize }; m_trtEngine = EnginePoolManager::instance().acquire( m_poolKey, m_options, _modelFilePath, - SUB_VALS, DIV_VALS, NORMALIZE, m_maxSlotsPerGpu); + SUB_VALS, DIV_VALS, NORMALIZE, m_maxSlotsPerGpu, + _fromPackage ? _packageModel.data() : nullptr, + _fromPackage ? _packageModel.size() : 0); m_usingSharedPool = (m_trtEngine != nullptr); } if (!m_trtEngine) { @@ -295,21 +336,33 @@ namespace ANSCENTER { m_options.engineFileDir = _modelFolder; m_options.precision = (_fp16 ? Precision::FP16 : Precision::FP32); - _modelFilePath = CreateFilePath(_modelFolder, "train_last.onnx"); - if (FileExist(_modelConfigFile)) { - ModelType modelType; - std::vector inputShape; - _classes = ANSUtilityHelper::GetConfigFileContent(_modelConfigFile, modelType, inputShape); - if (inputShape.size() == 2) { - if (inputShape[0] > 0) _modelConfig.inpHeight = inputShape[0]; - if (inputShape[1] > 0) _modelConfig.inpWidth = inputShape[1]; - } + if (_fromPackage) { + // The package already carries the classes and the geometry, + // and there is no folder to look in. All that is needed here + // is where the built plan lives. + _modelFilePath = PackagePlanBasePath(modelZipFilePath); + m_options.engineFileDir = GetParentFolder(_modelFilePath); + // The defaults above have just overwritten the input size the + // package asked for; put it back. + ApplyPackageInputSize(); } else { - _classFilePath = CreateFilePath(_modelFolder, "classes.names"); - std::ifstream isValid(_classFilePath); - if (!isValid) LoadClassesFromString(); - else LoadClassesFromFile(); + _modelFilePath = CreateFilePath(_modelFolder, "train_last.onnx"); + if (FileExist(_modelConfigFile)) { + ModelType modelType; + std::vector inputShape; + _classes = ANSUtilityHelper::GetConfigFileContent(_modelConfigFile, modelType, inputShape); + if (inputShape.size() == 2) { + if (inputShape[0] > 0) _modelConfig.inpHeight = inputShape[0]; + if (inputShape[1] > 0) _modelConfig.inpWidth = inputShape[1]; + } + } + else { + _classFilePath = CreateFilePath(_modelFolder, "classes.names"); + std::ifstream isValid(_classFilePath); + if (!isValid) LoadClassesFromString(); + else LoadClassesFromFile(); + } } labelMap.clear(); @@ -322,7 +375,9 @@ namespace ANSCENTER { static_cast(m_options.precision), m_options.maxBatchSize }; m_trtEngine = EnginePoolManager::instance().acquire( m_poolKey, m_options, _modelFilePath, - SUB_VALS, DIV_VALS, NORMALIZE, m_maxSlotsPerGpu); + SUB_VALS, DIV_VALS, NORMALIZE, m_maxSlotsPerGpu, + _fromPackage ? _packageModel.data() : nullptr, + _fromPackage ? _packageModel.size() : 0); m_usingSharedPool = (m_trtEngine != nullptr); } if (!m_trtEngine) { diff --git a/modules/ANSODEngine/ANSRTYOLO.h b/modules/ANSODEngine/ANSRTYOLO.h index 1973fba..5fd7b5b 100644 --- a/modules/ANSODEngine/ANSRTYOLO.h +++ b/modules/ANSODEngine/ANSRTYOLO.h @@ -52,6 +52,11 @@ namespace ANSCENTER { ~ANSRTYOLO(); private: + // Where a sealed package's TensorRT plan is cached, and what it is + // called. Nothing but the plan is ever written there — the model + // itself is parsed from memory. + static std::string PackagePlanBasePath(const std::string& packagePath); + std::string _modelFilePath; bool _modelLoadValid{ false }; bool _fp16{ false }; diff --git a/modules/ANSODEngine/dllmain.cpp b/modules/ANSODEngine/dllmain.cpp index 17de06c..609e3e6 100644 --- a/modules/ANSODEngine/dllmain.cpp +++ b/modules/ANSODEngine/dllmain.cpp @@ -8,6 +8,7 @@ #include // INT_MIN #include "ANSLicense.h" // ANS_DBG macro for DebugView #include "ANSODVendorGate.h" // ansod_vendor_gate::IsNvidiaGpuAvailable() +#include "ANSModelOpen.h" // modelpkg::IsPackagePath (sealed .ansmodel) // Process-wide flag: when true, all engines force single-GPU path (no pool, no idle timers). // Defined here, declared extern in EngineBuildLoadNetwork.inl. @@ -1576,6 +1577,13 @@ ANSODENGINE_API int OptimizeModelStr(const char* modelFilePath, const char* mode modelType, modelDetectionType, fp16, modelFilePath ? modelFilePath : "(null)"); try { bool optimizedResult = false; + // A sealed .ansmodel is already the artefact: there is no zip to + // inflate and no loose ONNX to point an optimiser at. It still goes + // through the handle below, which opens it in memory and builds the + // TensorRT plan from the decrypted bytes — what it must not go + // through is ModelOptimizer, which only knows how to unzip. + const bool sealedPackage = + ANSCENTER::modelpkg::IsPackagePath(modelFilePath ? modelFilePath : ""); // NOTE: odMutex was removed here. OptimizeModelStr creates its own // temporary Engine on the stack — no shared state with running // inference tasks. The global mutex was blocking ALL running tasks' @@ -1628,7 +1636,7 @@ ANSODENGINE_API int OptimizeModelStr(const char* modelFilePath, const char* mode } - if (modelType == 31) // If modelType is RTYOLO (31), handle separately. + if (modelType == 31 && !sealedPackage) // If modelType is RTYOLO (31), handle separately. { if (engineType == ANSCENTER::EngineType::NVIDIA_GPU) { if (modelDetectionType == 0) { @@ -1778,7 +1786,10 @@ ANSODENGINE_API int OptimizeModelStr(const char* modelFilePath, const char* mode Handle.reset(); // Destroy engines BEFORE guard restores cache - if (optimizedResult && !modelFolder.empty()) return 1; + // A package reports the folder its plan was cached in; when the + // handle declined to name one there is still nothing wrong, because + // nothing was ever meant to be written. + if (optimizedResult && (!modelFolder.empty() || sealedPackage)) return 1; else return 0; } catch (const std::exception& e) { diff --git a/modules/ANSODEngine/engine.h b/modules/ANSODEngine/engine.h index a6fbc06..64ba90c 100644 --- a/modules/ANSODEngine/engine.h +++ b/modules/ANSODEngine/engine.h @@ -188,6 +188,17 @@ public: /// Full path of the engine file resolved by the last buildLoadNetwork(). [[nodiscard]] const std::string& engineFilePath() const { return m_engineFilePath; } + // ── ONNX bytes held in memory (sealed .ansmodel packages) ─────────────── + // A package is decrypted into memory and the plaintext model must never + // reach the disk, so the parser reads these bytes instead of opening + // onnxModelPath. The path is still passed in and still matters: it names + // the TensorRT plan that gets cached beside it, and a plan is a built + // artefact, not the model. Set the bytes before any build call; they are + // wiped by clearOnnxBytes() and by the destructor. + void setOnnxBytes(const void* data, size_t size); + void clearOnnxBytes(); + [[nodiscard]] bool hasOnnxBytes() const { return !m_onnxBytes.empty(); } + /// Return the CUDA device index that preprocessing should target. /// In pool mode this returns the device of the first alive slot; /// in single-engine mode it returns m_options.deviceIndex. @@ -244,6 +255,9 @@ public: // retries until build succeeds or all candidates are exhausted. // Fixed-spatial models fall through to a single build() call. bool buildWithRetry(std::string onnxModelPath, const std::array& subVals, const std::array& divVals, bool normalize); + // Where the ONNX bytes come from: the in-memory buffer when one was set, + // otherwise the file at onnxModelPath. + bool readOnnxBytes(const std::string& onnxModelPath, std::vector& buffer) const; void getDeviceNames(std::vector& deviceNames); void clearGpuBuffers(); int getBindingIndexByName(const std::string& name); @@ -401,6 +415,7 @@ private: bool m_usingCachedEngine = false; bool m_skipEngineCache = false; // true = bypass TRTEngineCache (for OptimizeModelStr) bool m_skipOnnxRebuild = false; // true = don't delete/rebuild corrupt engine files (elastic growth) + std::vector m_onnxBytes; // model bytes in memory (package); empty = read from disk bool m_rebuiltFromOnnx = false; // last buildLoadNetwork built from ONNX (no matching cached engine) std::string m_engineFilePath; // engine file resolved by last buildLoadNetwork bool m_skipOnnxBuild = false; // true = skip ONNX→TRT build if no cached engine (demand growth) @@ -575,6 +590,8 @@ template Engine::Engine() { m_lastSetInputDims.clear(); } template Engine::~Engine() { + // A decrypted model must not outlive the engine that parsed it. + try { clearOnnxBytes(); } catch (...) {} // ========================================================================= // CRITICAL: Destructors must NEVER throw. Every CUDA/TRT call below is // wrapped in try/catch so that a failure in one step does not skip the