diff --git a/engines/TensorRTAPI/include/engine/EngineBuildLoadNetwork.inl b/engines/TensorRTAPI/include/engine/EngineBuildLoadNetwork.inl index f65379c..5a39932 100644 --- a/engines/TensorRTAPI/include/engine/EngineBuildLoadNetwork.inl +++ b/engines/TensorRTAPI/include/engine/EngineBuildLoadNetwork.inl @@ -316,7 +316,7 @@ bool Engine::buildLoadNetwork(std::string onnxModelPath, const std::array::buildLoadNetwork(std::string onnxModelPath, const std::array(TRT_CREATE_NETWORK(tempBuilder)); auto tempParser = std::unique_ptr(nvonnxparser::createParser(*tempNetwork, m_logger)); - std::ifstream onnxFile(onnxModelPath, std::ios::binary | std::ios::ate); - std::streamsize onnxSize = onnxFile.tellg(); - onnxFile.seekg(0, std::ios::beg); - std::vector onnxBuffer(onnxSize); - if (!onnxFile.read(onnxBuffer.data(), onnxSize)) { - std::cout << "Error: Failed to read ONNX file" << std::endl; + std::vector onnxBuffer; + if (!readOnnxBytes(onnxModelPath, onnxBuffer)) { + std::cout << "Error: Failed to read ONNX model" << std::endl; return false; } @@ -1465,7 +1462,7 @@ bool Engine::build(std::string onnxModelPath, const std::array& sub std::cout << "Engine file already exists: " << engineName << std::endl; return true; } - if (!FileExist(onnxModelPath)) { + if (!hasOnnxBytes() && !FileExist(onnxModelPath)) { std::cout << "Error: ONNX model file does not exist: " << onnxModelPath << std::endl; return false; } @@ -1502,16 +1499,14 @@ bool Engine::build(std::string onnxModelPath, const std::array& sub return false; } - // Read the onnx file into memory - std::ifstream file(onnxModelPath, std::ios::binary | std::ios::ate); - std::streamsize size = file.tellg(); - file.seekg(0, std::ios::beg); - - std::vector buffer(size); - if (!file.read(buffer.data(), size)) { - std::cout << "Error: Unable to read ONNX file" << std::endl; + // Get the onnx bytes: from the in-memory buffer of a sealed package, or + // from the file on disk. + std::vector buffer; + if (!readOnnxBytes(onnxModelPath, buffer)) { + std::cout << "Error: Unable to read ONNX model" << std::endl; return false; } + const std::streamsize size = static_cast(buffer.size()); std::cout << "ONNX model size: " << size / (1024 * 1024) << " MiB" << std::endl; @@ -2471,16 +2466,14 @@ bool Engine::buildWithRetry(std::string onnxModelPath, auto tempParser = std::unique_ptr( nvonnxparser::createParser(*tempNetwork, m_logger)); - std::ifstream onnxFile(onnxModelPath, std::ios::binary | std::ios::ate); - if (!onnxFile.good()) { + std::vector onnxBuffer; + const bool onnxReadable = readOnnxBytes(onnxModelPath, onnxBuffer); + if (!onnxReadable) { ANS_DBG("TRT_Build", "buildWithRetry WARN cannot open ONNX for pre-parse path=%s", onnxModelPath.c_str()); } - if (onnxFile.good()) { - std::streamsize onnxSize = onnxFile.tellg(); - onnxFile.seekg(0, std::ios::beg); - std::vector onnxBuffer(onnxSize); - if (onnxFile.read(onnxBuffer.data(), onnxSize)) { + if (onnxReadable) { + { unsigned long sehRetryParse = 0; bool retryParsed = parseOnnxModelSafe(tempParser.get(), onnxBuffer.data(), onnxBuffer.size(), &sehRetryParse); @@ -2508,9 +2501,6 @@ bool Engine::buildWithRetry(std::string onnxModelPath, (int)retryParsed, retryParsed ? tempNetwork->getNbInputs() : -1); } - } else { - ANS_DBG("TRT_Build", "buildWithRetry WARN ONNX read failed bytes=%zd", - (long long)onnxSize); } } } else { diff --git a/engines/TensorRTAPI/include/engine/EnginePoolManager.h b/engines/TensorRTAPI/include/engine/EnginePoolManager.h index c26e1fc..ce9260e 100644 --- a/engines/TensorRTAPI/include/engine/EnginePoolManager.h +++ b/engines/TensorRTAPI/include/engine/EnginePoolManager.h @@ -88,8 +88,20 @@ public: const std::array& subVals, const std::array& divVals, bool normalize, - int maxSlotsPerGpu) + int maxSlotsPerGpu, + const void* onnxData = nullptr, + size_t onnxSize = 0) { + // A sealed .ansmodel is decrypted in memory and never written out, so + // the caller hands the bytes down here: every Engine this call builds + // parses them instead of opening modelPath, which only names the + // TensorRT plan that gets cached. + auto newEngine = [&]() { + auto e = std::make_shared>(options); + if (onnxData != nullptr && onnxSize > 0) e->setOnnxBytes(onnxData, onnxSize); + return e; + }; + // Optimizer / temporary engines: maxSlotsPerGpu==0 means the caller // only needs a lightweight, non-shared engine (e.g., OptimizeModelStr). // Bypass the pool cache entirely: @@ -99,7 +111,7 @@ public: // Note: maxSlotsPerGpu==1 is now the normal "1 slot per GPU" multi-GPU // round-robin mode, so it goes through the pool path below. if (maxSlotsPerGpu == 0) { - auto engine = std::make_shared>(options); + auto engine = newEngine(); bool ok = engine->buildLoadNetwork(modelPath, subVals, divVals, normalize); return ok ? engine : nullptr; } @@ -141,13 +153,13 @@ public: } // Cache miss — create new Engine pool - auto engine = std::make_shared>(options); + auto engine = newEngine(); bool ok = engine->buildLoadNetwork(modelPath, subVals, divVals, normalize, maxSlotsPerGpu); if (!ok) { // Step 1: Force-evict all pools with refcount=0 to reclaim VRAM int evicted = forceEvictPending(); if (evicted > 0) { - engine = std::make_shared>(options); + engine = newEngine(); ok = engine->buildLoadNetwork(modelPath, subVals, divVals, normalize, maxSlotsPerGpu); } @@ -156,7 +168,7 @@ public: // consumes ~300-500 MB vs ~50-100 MB for a simple loadNetwork. // Lightweight mode: tasks queue for a single shared slot — slower but works. if (!ok) { - engine = std::make_shared>(options); + engine = newEngine(); ok = engine->buildLoadNetwork(modelPath, subVals, divVals, normalize); } @@ -183,7 +195,7 @@ public: } // Final retry — try lightweight again after delay - engine = std::make_shared>(options); + engine = newEngine(); ok = engine->buildLoadNetwork(modelPath, subVals, divVals, normalize); } diff --git a/engines/TensorRTAPI/include/engine/EngineUtilities.inl b/engines/TensorRTAPI/include/engine/EngineUtilities.inl index 15b6f6d..8c0abb3 100644 --- a/engines/TensorRTAPI/include/engine/EngineUtilities.inl +++ b/engines/TensorRTAPI/include/engine/EngineUtilities.inl @@ -4,6 +4,48 @@ #include // also defines TRT version macros #include // CUDNN_MAJOR/MINOR/PATCHLEVEL #include // cudaRuntimeGetVersion +#include // memset for wiping decrypted model bytes +#include + +// ============================================================================ +// ONNX bytes in memory +// +// A sealed .ansmodel is decrypted into RAM and stays there: handing the +// parser a buffer is what keeps the plaintext model off the disk. The build +// path still takes a path, because the path names the TensorRT plan that is +// cached beside it — a plan is a built artefact and carries no model. +// ============================================================================ +template +void Engine::setOnnxBytes(const void* data, size_t size) { + clearOnnxBytes(); + if (data == nullptr || size == 0) return; + const char* p = static_cast(data); + m_onnxBytes.assign(p, p + size); +} + +template +void Engine::clearOnnxBytes() { + if (m_onnxBytes.empty()) return; + std::memset(m_onnxBytes.data(), 0, m_onnxBytes.size()); + m_onnxBytes.clear(); + m_onnxBytes.shrink_to_fit(); +} + +template +bool Engine::readOnnxBytes(const std::string& onnxModelPath, + std::vector& buffer) const { + if (!m_onnxBytes.empty()) { + buffer = m_onnxBytes; + return true; + } + std::ifstream file(onnxModelPath, std::ios::binary | std::ios::ate); + if (!file.good()) return false; + const std::streamsize size = file.tellg(); + if (size <= 0) return false; + file.seekg(0, std::ios::beg); + buffer.resize(static_cast(size)); + return static_cast(file.read(buffer.data(), size)); +} template void Engine::transformOutput(std::vector>> &input, std::vector> &output) { diff --git a/modules/ANSODEngine/ANSRTYOLO.cpp b/modules/ANSODEngine/ANSRTYOLO.cpp index 7ccb552..fcba79e 100644 --- a/modules/ANSODEngine/ANSRTYOLO.cpp +++ b/modules/ANSODEngine/ANSRTYOLO.cpp @@ -11,12 +11,37 @@ namespace ANSCENTER { + // ==================================================================== + // Sealed packages (.ansmodel) + // + // TensorRT still writes a plan to disk and still reads it back on the + // next run — that is what makes the second start fast, and a plan is a + // built artefact that carries no model. What must never be written is + // the ONNX itself, so the builder parses the decrypted bytes from + // memory and the path below names nothing that is ever created: it + // only tells the plan where to live and what to be called. The folder + // is the one the zip flow used, so a package converted from a zip + // reuses the plan that zip already built. + // ==================================================================== + std::string ANSRTYOLO::PackagePlanBasePath(const std::string& packagePath) { + const std::string folder = GenerateOutputFolder( + packagePath, GetFileNameWithoutExtension(packagePath), false); + return CreateFilePath(folder, "train_last.onnx"); + } + // ==================================================================== // ANSODBase interface — OptimizeModel // ==================================================================== bool ANSRTYOLO::OptimizeModel(bool fp16, std::string& optimizedModelFolder) { std::lock_guard lock(_mutex); if (!ANSODBase::OptimizeModel(fp16, optimizedModelFolder)) return false; + if (_fromPackage) { + // The plan is built from the decrypted bytes when the engine + // loads; there is no ONNX on disk to pre-optimise. Report the + // folder the plan is cached in so the caller sees a real path. + optimizedModelFolder = GetParentFolder(_modelFilePath); + return true; + } if (!FileExist(_modelFilePath)) { _logger.LogFatal("ANSRTYOLO::OptimizeModel", "Raw model file path does not exist", __FILE__, __LINE__); return false; @@ -96,21 +121,33 @@ namespace ANSCENTER { m_options.engineFileDir = _modelFolder; m_options.precision = (_fp16 ? Precision::FP16 : Precision::FP32); - _modelFilePath = CreateFilePath(_modelFolder, "train_last.onnx"); - if (FileExist(_modelConfigFile)) { - ModelType modelType; - std::vector inputShape; - _classes = ANSUtilityHelper::GetConfigFileContent(_modelConfigFile, modelType, inputShape); - if (inputShape.size() == 2) { - if (inputShape[0] > 0) _modelConfig.inpHeight = inputShape[0]; - if (inputShape[1] > 0) _modelConfig.inpWidth = inputShape[1]; - } + if (_fromPackage) { + // The package already carries the classes and the geometry, + // and there is no folder to look in. All that is needed here + // is where the built plan lives. + _modelFilePath = PackagePlanBasePath(modelZipFilePath); + m_options.engineFileDir = GetParentFolder(_modelFilePath); + // The defaults above have just overwritten the input size the + // package asked for; put it back. + ApplyPackageInputSize(); } else { - _classFilePath = CreateFilePath(_modelFolder, "classes.names"); - std::ifstream isValid(_classFilePath); - if (!isValid) LoadClassesFromString(); - else LoadClassesFromFile(); + _modelFilePath = CreateFilePath(_modelFolder, "train_last.onnx"); + if (FileExist(_modelConfigFile)) { + ModelType modelType; + std::vector inputShape; + _classes = ANSUtilityHelper::GetConfigFileContent(_modelConfigFile, modelType, inputShape); + if (inputShape.size() == 2) { + if (inputShape[0] > 0) _modelConfig.inpHeight = inputShape[0]; + if (inputShape[1] > 0) _modelConfig.inpWidth = inputShape[1]; + } + } + else { + _classFilePath = CreateFilePath(_modelFolder, "classes.names"); + std::ifstream isValid(_classFilePath); + if (!isValid) LoadClassesFromString(); + else LoadClassesFromFile(); + } } if (this->_loadEngineOnCreation) { @@ -119,7 +156,9 @@ namespace ANSCENTER { static_cast(m_options.precision), m_options.maxBatchSize }; m_trtEngine = EnginePoolManager::instance().acquire( m_poolKey, m_options, _modelFilePath, - SUB_VALS, DIV_VALS, NORMALIZE, m_maxSlotsPerGpu); + SUB_VALS, DIV_VALS, NORMALIZE, m_maxSlotsPerGpu, + _fromPackage ? _packageModel.data() : nullptr, + _fromPackage ? _packageModel.size() : 0); m_usingSharedPool = (m_trtEngine != nullptr); } if (!m_trtEngine) { @@ -221,7 +260,9 @@ namespace ANSCENTER { static_cast(m_options.precision), m_options.maxBatchSize }; m_trtEngine = EnginePoolManager::instance().acquire( m_poolKey, m_options, _modelFilePath, - SUB_VALS, DIV_VALS, NORMALIZE, m_maxSlotsPerGpu); + SUB_VALS, DIV_VALS, NORMALIZE, m_maxSlotsPerGpu, + _fromPackage ? _packageModel.data() : nullptr, + _fromPackage ? _packageModel.size() : 0); m_usingSharedPool = (m_trtEngine != nullptr); } if (!m_trtEngine) { @@ -295,21 +336,33 @@ namespace ANSCENTER { m_options.engineFileDir = _modelFolder; m_options.precision = (_fp16 ? Precision::FP16 : Precision::FP32); - _modelFilePath = CreateFilePath(_modelFolder, "train_last.onnx"); - if (FileExist(_modelConfigFile)) { - ModelType modelType; - std::vector inputShape; - _classes = ANSUtilityHelper::GetConfigFileContent(_modelConfigFile, modelType, inputShape); - if (inputShape.size() == 2) { - if (inputShape[0] > 0) _modelConfig.inpHeight = inputShape[0]; - if (inputShape[1] > 0) _modelConfig.inpWidth = inputShape[1]; - } + if (_fromPackage) { + // The package already carries the classes and the geometry, + // and there is no folder to look in. All that is needed here + // is where the built plan lives. + _modelFilePath = PackagePlanBasePath(modelZipFilePath); + m_options.engineFileDir = GetParentFolder(_modelFilePath); + // The defaults above have just overwritten the input size the + // package asked for; put it back. + ApplyPackageInputSize(); } else { - _classFilePath = CreateFilePath(_modelFolder, "classes.names"); - std::ifstream isValid(_classFilePath); - if (!isValid) LoadClassesFromString(); - else LoadClassesFromFile(); + _modelFilePath = CreateFilePath(_modelFolder, "train_last.onnx"); + if (FileExist(_modelConfigFile)) { + ModelType modelType; + std::vector inputShape; + _classes = ANSUtilityHelper::GetConfigFileContent(_modelConfigFile, modelType, inputShape); + if (inputShape.size() == 2) { + if (inputShape[0] > 0) _modelConfig.inpHeight = inputShape[0]; + if (inputShape[1] > 0) _modelConfig.inpWidth = inputShape[1]; + } + } + else { + _classFilePath = CreateFilePath(_modelFolder, "classes.names"); + std::ifstream isValid(_classFilePath); + if (!isValid) LoadClassesFromString(); + else LoadClassesFromFile(); + } } labelMap.clear(); @@ -322,7 +375,9 @@ namespace ANSCENTER { static_cast(m_options.precision), m_options.maxBatchSize }; m_trtEngine = EnginePoolManager::instance().acquire( m_poolKey, m_options, _modelFilePath, - SUB_VALS, DIV_VALS, NORMALIZE, m_maxSlotsPerGpu); + SUB_VALS, DIV_VALS, NORMALIZE, m_maxSlotsPerGpu, + _fromPackage ? _packageModel.data() : nullptr, + _fromPackage ? _packageModel.size() : 0); m_usingSharedPool = (m_trtEngine != nullptr); } if (!m_trtEngine) { diff --git a/modules/ANSODEngine/ANSRTYOLO.h b/modules/ANSODEngine/ANSRTYOLO.h index 1973fba..5fd7b5b 100644 --- a/modules/ANSODEngine/ANSRTYOLO.h +++ b/modules/ANSODEngine/ANSRTYOLO.h @@ -52,6 +52,11 @@ namespace ANSCENTER { ~ANSRTYOLO(); private: + // Where a sealed package's TensorRT plan is cached, and what it is + // called. Nothing but the plan is ever written there — the model + // itself is parsed from memory. + static std::string PackagePlanBasePath(const std::string& packagePath); + std::string _modelFilePath; bool _modelLoadValid{ false }; bool _fp16{ false }; diff --git a/modules/ANSODEngine/dllmain.cpp b/modules/ANSODEngine/dllmain.cpp index 17de06c..609e3e6 100644 --- a/modules/ANSODEngine/dllmain.cpp +++ b/modules/ANSODEngine/dllmain.cpp @@ -8,6 +8,7 @@ #include // INT_MIN #include "ANSLicense.h" // ANS_DBG macro for DebugView #include "ANSODVendorGate.h" // ansod_vendor_gate::IsNvidiaGpuAvailable() +#include "ANSModelOpen.h" // modelpkg::IsPackagePath (sealed .ansmodel) // Process-wide flag: when true, all engines force single-GPU path (no pool, no idle timers). // Defined here, declared extern in EngineBuildLoadNetwork.inl. @@ -1576,6 +1577,13 @@ ANSODENGINE_API int OptimizeModelStr(const char* modelFilePath, const char* mode modelType, modelDetectionType, fp16, modelFilePath ? modelFilePath : "(null)"); try { bool optimizedResult = false; + // A sealed .ansmodel is already the artefact: there is no zip to + // inflate and no loose ONNX to point an optimiser at. It still goes + // through the handle below, which opens it in memory and builds the + // TensorRT plan from the decrypted bytes — what it must not go + // through is ModelOptimizer, which only knows how to unzip. + const bool sealedPackage = + ANSCENTER::modelpkg::IsPackagePath(modelFilePath ? modelFilePath : ""); // NOTE: odMutex was removed here. OptimizeModelStr creates its own // temporary Engine on the stack — no shared state with running // inference tasks. The global mutex was blocking ALL running tasks' @@ -1628,7 +1636,7 @@ ANSODENGINE_API int OptimizeModelStr(const char* modelFilePath, const char* mode } - if (modelType == 31) // If modelType is RTYOLO (31), handle separately. + if (modelType == 31 && !sealedPackage) // If modelType is RTYOLO (31), handle separately. { if (engineType == ANSCENTER::EngineType::NVIDIA_GPU) { if (modelDetectionType == 0) { @@ -1778,7 +1786,10 @@ ANSODENGINE_API int OptimizeModelStr(const char* modelFilePath, const char* mode Handle.reset(); // Destroy engines BEFORE guard restores cache - if (optimizedResult && !modelFolder.empty()) return 1; + // A package reports the folder its plan was cached in; when the + // handle declined to name one there is still nothing wrong, because + // nothing was ever meant to be written. + if (optimizedResult && (!modelFolder.empty() || sealedPackage)) return 1; else return 0; } catch (const std::exception& e) { diff --git a/modules/ANSODEngine/engine.h b/modules/ANSODEngine/engine.h index a6fbc06..64ba90c 100644 --- a/modules/ANSODEngine/engine.h +++ b/modules/ANSODEngine/engine.h @@ -188,6 +188,17 @@ public: /// Full path of the engine file resolved by the last buildLoadNetwork(). [[nodiscard]] const std::string& engineFilePath() const { return m_engineFilePath; } + // ── ONNX bytes held in memory (sealed .ansmodel packages) ─────────────── + // A package is decrypted into memory and the plaintext model must never + // reach the disk, so the parser reads these bytes instead of opening + // onnxModelPath. The path is still passed in and still matters: it names + // the TensorRT plan that gets cached beside it, and a plan is a built + // artefact, not the model. Set the bytes before any build call; they are + // wiped by clearOnnxBytes() and by the destructor. + void setOnnxBytes(const void* data, size_t size); + void clearOnnxBytes(); + [[nodiscard]] bool hasOnnxBytes() const { return !m_onnxBytes.empty(); } + /// Return the CUDA device index that preprocessing should target. /// In pool mode this returns the device of the first alive slot; /// in single-engine mode it returns m_options.deviceIndex. @@ -244,6 +255,9 @@ public: // retries until build succeeds or all candidates are exhausted. // Fixed-spatial models fall through to a single build() call. bool buildWithRetry(std::string onnxModelPath, const std::array& subVals, const std::array& divVals, bool normalize); + // Where the ONNX bytes come from: the in-memory buffer when one was set, + // otherwise the file at onnxModelPath. + bool readOnnxBytes(const std::string& onnxModelPath, std::vector& buffer) const; void getDeviceNames(std::vector& deviceNames); void clearGpuBuffers(); int getBindingIndexByName(const std::string& name); @@ -401,6 +415,7 @@ private: bool m_usingCachedEngine = false; bool m_skipEngineCache = false; // true = bypass TRTEngineCache (for OptimizeModelStr) bool m_skipOnnxRebuild = false; // true = don't delete/rebuild corrupt engine files (elastic growth) + std::vector m_onnxBytes; // model bytes in memory (package); empty = read from disk bool m_rebuiltFromOnnx = false; // last buildLoadNetwork built from ONNX (no matching cached engine) std::string m_engineFilePath; // engine file resolved by last buildLoadNetwork bool m_skipOnnxBuild = false; // true = skip ONNX→TRT build if no cached engine (demand growth) @@ -575,6 +590,8 @@ template Engine::Engine() { m_lastSetInputDims.clear(); } template Engine::~Engine() { + // A decrypted model must not outlive the engine that parsed it. + try { clearOnnxBytes(); } catch (...) {} // ========================================================================= // CRITICAL: Destructors must NEVER throw. Every CUDA/TRT call below is // wrapped in try/catch so that a failure in one step does not skip the