feat(ANSODEngine): run a sealed model on TensorRT without unsealing it
A .ansmodel reaching the GPU path went to ModelOptimizer, which knows one way to open a model — unzip it — and failed on the first byte. Even past that, the TensorRT engine only ever parsed an ONNX file, so the one thing the format exists to prevent, a decrypted model on disk, was the only way it could have run. The builder now takes its bytes from memory when it has them. The path it is given still names the plan that gets cached beside it, because a plan is a built artefact and carries no model: the second run loads it and never needs the ONNX at all. For a package the plan lands in the folder the zip flow used, so a model converted from a zip reuses the plan that zip already built. OptimizeModelStr recognises a package and skips the unzip-and-optimise leg entirely: the handle below it opens the package in memory and builds the plan from the decrypted bytes, which is the same work by a shorter road. Still on the file path: the other TensorRT handlers (segmentation, pose, classification). Only the YOLO detector reads a package today. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
@@ -11,12 +11,37 @@
|
||||
|
||||
namespace ANSCENTER {
|
||||
|
||||
// ====================================================================
|
||||
// Sealed packages (.ansmodel)
|
||||
//
|
||||
// TensorRT still writes a plan to disk and still reads it back on the
|
||||
// next run — that is what makes the second start fast, and a plan is a
|
||||
// built artefact that carries no model. What must never be written is
|
||||
// the ONNX itself, so the builder parses the decrypted bytes from
|
||||
// memory and the path below names nothing that is ever created: it
|
||||
// only tells the plan where to live and what to be called. The folder
|
||||
// is the one the zip flow used, so a package converted from a zip
|
||||
// reuses the plan that zip already built.
|
||||
// ====================================================================
|
||||
std::string ANSRTYOLO::PackagePlanBasePath(const std::string& packagePath) {
|
||||
const std::string folder = GenerateOutputFolder(
|
||||
packagePath, GetFileNameWithoutExtension(packagePath), false);
|
||||
return CreateFilePath(folder, "train_last.onnx");
|
||||
}
|
||||
|
||||
// ====================================================================
|
||||
// ANSODBase interface — OptimizeModel
|
||||
// ====================================================================
|
||||
bool ANSRTYOLO::OptimizeModel(bool fp16, std::string& optimizedModelFolder) {
|
||||
std::lock_guard<std::recursive_mutex> lock(_mutex);
|
||||
if (!ANSODBase::OptimizeModel(fp16, optimizedModelFolder)) return false;
|
||||
if (_fromPackage) {
|
||||
// The plan is built from the decrypted bytes when the engine
|
||||
// loads; there is no ONNX on disk to pre-optimise. Report the
|
||||
// folder the plan is cached in so the caller sees a real path.
|
||||
optimizedModelFolder = GetParentFolder(_modelFilePath);
|
||||
return true;
|
||||
}
|
||||
if (!FileExist(_modelFilePath)) {
|
||||
_logger.LogFatal("ANSRTYOLO::OptimizeModel", "Raw model file path does not exist", __FILE__, __LINE__);
|
||||
return false;
|
||||
@@ -96,21 +121,33 @@ namespace ANSCENTER {
|
||||
m_options.engineFileDir = _modelFolder;
|
||||
m_options.precision = (_fp16 ? Precision::FP16 : Precision::FP32);
|
||||
|
||||
_modelFilePath = CreateFilePath(_modelFolder, "train_last.onnx");
|
||||
if (FileExist(_modelConfigFile)) {
|
||||
ModelType modelType;
|
||||
std::vector<int> inputShape;
|
||||
_classes = ANSUtilityHelper::GetConfigFileContent(_modelConfigFile, modelType, inputShape);
|
||||
if (inputShape.size() == 2) {
|
||||
if (inputShape[0] > 0) _modelConfig.inpHeight = inputShape[0];
|
||||
if (inputShape[1] > 0) _modelConfig.inpWidth = inputShape[1];
|
||||
}
|
||||
if (_fromPackage) {
|
||||
// The package already carries the classes and the geometry,
|
||||
// and there is no folder to look in. All that is needed here
|
||||
// is where the built plan lives.
|
||||
_modelFilePath = PackagePlanBasePath(modelZipFilePath);
|
||||
m_options.engineFileDir = GetParentFolder(_modelFilePath);
|
||||
// The defaults above have just overwritten the input size the
|
||||
// package asked for; put it back.
|
||||
ApplyPackageInputSize();
|
||||
}
|
||||
else {
|
||||
_classFilePath = CreateFilePath(_modelFolder, "classes.names");
|
||||
std::ifstream isValid(_classFilePath);
|
||||
if (!isValid) LoadClassesFromString();
|
||||
else LoadClassesFromFile();
|
||||
_modelFilePath = CreateFilePath(_modelFolder, "train_last.onnx");
|
||||
if (FileExist(_modelConfigFile)) {
|
||||
ModelType modelType;
|
||||
std::vector<int> inputShape;
|
||||
_classes = ANSUtilityHelper::GetConfigFileContent(_modelConfigFile, modelType, inputShape);
|
||||
if (inputShape.size() == 2) {
|
||||
if (inputShape[0] > 0) _modelConfig.inpHeight = inputShape[0];
|
||||
if (inputShape[1] > 0) _modelConfig.inpWidth = inputShape[1];
|
||||
}
|
||||
}
|
||||
else {
|
||||
_classFilePath = CreateFilePath(_modelFolder, "classes.names");
|
||||
std::ifstream isValid(_classFilePath);
|
||||
if (!isValid) LoadClassesFromString();
|
||||
else LoadClassesFromFile();
|
||||
}
|
||||
}
|
||||
|
||||
if (this->_loadEngineOnCreation) {
|
||||
@@ -119,7 +156,9 @@ namespace ANSCENTER {
|
||||
static_cast<int>(m_options.precision), m_options.maxBatchSize };
|
||||
m_trtEngine = EnginePoolManager<float>::instance().acquire(
|
||||
m_poolKey, m_options, _modelFilePath,
|
||||
SUB_VALS, DIV_VALS, NORMALIZE, m_maxSlotsPerGpu);
|
||||
SUB_VALS, DIV_VALS, NORMALIZE, m_maxSlotsPerGpu,
|
||||
_fromPackage ? _packageModel.data() : nullptr,
|
||||
_fromPackage ? _packageModel.size() : 0);
|
||||
m_usingSharedPool = (m_trtEngine != nullptr);
|
||||
}
|
||||
if (!m_trtEngine) {
|
||||
@@ -221,7 +260,9 @@ namespace ANSCENTER {
|
||||
static_cast<int>(m_options.precision), m_options.maxBatchSize };
|
||||
m_trtEngine = EnginePoolManager<float>::instance().acquire(
|
||||
m_poolKey, m_options, _modelFilePath,
|
||||
SUB_VALS, DIV_VALS, NORMALIZE, m_maxSlotsPerGpu);
|
||||
SUB_VALS, DIV_VALS, NORMALIZE, m_maxSlotsPerGpu,
|
||||
_fromPackage ? _packageModel.data() : nullptr,
|
||||
_fromPackage ? _packageModel.size() : 0);
|
||||
m_usingSharedPool = (m_trtEngine != nullptr);
|
||||
}
|
||||
if (!m_trtEngine) {
|
||||
@@ -295,21 +336,33 @@ namespace ANSCENTER {
|
||||
m_options.engineFileDir = _modelFolder;
|
||||
m_options.precision = (_fp16 ? Precision::FP16 : Precision::FP32);
|
||||
|
||||
_modelFilePath = CreateFilePath(_modelFolder, "train_last.onnx");
|
||||
if (FileExist(_modelConfigFile)) {
|
||||
ModelType modelType;
|
||||
std::vector<int> inputShape;
|
||||
_classes = ANSUtilityHelper::GetConfigFileContent(_modelConfigFile, modelType, inputShape);
|
||||
if (inputShape.size() == 2) {
|
||||
if (inputShape[0] > 0) _modelConfig.inpHeight = inputShape[0];
|
||||
if (inputShape[1] > 0) _modelConfig.inpWidth = inputShape[1];
|
||||
}
|
||||
if (_fromPackage) {
|
||||
// The package already carries the classes and the geometry,
|
||||
// and there is no folder to look in. All that is needed here
|
||||
// is where the built plan lives.
|
||||
_modelFilePath = PackagePlanBasePath(modelZipFilePath);
|
||||
m_options.engineFileDir = GetParentFolder(_modelFilePath);
|
||||
// The defaults above have just overwritten the input size the
|
||||
// package asked for; put it back.
|
||||
ApplyPackageInputSize();
|
||||
}
|
||||
else {
|
||||
_classFilePath = CreateFilePath(_modelFolder, "classes.names");
|
||||
std::ifstream isValid(_classFilePath);
|
||||
if (!isValid) LoadClassesFromString();
|
||||
else LoadClassesFromFile();
|
||||
_modelFilePath = CreateFilePath(_modelFolder, "train_last.onnx");
|
||||
if (FileExist(_modelConfigFile)) {
|
||||
ModelType modelType;
|
||||
std::vector<int> inputShape;
|
||||
_classes = ANSUtilityHelper::GetConfigFileContent(_modelConfigFile, modelType, inputShape);
|
||||
if (inputShape.size() == 2) {
|
||||
if (inputShape[0] > 0) _modelConfig.inpHeight = inputShape[0];
|
||||
if (inputShape[1] > 0) _modelConfig.inpWidth = inputShape[1];
|
||||
}
|
||||
}
|
||||
else {
|
||||
_classFilePath = CreateFilePath(_modelFolder, "classes.names");
|
||||
std::ifstream isValid(_classFilePath);
|
||||
if (!isValid) LoadClassesFromString();
|
||||
else LoadClassesFromFile();
|
||||
}
|
||||
}
|
||||
|
||||
labelMap.clear();
|
||||
@@ -322,7 +375,9 @@ namespace ANSCENTER {
|
||||
static_cast<int>(m_options.precision), m_options.maxBatchSize };
|
||||
m_trtEngine = EnginePoolManager<float>::instance().acquire(
|
||||
m_poolKey, m_options, _modelFilePath,
|
||||
SUB_VALS, DIV_VALS, NORMALIZE, m_maxSlotsPerGpu);
|
||||
SUB_VALS, DIV_VALS, NORMALIZE, m_maxSlotsPerGpu,
|
||||
_fromPackage ? _packageModel.data() : nullptr,
|
||||
_fromPackage ? _packageModel.size() : 0);
|
||||
m_usingSharedPool = (m_trtEngine != nullptr);
|
||||
}
|
||||
if (!m_trtEngine) {
|
||||
|
||||
@@ -52,6 +52,11 @@ namespace ANSCENTER {
|
||||
~ANSRTYOLO();
|
||||
|
||||
private:
|
||||
// Where a sealed package's TensorRT plan is cached, and what it is
|
||||
// called. Nothing but the plan is ever written there — the model
|
||||
// itself is parsed from memory.
|
||||
static std::string PackagePlanBasePath(const std::string& packagePath);
|
||||
|
||||
std::string _modelFilePath;
|
||||
bool _modelLoadValid{ false };
|
||||
bool _fp16{ false };
|
||||
|
||||
@@ -8,6 +8,7 @@
|
||||
#include <climits> // INT_MIN
|
||||
#include "ANSLicense.h" // ANS_DBG macro for DebugView
|
||||
#include "ANSODVendorGate.h" // ansod_vendor_gate::IsNvidiaGpuAvailable()
|
||||
#include "ANSModelOpen.h" // modelpkg::IsPackagePath (sealed .ansmodel)
|
||||
|
||||
// Process-wide flag: when true, all engines force single-GPU path (no pool, no idle timers).
|
||||
// Defined here, declared extern in EngineBuildLoadNetwork.inl.
|
||||
@@ -1576,6 +1577,13 @@ ANSODENGINE_API int OptimizeModelStr(const char* modelFilePath, const char* mode
|
||||
modelType, modelDetectionType, fp16, modelFilePath ? modelFilePath : "(null)");
|
||||
try {
|
||||
bool optimizedResult = false;
|
||||
// A sealed .ansmodel is already the artefact: there is no zip to
|
||||
// inflate and no loose ONNX to point an optimiser at. It still goes
|
||||
// through the handle below, which opens it in memory and builds the
|
||||
// TensorRT plan from the decrypted bytes — what it must not go
|
||||
// through is ModelOptimizer, which only knows how to unzip.
|
||||
const bool sealedPackage =
|
||||
ANSCENTER::modelpkg::IsPackagePath(modelFilePath ? modelFilePath : "");
|
||||
// NOTE: odMutex was removed here. OptimizeModelStr creates its own
|
||||
// temporary Engine<float> on the stack — no shared state with running
|
||||
// inference tasks. The global mutex was blocking ALL running tasks'
|
||||
@@ -1628,7 +1636,7 @@ ANSODENGINE_API int OptimizeModelStr(const char* modelFilePath, const char* mode
|
||||
}
|
||||
|
||||
|
||||
if (modelType == 31) // If modelType is RTYOLO (31), handle separately.
|
||||
if (modelType == 31 && !sealedPackage) // If modelType is RTYOLO (31), handle separately.
|
||||
{
|
||||
if (engineType == ANSCENTER::EngineType::NVIDIA_GPU) {
|
||||
if (modelDetectionType == 0) {
|
||||
@@ -1778,7 +1786,10 @@ ANSODENGINE_API int OptimizeModelStr(const char* modelFilePath, const char* mode
|
||||
Handle.reset(); // Destroy engines BEFORE guard restores cache
|
||||
|
||||
|
||||
if (optimizedResult && !modelFolder.empty()) return 1;
|
||||
// A package reports the folder its plan was cached in; when the
|
||||
// handle declined to name one there is still nothing wrong, because
|
||||
// nothing was ever meant to be written.
|
||||
if (optimizedResult && (!modelFolder.empty() || sealedPackage)) return 1;
|
||||
else return 0;
|
||||
}
|
||||
catch (const std::exception& e) {
|
||||
|
||||
@@ -188,6 +188,17 @@ public:
|
||||
/// Full path of the engine file resolved by the last buildLoadNetwork().
|
||||
[[nodiscard]] const std::string& engineFilePath() const { return m_engineFilePath; }
|
||||
|
||||
// ── ONNX bytes held in memory (sealed .ansmodel packages) ───────────────
|
||||
// A package is decrypted into memory and the plaintext model must never
|
||||
// reach the disk, so the parser reads these bytes instead of opening
|
||||
// onnxModelPath. The path is still passed in and still matters: it names
|
||||
// the TensorRT plan that gets cached beside it, and a plan is a built
|
||||
// artefact, not the model. Set the bytes before any build call; they are
|
||||
// wiped by clearOnnxBytes() and by the destructor.
|
||||
void setOnnxBytes(const void* data, size_t size);
|
||||
void clearOnnxBytes();
|
||||
[[nodiscard]] bool hasOnnxBytes() const { return !m_onnxBytes.empty(); }
|
||||
|
||||
/// Return the CUDA device index that preprocessing should target.
|
||||
/// In pool mode this returns the device of the first alive slot;
|
||||
/// in single-engine mode it returns m_options.deviceIndex.
|
||||
@@ -244,6 +255,9 @@ public:
|
||||
// retries until build succeeds or all candidates are exhausted.
|
||||
// Fixed-spatial models fall through to a single build() call.
|
||||
bool buildWithRetry(std::string onnxModelPath, const std::array<float, 3>& subVals, const std::array<float, 3>& divVals, bool normalize);
|
||||
// Where the ONNX bytes come from: the in-memory buffer when one was set,
|
||||
// otherwise the file at onnxModelPath.
|
||||
bool readOnnxBytes(const std::string& onnxModelPath, std::vector<char>& buffer) const;
|
||||
void getDeviceNames(std::vector<std::string>& deviceNames);
|
||||
void clearGpuBuffers();
|
||||
int getBindingIndexByName(const std::string& name);
|
||||
@@ -401,6 +415,7 @@ private:
|
||||
bool m_usingCachedEngine = false;
|
||||
bool m_skipEngineCache = false; // true = bypass TRTEngineCache (for OptimizeModelStr)
|
||||
bool m_skipOnnxRebuild = false; // true = don't delete/rebuild corrupt engine files (elastic growth)
|
||||
std::vector<char> m_onnxBytes; // model bytes in memory (package); empty = read from disk
|
||||
bool m_rebuiltFromOnnx = false; // last buildLoadNetwork built from ONNX (no matching cached engine)
|
||||
std::string m_engineFilePath; // engine file resolved by last buildLoadNetwork
|
||||
bool m_skipOnnxBuild = false; // true = skip ONNX→TRT build if no cached engine (demand growth)
|
||||
@@ -575,6 +590,8 @@ template <typename T> Engine<T>::Engine() {
|
||||
m_lastSetInputDims.clear();
|
||||
}
|
||||
template <typename T> Engine<T>::~Engine() {
|
||||
// A decrypted model must not outlive the engine that parsed it.
|
||||
try { clearOnnxBytes(); } catch (...) {}
|
||||
// =========================================================================
|
||||
// CRITICAL: Destructors must NEVER throw. Every CUDA/TRT call below is
|
||||
// wrapped in try/catch so that a failure in one step does not skip the
|
||||
|
||||
Reference in New Issue
Block a user