diff --git a/engines/TensorRTAPI/include/engine/EngineBuildLoadNetwork.inl b/engines/TensorRTAPI/include/engine/EngineBuildLoadNetwork.inl index 62a5d0e..f65379c 100644 --- a/engines/TensorRTAPI/include/engine/EngineBuildLoadNetwork.inl +++ b/engines/TensorRTAPI/include/engine/EngineBuildLoadNetwork.inl @@ -265,6 +265,8 @@ bool Engine::buildLoadNetwork(std::string onnxModelPath, const std::array::buildLoadNetwork(std::string onnxModelPath, const std::array::buildLoadNetwork(std::string onnxModelPath, const std::array +#include //#define DEBUG_TIME #ifdef DEBUG_TIME @@ -33,6 +35,55 @@ namespace { const size_t totalMiB = prop.totalGlobalMem / (1024ULL * 1024ULL); return (totalMiB >= 7680) ? desiredBatch : 1; } + + // A ServerOptimised zip is a per-GPU cache of a previous engine build. When + // one was supplied but the engine still had to be rebuilt from ONNX, the + // zip's cached engine is stale (batch profile or TRT/CUDA version changed). + // Delete the stale engine variants for this GPU from the model folder (so a + // regenerated zip doesn't carry two engines) and the zip itself — the + // ANSVIS Server recreates the zip from the model folder when it is missing. + // Base model zips (the ONNX source artifact) are never touched: only zips + // whose parent folder is ServerOptimised qualify. + void PurgeStaleOptimisedModel(const std::string& modelZipFilePath, + const std::string& currentEnginePath) { + namespace fs = std::filesystem; + try { + const fs::path zip(modelZipFilePath); + std::string parent = zip.parent_path().filename().string(); + for (char& c : parent) c = (char)std::tolower((unsigned char)c); + if (parent != "serveroptimised" && parent != "serveroptimized") + return; + + // Stale variants share the ".engine.." prefix of the + // freshly built engine; other GPUs' engines on multi-GPU machines + // have a different prefix and are left alone. + const fs::path engine(currentEnginePath); + const std::string engineName = engine.filename().string(); + const size_t tagPos = engineName.find(".engine."); + if (tagPos != std::string::npos && engine.has_parent_path()) { + const size_t gpuEnd = engineName.find('.', tagPos + 8); + const std::string prefix = (gpuEnd != std::string::npos) + ? engineName.substr(0, gpuEnd + 1) : engineName; + for (const auto& entry : fs::directory_iterator(engine.parent_path())) { + if (!entry.is_regular_file()) continue; + const std::string name = entry.path().filename().string(); + if (name != engineName && name.rfind(prefix, 0) == 0) { + std::error_code ec; + fs::remove(entry.path(), ec); + if (!ec) std::cout << "[ANSFR] Purged stale engine: " << name << std::endl; + } + } + } + + std::error_code ec; + fs::remove(zip, ec); + if (!ec) std::cout << "[ANSFR] Purged stale optimised model zip (server will regenerate): " + << modelZipFilePath << std::endl; + } + catch (const std::exception&) { + // Cleanup is best-effort; a leftover stale zip only costs a rebuild. + } + } } namespace ANSCENTER { @@ -147,6 +198,13 @@ namespace ANSCENTER { return false; } + // If we were handed a ServerOptimised zip yet still had to + // rebuild from ONNX, that zip's engine is stale — remove it + // so the server regenerates it with the fresh engine. + if (m_trtEngine->rebuiltFromOnnx()) { + PurgeStaleOptimisedModel(modelZipFilePath, m_trtEngine->engineFilePath()); + } + // Create CUDA stream for GPU preprocessing (lazy init) if (!m_gpuStream) { m_gpuStream = std::make_unique(); diff --git a/modules/ANSODEngine/engine.h b/modules/ANSODEngine/engine.h index b0966b2..a6fbc06 100644 --- a/modules/ANSODEngine/engine.h +++ b/modules/ANSODEngine/engine.h @@ -180,6 +180,14 @@ public: [[nodiscard]] const std::vector &getOutputDims() const override { return m_outputDims; }; [[nodiscard]] const ANSCENTER::Options& getOptions() const { return m_options; } + /// True when the last buildLoadNetwork() had to build the engine from ONNX + /// because no cached engine matched the current profile (GPU / precision / + /// batch / TRT version). Callers use this to invalidate stale caches that + /// carried the old engine (e.g. ServerOptimised model zips). + [[nodiscard]] bool rebuiltFromOnnx() const { return m_rebuiltFromOnnx; } + /// Full path of the engine file resolved by the last buildLoadNetwork(). + [[nodiscard]] const std::string& engineFilePath() const { return m_engineFilePath; } + /// Return the CUDA device index that preprocessing should target. /// In pool mode this returns the device of the first alive slot; /// in single-engine mode it returns m_options.deviceIndex. @@ -393,6 +401,8 @@ private: bool m_usingCachedEngine = false; bool m_skipEngineCache = false; // true = bypass TRTEngineCache (for OptimizeModelStr) bool m_skipOnnxRebuild = false; // true = don't delete/rebuild corrupt engine files (elastic growth) + bool m_rebuiltFromOnnx = false; // last buildLoadNetwork built from ONNX (no matching cached engine) + std::string m_engineFilePath; // engine file resolved by last buildLoadNetwork bool m_skipOnnxBuild = false; // true = skip ONNX→TRT build if no cached engine (demand growth) bool m_lastLoadFailedVRAM = false; // set by loadNetwork when failure is due to insufficient VRAM bool m_forceNoPool = false; // true = force single-GPU path, skip multi-GPU pool