ANSFR: purge stale ServerOptimised zip when engine rebuilds from ONNX
A ServerOptimised zip is a per-GPU cache of a previous engine build. When the engine profile changes (batch gate, TRT/CUDA upgrade), the cached engine inside it goes stale: every re-extract (e.g. after Temp cleanup) restores the old engine and costs a full ONNX rebuild, forever, because the server only regenerates the zip when the file is missing. Engine now records rebuiltFromOnnx() + engineFilePath() in buildLoadNetwork (set only on the generate-from-ONNX branch; cache hits and alternative-precision loads don't raise it). After a rebuild, ANSFaceRecognizer::Initialize deletes the stale same-GPU engine variants from the model folder and the supplied optimised zip, so the server's next optimize pass recreates it with the fresh engine. Safety: only zips whose parent folder is ServerOptimised/-ized qualify — base model zips can never match. Engine purge is scoped to the "<model>.engine.<GPU>." prefix, leaving other GPUs' engines intact on multi-GPU machines. All deletes are best-effort (error_code); multi-GPU pool paths never set the flag, so the purge is a no-op there. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -265,6 +265,8 @@ bool Engine<T>::buildLoadNetwork(std::string onnxModelPath, const std::array<flo
|
|||||||
// It is full path
|
// It is full path
|
||||||
std::string engineName = serializeEngineOptions(m_options, onnxModelPath);
|
std::string engineName = serializeEngineOptions(m_options, onnxModelPath);
|
||||||
std::string engineDir = m_options.engineFileDir;
|
std::string engineDir = m_options.engineFileDir;
|
||||||
|
m_engineFilePath = engineName;
|
||||||
|
m_rebuiltFromOnnx = false;
|
||||||
|
|
||||||
if (FileExist(engineName)) {
|
if (FileExist(engineName)) {
|
||||||
if (m_verbose) { std::cout << "Engine file found: " << engineName << std::endl; }
|
if (m_verbose) { std::cout << "Engine file found: " << engineName << std::endl; }
|
||||||
@@ -335,6 +337,7 @@ bool Engine<T>::buildLoadNetwork(std::string onnxModelPath, const std::array<flo
|
|||||||
|
|
||||||
if (FileExist(alternativeEngineName)) {
|
if (FileExist(alternativeEngineName)) {
|
||||||
std::cout << "Found alternative precision engine: " << alternativeEngineName << std::endl;
|
std::cout << "Found alternative precision engine: " << alternativeEngineName << std::endl;
|
||||||
|
m_engineFilePath = alternativeEngineName;
|
||||||
return loadNetwork(alternativeEngineName, subVals, divVals, normalize);
|
return loadNetwork(alternativeEngineName, subVals, divVals, normalize);
|
||||||
}
|
}
|
||||||
else {
|
else {
|
||||||
@@ -358,11 +361,13 @@ bool Engine<T>::buildLoadNetwork(std::string onnxModelPath, const std::array<flo
|
|||||||
std::cout << "FP16 engine not found, but FP32 engine exists: " << fp32EngineName << std::endl;
|
std::cout << "FP16 engine not found, but FP32 engine exists: " << fp32EngineName << std::endl;
|
||||||
std::cout << "Loading existing FP32 engine..." << std::endl;
|
std::cout << "Loading existing FP32 engine..." << std::endl;
|
||||||
m_options.precision = ANSCENTER::Precision::FP32;
|
m_options.precision = ANSCENTER::Precision::FP32;
|
||||||
|
m_engineFilePath = fp32EngineName;
|
||||||
return loadNetwork(fp32EngineName, subVals, divVals, normalize);
|
return loadNetwork(fp32EngineName, subVals, divVals, normalize);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// ONNX model exists, generate engine
|
// ONNX model exists, generate engine
|
||||||
|
m_rebuiltFromOnnx = true;
|
||||||
std::cout << "========================================" << std::endl;
|
std::cout << "========================================" << std::endl;
|
||||||
std::cout << "Engine not found, generating from ONNX model" << std::endl;
|
std::cout << "Engine not found, generating from ONNX model" << std::endl;
|
||||||
std::cout << "========================================" << std::endl;
|
std::cout << "========================================" << std::endl;
|
||||||
|
|||||||
@@ -1,5 +1,7 @@
|
|||||||
#include "ANSFaceRecognizer.h"
|
#include "ANSFaceRecognizer.h"
|
||||||
#include "engine.h"
|
#include "engine.h"
|
||||||
|
#include <cctype>
|
||||||
|
#include <filesystem>
|
||||||
|
|
||||||
//#define DEBUG_TIME
|
//#define DEBUG_TIME
|
||||||
#ifdef DEBUG_TIME
|
#ifdef DEBUG_TIME
|
||||||
@@ -33,6 +35,55 @@ namespace {
|
|||||||
const size_t totalMiB = prop.totalGlobalMem / (1024ULL * 1024ULL);
|
const size_t totalMiB = prop.totalGlobalMem / (1024ULL * 1024ULL);
|
||||||
return (totalMiB >= 7680) ? desiredBatch : 1;
|
return (totalMiB >= 7680) ? desiredBatch : 1;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// A ServerOptimised zip is a per-GPU cache of a previous engine build. When
|
||||||
|
// one was supplied but the engine still had to be rebuilt from ONNX, the
|
||||||
|
// zip's cached engine is stale (batch profile or TRT/CUDA version changed).
|
||||||
|
// Delete the stale engine variants for this GPU from the model folder (so a
|
||||||
|
// regenerated zip doesn't carry two engines) and the zip itself — the
|
||||||
|
// ANSVIS Server recreates the zip from the model folder when it is missing.
|
||||||
|
// Base model zips (the ONNX source artifact) are never touched: only zips
|
||||||
|
// whose parent folder is ServerOptimised qualify.
|
||||||
|
void PurgeStaleOptimisedModel(const std::string& modelZipFilePath,
|
||||||
|
const std::string& currentEnginePath) {
|
||||||
|
namespace fs = std::filesystem;
|
||||||
|
try {
|
||||||
|
const fs::path zip(modelZipFilePath);
|
||||||
|
std::string parent = zip.parent_path().filename().string();
|
||||||
|
for (char& c : parent) c = (char)std::tolower((unsigned char)c);
|
||||||
|
if (parent != "serveroptimised" && parent != "serveroptimized")
|
||||||
|
return;
|
||||||
|
|
||||||
|
// Stale variants share the "<model>.engine.<GPU>." prefix of the
|
||||||
|
// freshly built engine; other GPUs' engines on multi-GPU machines
|
||||||
|
// have a different prefix and are left alone.
|
||||||
|
const fs::path engine(currentEnginePath);
|
||||||
|
const std::string engineName = engine.filename().string();
|
||||||
|
const size_t tagPos = engineName.find(".engine.");
|
||||||
|
if (tagPos != std::string::npos && engine.has_parent_path()) {
|
||||||
|
const size_t gpuEnd = engineName.find('.', tagPos + 8);
|
||||||
|
const std::string prefix = (gpuEnd != std::string::npos)
|
||||||
|
? engineName.substr(0, gpuEnd + 1) : engineName;
|
||||||
|
for (const auto& entry : fs::directory_iterator(engine.parent_path())) {
|
||||||
|
if (!entry.is_regular_file()) continue;
|
||||||
|
const std::string name = entry.path().filename().string();
|
||||||
|
if (name != engineName && name.rfind(prefix, 0) == 0) {
|
||||||
|
std::error_code ec;
|
||||||
|
fs::remove(entry.path(), ec);
|
||||||
|
if (!ec) std::cout << "[ANSFR] Purged stale engine: " << name << std::endl;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
std::error_code ec;
|
||||||
|
fs::remove(zip, ec);
|
||||||
|
if (!ec) std::cout << "[ANSFR] Purged stale optimised model zip (server will regenerate): "
|
||||||
|
<< modelZipFilePath << std::endl;
|
||||||
|
}
|
||||||
|
catch (const std::exception&) {
|
||||||
|
// Cleanup is best-effort; a leftover stale zip only costs a rebuild.
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
namespace ANSCENTER {
|
namespace ANSCENTER {
|
||||||
@@ -147,6 +198,13 @@ namespace ANSCENTER {
|
|||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// If we were handed a ServerOptimised zip yet still had to
|
||||||
|
// rebuild from ONNX, that zip's engine is stale — remove it
|
||||||
|
// so the server regenerates it with the fresh engine.
|
||||||
|
if (m_trtEngine->rebuiltFromOnnx()) {
|
||||||
|
PurgeStaleOptimisedModel(modelZipFilePath, m_trtEngine->engineFilePath());
|
||||||
|
}
|
||||||
|
|
||||||
// Create CUDA stream for GPU preprocessing (lazy init)
|
// Create CUDA stream for GPU preprocessing (lazy init)
|
||||||
if (!m_gpuStream) {
|
if (!m_gpuStream) {
|
||||||
m_gpuStream = std::make_unique<cv::cuda::Stream>();
|
m_gpuStream = std::make_unique<cv::cuda::Stream>();
|
||||||
|
|||||||
@@ -180,6 +180,14 @@ public:
|
|||||||
[[nodiscard]] const std::vector<nvinfer1::Dims> &getOutputDims() const override { return m_outputDims; };
|
[[nodiscard]] const std::vector<nvinfer1::Dims> &getOutputDims() const override { return m_outputDims; };
|
||||||
[[nodiscard]] const ANSCENTER::Options& getOptions() const { return m_options; }
|
[[nodiscard]] const ANSCENTER::Options& getOptions() const { return m_options; }
|
||||||
|
|
||||||
|
/// True when the last buildLoadNetwork() had to build the engine from ONNX
|
||||||
|
/// because no cached engine matched the current profile (GPU / precision /
|
||||||
|
/// batch / TRT version). Callers use this to invalidate stale caches that
|
||||||
|
/// carried the old engine (e.g. ServerOptimised model zips).
|
||||||
|
[[nodiscard]] bool rebuiltFromOnnx() const { return m_rebuiltFromOnnx; }
|
||||||
|
/// Full path of the engine file resolved by the last buildLoadNetwork().
|
||||||
|
[[nodiscard]] const std::string& engineFilePath() const { return m_engineFilePath; }
|
||||||
|
|
||||||
/// Return the CUDA device index that preprocessing should target.
|
/// Return the CUDA device index that preprocessing should target.
|
||||||
/// In pool mode this returns the device of the first alive slot;
|
/// In pool mode this returns the device of the first alive slot;
|
||||||
/// in single-engine mode it returns m_options.deviceIndex.
|
/// in single-engine mode it returns m_options.deviceIndex.
|
||||||
@@ -393,6 +401,8 @@ private:
|
|||||||
bool m_usingCachedEngine = false;
|
bool m_usingCachedEngine = false;
|
||||||
bool m_skipEngineCache = false; // true = bypass TRTEngineCache (for OptimizeModelStr)
|
bool m_skipEngineCache = false; // true = bypass TRTEngineCache (for OptimizeModelStr)
|
||||||
bool m_skipOnnxRebuild = false; // true = don't delete/rebuild corrupt engine files (elastic growth)
|
bool m_skipOnnxRebuild = false; // true = don't delete/rebuild corrupt engine files (elastic growth)
|
||||||
|
bool m_rebuiltFromOnnx = false; // last buildLoadNetwork built from ONNX (no matching cached engine)
|
||||||
|
std::string m_engineFilePath; // engine file resolved by last buildLoadNetwork
|
||||||
bool m_skipOnnxBuild = false; // true = skip ONNX→TRT build if no cached engine (demand growth)
|
bool m_skipOnnxBuild = false; // true = skip ONNX→TRT build if no cached engine (demand growth)
|
||||||
bool m_lastLoadFailedVRAM = false; // set by loadNetwork when failure is due to insufficient VRAM
|
bool m_lastLoadFailedVRAM = false; // set by loadNetwork when failure is due to insufficient VRAM
|
||||||
bool m_forceNoPool = false; // true = force single-GPU path, skip multi-GPU pool
|
bool m_forceNoPool = false; // true = force single-GPU path, skip multi-GPU pool
|
||||||
|
|||||||
Reference in New Issue
Block a user