From dc699f5a791269ae76afe912dfa42c2d1be11c1c Mon Sep 17 00:00:00 2001 From: Tuan Nghia Nguyen Date: Wed, 14 Oct 2026 13:15:44 +1100 Subject: [PATCH] fix(EPLoader): remove the staging folders that dead processes left in %TEMP% Each process copies the ONNX Runtime and provider DLLs (~200 MB with CUDA and TensorRT) into %TEMP%\anscenter_ort_, and only a clean Shutdown() deletes it. A process that is killed or crashes leaves it for good. On a customer's training PC (2026-10-14) ANSGateway's watchdog restarted ANSVIS_Server every minute while its REST API was down, leaving a folder a minute in C:\Windows\Temp until the drive filled and training failed. Before staging, each process now deletes every anscenter_ort_ folder whose PID is not a running process. A live process's folder is never touched, since its provider DLLs load lazily from there. Both copies of EPLoader (core/ANSLibsLoader and engines/ONNXEngine) get the sweep. Compile-checked with MinGW against the Windows headers; not yet run on Windows. Co-Authored-By: Claude Opus 5.5 --- core/ANSLibsLoader/EPLoader.cpp | 60 +++++++++++++++++++++++++++++++++ engines/ONNXEngine/EPLoader.cpp | 60 +++++++++++++++++++++++++++++++++ 2 files changed, 120 insertions(+) diff --git a/core/ANSLibsLoader/EPLoader.cpp b/core/ANSLibsLoader/EPLoader.cpp index c789530..38b7161 100644 --- a/core/ANSLibsLoader/EPLoader.cpp +++ b/core/ANSLibsLoader/EPLoader.cpp @@ -325,6 +325,65 @@ namespace ANSCENTER { RemoveDirectoryA(dir.c_str()); std::cout << "[EPLoader] Temp staging dir deleted: " << dir << std::endl; } + // ── SweepStaleTempDirs ──────────────────────────────────────────────────── + // Removes the staging folders that dead processes left behind. + // + // Only a clean Shutdown() deletes a process's folder. A process that is + // killed or crashes leaves its ~200 MB of DLLs in %TEMP% for good, and a + // service restarted in a loop (ANSGateway's ANSVIS watchdog restarts + // ANSVIS_Server every minute while its REST API is down) fills the system + // drive: C:\Windows\Temp grew by a folder a minute on a customer's + // training PC until training failed for lack of disk. + // + // A folder is removed only when its PID is not a running process. A live + // process's folder is never touched, even partly: its provider DLLs + // (TensorRT, CUDA) load lazily from there, after start-up. A PID reused by + // an unrelated process keeps its stale folder until that one exits — + // rare, and bounded. + static bool ProcessIsRunning(DWORD pid) + { + HANDLE h = OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, FALSE, pid); + if (!h) { + // No such process. Anything else (access denied: another user's + // process, a protected one) means it exists. + return GetLastError() != ERROR_INVALID_PARAMETER; + } + DWORD code = 0; + const bool running = GetExitCodeProcess(h, &code) && code == STILL_ACTIVE; + CloseHandle(h); + return running; + } + + static void SweepStaleTempDirs() + { + char tmp[MAX_PATH] = {}; + if (!GetTempPathA(MAX_PATH, tmp)) return; + const std::string base(tmp); + const std::string prefix = "anscenter_ort_"; + const DWORD self = GetCurrentProcessId(); + + WIN32_FIND_DATAA fd{}; + HANDLE hFind = FindFirstFileA((base + prefix + "*").c_str(), &fd); + if (hFind == INVALID_HANDLE_VALUE) return; + std::vector stale; + do { + if (!(fd.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY)) continue; + const std::string name(fd.cFileName); + const std::string digits = name.substr(prefix.size()); + if (digits.empty() || digits.size() > 10 || + digits.find_first_not_of("0123456789") != std::string::npos) + continue; + const unsigned long long pid = std::stoull(digits); + if (pid == 0 || pid > 0xFFFFFFFFull || DWORD(pid) == self) continue; + if (!ProcessIsRunning(DWORD(pid))) stale.push_back(base + name); + } while (FindNextFileA(hFind, &fd)); + FindClose(hFind); + + for (const std::string& dir : stale) DeleteTempDir(dir); + if (!stale.empty()) + std::cout << "[EPLoader] Removed " << stale.size() + << " staging dir(s) left by processes that are gone." << std::endl; + } #endif // _WIN32 @@ -345,6 +404,7 @@ namespace ANSCENTER { #ifdef _WIN32 // ── Windows: stage ALL ep_dir DLLs into a process-unique temp folder ──── + SweepStaleTempDirs(); std::string temp_dir = MakeTempDir(); std::cout << "[EPLoader] Staging dir : " << temp_dir << std::endl; diff --git a/engines/ONNXEngine/EPLoader.cpp b/engines/ONNXEngine/EPLoader.cpp index 174b339..84b1e8d 100644 --- a/engines/ONNXEngine/EPLoader.cpp +++ b/engines/ONNXEngine/EPLoader.cpp @@ -383,6 +383,65 @@ namespace ANSCENTER { RemoveDirectoryA(dir.c_str()); std::cout << "[EPLoader] Temp staging dir deleted: " << dir << std::endl; } + // ── SweepStaleTempDirs ──────────────────────────────────────────────────── + // Removes the staging folders that dead processes left behind. + // + // Only a clean Shutdown() deletes a process's folder. A process that is + // killed or crashes leaves its ~200 MB of DLLs in %TEMP% for good, and a + // service restarted in a loop (ANSGateway's ANSVIS watchdog restarts + // ANSVIS_Server every minute while its REST API is down) fills the system + // drive: C:\Windows\Temp grew by a folder a minute on a customer's + // training PC until training failed for lack of disk. + // + // A folder is removed only when its PID is not a running process. A live + // process's folder is never touched, even partly: its provider DLLs + // (TensorRT, CUDA) load lazily from there, after start-up. A PID reused by + // an unrelated process keeps its stale folder until that one exits — + // rare, and bounded. + static bool ProcessIsRunning(DWORD pid) + { + HANDLE h = OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, FALSE, pid); + if (!h) { + // No such process. Anything else (access denied: another user's + // process, a protected one) means it exists. + return GetLastError() != ERROR_INVALID_PARAMETER; + } + DWORD code = 0; + const bool running = GetExitCodeProcess(h, &code) && code == STILL_ACTIVE; + CloseHandle(h); + return running; + } + + static void SweepStaleTempDirs() + { + char tmp[MAX_PATH] = {}; + if (!GetTempPathA(MAX_PATH, tmp)) return; + const std::string base(tmp); + const std::string prefix = "anscenter_ort_"; + const DWORD self = GetCurrentProcessId(); + + WIN32_FIND_DATAA fd{}; + HANDLE hFind = FindFirstFileA((base + prefix + "*").c_str(), &fd); + if (hFind == INVALID_HANDLE_VALUE) return; + std::vector stale; + do { + if (!(fd.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY)) continue; + const std::string name(fd.cFileName); + const std::string digits = name.substr(prefix.size()); + if (digits.empty() || digits.size() > 10 || + digits.find_first_not_of("0123456789") != std::string::npos) + continue; + const unsigned long long pid = std::stoull(digits); + if (pid == 0 || pid > 0xFFFFFFFFull || DWORD(pid) == self) continue; + if (!ProcessIsRunning(DWORD(pid))) stale.push_back(base + name); + } while (FindNextFileA(hFind, &fd)); + FindClose(hFind); + + for (const std::string& dir : stale) DeleteTempDir(dir); + if (!stale.empty()) + std::cout << "[EPLoader] Removed " << stale.size() + << " staging dir(s) left by processes that are gone." << std::endl; + } #endif // _WIN32 @@ -418,6 +477,7 @@ namespace ANSCENTER { // so it resolves providers relative to %TEMP% — where they don't exist. // Staging ALL provider DLLs alongside our renamed copy fixes this. + SweepStaleTempDirs(); std::string temp_dir = MakeTempDir(); std::cout << "[EPLoader] Staging dir : " << temp_dir << std::endl;