Author SHA1 Message Date
nghianguyenandClaude Opus 5.5 dc699f5a79 fix(EPLoader): remove the staging folders that dead processes left in %TEMP%
Each process copies the ONNX Runtime and provider DLLs (~200 MB with CUDA
and TensorRT) into %TEMP%\anscenter_ort_<pid>, and only a clean Shutdown()
deletes it. A process that is killed or crashes leaves it for good. On a
customer's training PC (2026-10-14) ANSGateway's watchdog restarted
ANSVIS_Server every minute while its REST API was down, leaving a folder a
minute in C:\Windows\Temp until the drive filled and training failed.

Before staging, each process now deletes every anscenter_ort_<pid> folder
whose PID is not a running process. A live process's folder is never
touched, since its provider DLLs load lazily from there. Both copies of
EPLoader (core/ANSLibsLoader and engines/ONNXEngine) get the sweep.

Compile-checked with MinGW against the Windows headers; not yet run on
Windows.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-14 13:15:50 +11:00
6 changed files with 140 additions and 215 deletions
+60
View File
@@ -325,6 +325,65 @@ namespace ANSCENTER {
RemoveDirectoryA(dir.c_str()); RemoveDirectoryA(dir.c_str());
std::cout << "[EPLoader] Temp staging dir deleted: " << dir << std::endl; std::cout << "[EPLoader] Temp staging dir deleted: " << dir << std::endl;
} }
// ── SweepStaleTempDirs ────────────────────────────────────────────────────
// Removes the staging folders that dead processes left behind.
//
// Only a clean Shutdown() deletes a process's folder. A process that is
// killed or crashes leaves its ~200 MB of DLLs in %TEMP% for good, and a
// service restarted in a loop (ANSGateway's ANSVIS watchdog restarts
// ANSVIS_Server every minute while its REST API is down) fills the system
// drive: C:\Windows\Temp grew by a folder a minute on a customer's
// training PC until training failed for lack of disk.
//
// A folder is removed only when its PID is not a running process. A live
// process's folder is never touched, even partly: its provider DLLs
// (TensorRT, CUDA) load lazily from there, after start-up. A PID reused by
// an unrelated process keeps its stale folder until that one exits —
// rare, and bounded.
static bool ProcessIsRunning(DWORD pid)
{
HANDLE h = OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, FALSE, pid);
if (!h) {
// No such process. Anything else (access denied: another user's
// process, a protected one) means it exists.
return GetLastError() != ERROR_INVALID_PARAMETER;
}
DWORD code = 0;
const bool running = GetExitCodeProcess(h, &code) && code == STILL_ACTIVE;
CloseHandle(h);
return running;
}
static void SweepStaleTempDirs()
{
char tmp[MAX_PATH] = {};
if (!GetTempPathA(MAX_PATH, tmp)) return;
const std::string base(tmp);
const std::string prefix = "anscenter_ort_";
const DWORD self = GetCurrentProcessId();
WIN32_FIND_DATAA fd{};
HANDLE hFind = FindFirstFileA((base + prefix + "*").c_str(), &fd);
if (hFind == INVALID_HANDLE_VALUE) return;
std::vector<std::string> stale;
do {
if (!(fd.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY)) continue;
const std::string name(fd.cFileName);
const std::string digits = name.substr(prefix.size());
if (digits.empty() || digits.size() > 10 ||
digits.find_first_not_of("0123456789") != std::string::npos)
continue;
const unsigned long long pid = std::stoull(digits);
if (pid == 0 || pid > 0xFFFFFFFFull || DWORD(pid) == self) continue;
if (!ProcessIsRunning(DWORD(pid))) stale.push_back(base + name);
} while (FindNextFileA(hFind, &fd));
FindClose(hFind);
for (const std::string& dir : stale) DeleteTempDir(dir);
if (!stale.empty())
std::cout << "[EPLoader] Removed " << stale.size()
<< " staging dir(s) left by processes that are gone." << std::endl;
}
#endif // _WIN32 #endif // _WIN32
@@ -345,6 +404,7 @@ namespace ANSCENTER {
#ifdef _WIN32 #ifdef _WIN32
// ── Windows: stage ALL ep_dir DLLs into a process-unique temp folder ──── // ── Windows: stage ALL ep_dir DLLs into a process-unique temp folder ────
SweepStaleTempDirs();
std::string temp_dir = MakeTempDir(); std::string temp_dir = MakeTempDir();
std::cout << "[EPLoader] Staging dir : " << temp_dir << std::endl; std::cout << "[EPLoader] Staging dir : " << temp_dir << std::endl;
-74
View File
@@ -1,74 +0,0 @@
#ifndef ANSCSPRNG_H
#define ANSCSPRNG_H
#pragma once
// ANSCsprng — random bytes from the operating system, for keys, salts and
// nonces.
//
// TAKEN FROM ANSMINDCORE (src/core/csprng.hpp). NOT std::random_device or a
// seeded generator: fine for an id that only needs to be distinct, and wrong
// for anything a guesser must not predict. The OS generator is the only
// source used for key material here.
//
// Windows BCryptGenRandom with the system-preferred RNG (bcrypt.lib)
// Linux getrandom(2), looping over short reads and EINTR
// macOS arc4random_buf, which cannot fail
//
// Returns false when the OS refused — callers treat that as "cannot seal",
// never as "seal with whatever was in the buffer".
#include <cstddef>
#include <cstdint>
#if defined(_WIN32)
#ifndef WIN32_LEAN_AND_MEAN
#define WIN32_LEAN_AND_MEAN
#endif
#ifndef NOMINMAX
#define NOMINMAX
#endif
#include <windows.h>
#include <bcrypt.h>
#if defined(_MSC_VER)
#pragma comment(lib, "bcrypt.lib")
#endif
#elif defined(__APPLE__)
#include <stdlib.h>
#else
#include <cerrno>
#include <sys/random.h>
#endif
namespace ANSCENTER {
inline bool randomBytes(uint8_t* out, size_t n) {
if (n == 0) return true;
if (!out) return false;
#if defined(_WIN32)
while (n > 0) {
const ULONG chunk = n > 0x7fffffffu ? 0x7fffffffu : ULONG(n);
if (BCryptGenRandom(nullptr, out, chunk, BCRYPT_USE_SYSTEM_PREFERRED_RNG) != 0)
return false;
out += chunk;
n -= chunk;
}
return true;
#elif defined(__APPLE__)
arc4random_buf(out, n);
return true;
#else
while (n > 0) {
const ssize_t got = getrandom(out, n, 0);
if (got < 0) {
if (errno == EINTR) continue;
return false;
}
out += size_t(got);
n -= size_t(got);
}
return true;
#endif
}
} // namespace ANSCENTER
#endif // ANSCSPRNG_H
+18 -62
View File
@@ -75,7 +75,6 @@
#include <nlohmann/json.hpp> #include <nlohmann/json.hpp>
#include "ANSAead.h" #include "ANSAead.h"
#include "ANSCsprng.h"
#include "ANSPackageBits.h" #include "ANSPackageBits.h"
extern "C" { extern "C" {
@@ -579,15 +578,9 @@ struct PackRequest {
Json card; // optional, counts only Json card; // optional, counts only
std::vector<PackModel> models; std::vector<PackModel> models;
uint32_t chunk = kDefaultChunk; // smaller in tests, to exercise chunking uint32_t chunk = kDefaultChunk; // smaller in tests, to exercise chunking
// TESTS ONLY. pack() draws the content key, the ephemeral X25519 key and // Deterministic nonces are for TESTS ONLY: a real package draws them from
// every nonce from the operating system's CSPRNG. Setting this derives them // the platform CSPRNG, because a repeated (key, nonce) pair is fatal.
// from `testSeed` instead, so a fixture can be written byte-for-byte again — uint8_t nonceSeed = 0;
// and so anyone holding the package can recover its content key by trying
// the 256 seeds, without the base key. It is honoured only in a build that
// defines ANSMODEL_DETERMINISTIC_PACK_FOR_TESTS; anywhere else pack() refuses
// rather than write a package that only looks sealed.
bool insecureDeterministicForTests = false;
uint8_t testSeed = 0;
}; };
inline bool pack(const std::string& path, const PackRequest& req, inline bool pack(const std::string& path, const PackRequest& req,
@@ -596,58 +589,11 @@ inline bool pack(const std::string& path, const PackRequest& req,
err = "a package with no model describes nothing"; err = "a package with no model describes nothing";
return false; return false;
} }
// Everything the package's secrecy rests on, drawn up front: the content // A content key per package. Tests pass a seed; a real packer must use the
// key, the ephemeral private key it is wrapped with, the wrap nonce and one // platform CSPRNG here.
// nonce base per sealed part. From the OS CSPRNG — anything a holder of the
// file could re-derive would open it without the base key.
Key contentKey{}; Key contentKey{};
Key esk{}; for (size_t i = 0; i < contentKey.size(); ++i)
std::array<uint8_t, aead::kNonceBytes> wrapNonce{}; contentKey[i] = uint8_t(req.nonceSeed * 31 + i * 7 + 11);
std::array<uint8_t, 4> manifestBase{};
std::vector<std::array<uint8_t, 4>> bases(req.models.size());
struct Wipe {
Key& a;
Key& b;
~Wipe() {
crypto_wipe(a.data(), a.size());
crypto_wipe(b.data(), b.size());
}
} wipe{contentKey, esk};
if (req.insecureDeterministicForTests) {
#if defined(ANSMODEL_DETERMINISTIC_PACK_FOR_TESTS)
for (size_t i = 0; i < contentKey.size(); ++i)
contentKey[i] = uint8_t(req.testSeed * 31 + i * 7 + 11);
for (size_t i = 0; i < esk.size(); ++i)
esk[i] = uint8_t(req.testSeed * 13 + i * 5 + 3);
for (size_t i = 0; i < wrapNonce.size(); ++i)
wrapNonce[i] = uint8_t(req.testSeed + 100 + i);
manifestBase = {uint8_t(req.testSeed + 1), 2, 3, 4};
for (size_t i = 0; i < bases.size(); ++i)
bases[i] = {uint8_t(req.testSeed + 10 + i), 5, 6, 7};
#else
err = "deterministic packing is for tests only, and this build is not one";
return false;
#endif
} else {
bool ok = ANSCENTER::randomBytes(contentKey.data(), contentKey.size()) &&
ANSCENTER::randomBytes(esk.data(), esk.size()) &&
ANSCENTER::randomBytes(wrapNonce.data(), wrapNonce.size()) &&
ANSCENTER::randomBytes(manifestBase.data(), manifestBase.size());
// The parts share the content key, so their bases must differ: two parts
// on one base would reuse every chunk nonce they have in common.
for (size_t i = 0; ok && i < bases.size(); ++i) {
bool clash = true;
while (ok && clash) {
ok = ANSCENTER::randomBytes(bases[i].data(), bases[i].size());
clash = bases[i] == manifestBase;
for (size_t j = 0; j < i; ++j) clash = clash || bases[i] == bases[j];
}
}
if (!ok) {
err = "the operating system gave no random bytes, so nothing was sealed";
return false;
}
}
auto sealChunked = [&](const std::vector<uint8_t>& plain, auto sealChunked = [&](const std::vector<uint8_t>& plain,
const std::string& payloadId, const std::string& payloadId,
@@ -670,6 +616,7 @@ inline bool pack(const std::string& path, const PackRequest& req,
const std::string manifestText = req.manifest.dump(); const std::string manifestText = req.manifest.dump();
const std::vector<uint8_t> manifestBytes(manifestText.begin(), const std::vector<uint8_t> manifestBytes(manifestText.begin(),
manifestText.end()); manifestText.end());
std::array<uint8_t, 4> manifestBase{uint8_t(req.nonceSeed + 1), 2, 3, 4};
std::string manifestBlob; std::string manifestBlob;
{ {
const auto sealed = const auto sealed =
@@ -681,14 +628,20 @@ inline bool pack(const std::string& path, const PackRequest& req,
} }
std::vector<std::string> blobs; std::vector<std::string> blobs;
std::vector<std::array<uint8_t, 4>> bases;
for (size_t i = 0; i < req.models.size(); ++i) { for (size_t i = 0; i < req.models.size(); ++i) {
std::array<uint8_t, 4> base{uint8_t(req.nonceSeed + 10 + i), 5, 6, 7};
std::string blob; std::string blob;
sealChunked(req.models[i].bytes, req.models[i].id, bases[i], blob); sealChunked(req.models[i].bytes, req.models[i].id, base, blob);
blobs.push_back(std::move(blob)); blobs.push_back(std::move(blob));
bases.push_back(base);
} }
// Wrap the content key to the base key. One wrap here; the header's shape // Wrap the content key to the base key. One wrap here; the header's shape
// takes several, which is how a package opens on more than one key. // takes several, which is how a package opens on more than one key.
std::array<uint8_t, aead::kNonceBytes> wrapNonce{};
for (size_t i = 0; i < wrapNonce.size(); ++i)
wrapNonce[i] = uint8_t(req.nonceSeed + 100 + i);
const std::vector<uint8_t> keyPlain(contentKey.begin(), contentKey.end()); const std::vector<uint8_t> keyPlain(contentKey.begin(), contentKey.end());
Key wrapKey = req.key; Key wrapKey = req.key;
@@ -697,6 +650,9 @@ inline bool pack(const std::string& path, const PackRequest& req,
// An ephemeral key pair per package: its public half travels in the // An ephemeral key pair per package: its public half travels in the
// header, its private half dies here, and the packer never holds anything // header, its private half dies here, and the packer never holds anything
// that opens the result. // that opens the result.
Key esk{};
for (size_t i = 0; i < esk.size(); ++i)
esk[i] = uint8_t(req.nonceSeed * 13 + i * 5 + 3);
epk.assign(32, 0); epk.assign(32, 0);
crypto_x25519_public_key(epk.data(), esk.data()); crypto_x25519_public_key(epk.data(), esk.data());
uint8_t shared[32]; uint8_t shared[32];
-5
View File
@@ -40,9 +40,4 @@ else()
"(run ansmodel-keys emit --product anscore, then re-run cmake)") "(run ansmodel-keys emit --product anscore, then re-run cmake)")
endif() endif()
# BCryptGenRandom, behind the content key and nonces pack() draws (ANSCsprng.h).
if(WIN32)
target_link_libraries(ANSModelPackage PUBLIC bcrypt)
endif()
set_target_properties(ANSModelPackage PROPERTIES POSITION_INDEPENDENT_CODE ON) set_target_properties(ANSModelPackage PROPERTIES POSITION_INDEPENDENT_CODE ON)
+60
View File
@@ -383,6 +383,65 @@ namespace ANSCENTER {
RemoveDirectoryA(dir.c_str()); RemoveDirectoryA(dir.c_str());
std::cout << "[EPLoader] Temp staging dir deleted: " << dir << std::endl; std::cout << "[EPLoader] Temp staging dir deleted: " << dir << std::endl;
} }
// ── SweepStaleTempDirs ────────────────────────────────────────────────────
// Removes the staging folders that dead processes left behind.
//
// Only a clean Shutdown() deletes a process's folder. A process that is
// killed or crashes leaves its ~200 MB of DLLs in %TEMP% for good, and a
// service restarted in a loop (ANSGateway's ANSVIS watchdog restarts
// ANSVIS_Server every minute while its REST API is down) fills the system
// drive: C:\Windows\Temp grew by a folder a minute on a customer's
// training PC until training failed for lack of disk.
//
// A folder is removed only when its PID is not a running process. A live
// process's folder is never touched, even partly: its provider DLLs
// (TensorRT, CUDA) load lazily from there, after start-up. A PID reused by
// an unrelated process keeps its stale folder until that one exits —
// rare, and bounded.
static bool ProcessIsRunning(DWORD pid)
{
HANDLE h = OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, FALSE, pid);
if (!h) {
// No such process. Anything else (access denied: another user's
// process, a protected one) means it exists.
return GetLastError() != ERROR_INVALID_PARAMETER;
}
DWORD code = 0;
const bool running = GetExitCodeProcess(h, &code) && code == STILL_ACTIVE;
CloseHandle(h);
return running;
}
static void SweepStaleTempDirs()
{
char tmp[MAX_PATH] = {};
if (!GetTempPathA(MAX_PATH, tmp)) return;
const std::string base(tmp);
const std::string prefix = "anscenter_ort_";
const DWORD self = GetCurrentProcessId();
WIN32_FIND_DATAA fd{};
HANDLE hFind = FindFirstFileA((base + prefix + "*").c_str(), &fd);
if (hFind == INVALID_HANDLE_VALUE) return;
std::vector<std::string> stale;
do {
if (!(fd.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY)) continue;
const std::string name(fd.cFileName);
const std::string digits = name.substr(prefix.size());
if (digits.empty() || digits.size() > 10 ||
digits.find_first_not_of("0123456789") != std::string::npos)
continue;
const unsigned long long pid = std::stoull(digits);
if (pid == 0 || pid > 0xFFFFFFFFull || DWORD(pid) == self) continue;
if (!ProcessIsRunning(DWORD(pid))) stale.push_back(base + name);
} while (FindNextFileA(hFind, &fd));
FindClose(hFind);
for (const std::string& dir : stale) DeleteTempDir(dir);
if (!stale.empty())
std::cout << "[EPLoader] Removed " << stale.size()
<< " staging dir(s) left by processes that are gone." << std::endl;
}
#endif // _WIN32 #endif // _WIN32
@@ -418,6 +477,7 @@ namespace ANSCENTER {
// so it resolves providers relative to %TEMP% — where they don't exist. // so it resolves providers relative to %TEMP% — where they don't exist.
// Staging ALL provider DLLs alongside our renamed copy fixes this. // Staging ALL provider DLLs alongside our renamed copy fixes this.
SweepStaleTempDirs();
std::string temp_dir = MakeTempDir(); std::string temp_dir = MakeTempDir();
std::cout << "[EPLoader] Staging dir : " << temp_dir << std::endl; std::cout << "[EPLoader] Staging dir : " << temp_dir << std::endl;
@@ -162,6 +162,7 @@ static void testRoundTripAndRefusals(const fs::path& dir) {
req.useX25519 = true; req.useX25519 = true;
req.basePublic = id.basePub; req.basePublic = id.basePub;
req.chunk = 2048; req.chunk = 2048;
req.nonceSeed = 9;
req.signerId = "anssig-2026-09"; req.signerId = "anssig-2026-09";
req.signerSecret = id.signerSk; req.signerSecret = id.signerSk;
req.signerPublic = id.signerPk; req.signerPublic = id.signerPk;
@@ -251,79 +252,6 @@ static void testRoundTripAndRefusals(const fs::path& dir) {
// ANSODBase asks these three questions and nothing else: is this path a // ANSODBase asks these three questions and nothing else: is this path a
// package, can this build open one, and what came out of it. The engines // package, can this build open one, and what came out of it. The engines
// themselves need CUDA to compile, so this is where that contract is checked. // themselves need CUDA to compile, so this is where that contract is checked.
// ── every package gets its own key ────────────────────────────────────────
// pack() used to derive the content key, the ephemeral key and the nonces from
// one byte of seed, and the training box passed a seed made of the project's
// name and version: anyone holding a package could try the 256 seeds and read
// it without the base key. Two packs of the very same inputs must now share
// nothing that seals them — and still open with the base key.
static void testEachPackIsSealedAfresh(const fs::path& dir) {
const Identity id = fixtureIdentity();
const modelpkg::Keyring ring = fixtureKeyring(id);
modelpkg::PackRequest req;
req.keyId = "ansk-v1-base";
req.useX25519 = true;
req.basePublic = id.basePub;
req.chunk = 2048;
req.manifest = Json{{"task", "detection"},
{"family", "yolo"},
{"runtime", "onnx"},
{"models", Json::array({Json{{"id", "m0"},
{"payload", "p0"},
{"entry", true}}})}};
const auto model = fakeModel(5000, 11);
req.models.push_back({"p0", model});
req.models.push_back({"p1", model}); // two parts under one key: bases differ
const std::string pa = (dir / "fresh-a.ansmodel").string();
const std::string pb = (dir / "fresh-b.ansmodel").string();
std::string err;
CHECK(modelpkg::pack(pa, req, err), ("pack a: " + err).c_str());
CHECK(modelpkg::pack(pb, req, err), ("pack b: " + err).c_str());
modelpkg::Package a, b;
CHECK(modelpkg::open(pa, ring, a, err) && modelpkg::open(pb, ring, b, err),
("both open with the base key: " + err).c_str());
CHECK(a.contentKey != b.contentKey,
"two packs of identical inputs have different content keys");
bool seeded = false;
for (int seed = 0; seed < 256; ++seed) {
modelpkg::Key guess{};
for (size_t i = 0; i < guess.size(); ++i)
guess[i] = uint8_t(seed * 31 + i * 7 + 11);
seeded = seeded || guess == a.contentKey || guess == b.contentKey;
}
CHECK(!seeded, "and neither is one of the 256 a seed could have made");
const Json& wa = a.header["wraps"][0];
const Json& wb = b.header["wraps"][0];
CHECK(wa.value("epk", std::string()) != wb.value("epk", std::string()),
"each has its own ephemeral key");
CHECK(wa.value("nonce", std::string()) != wb.value("nonce", std::string()) &&
wa.value("ct", std::string()) != wb.value("ct", std::string()),
"and its own wrap nonce and wrapped key");
CHECK(a.header["payloads"][0].value("sha256", std::string()) !=
b.header["payloads"][0].value("sha256", std::string()),
"so the same model seals to different ciphertext");
CHECK(a.header["payloads"][0].value("nonceBase", std::string()) !=
a.header["payloads"][1].value("nonceBase", std::string()) &&
a.header["payloads"][0].value("nonceBase", std::string()) !=
a.header["manifest"].value("nonceBase", std::string()),
"and no two parts of one package share a nonce base");
std::vector<uint8_t> ga, gb;
CHECK(modelpkg::readPayload(a, "p0", ga, err) && ga == model &&
modelpkg::readPayload(b, "p1", gb, err) && gb == model,
"and both still give back the model that went in");
// The seeded path is for tests built as such; this one is not.
req.insecureDeterministicForTests = true;
CHECK(!modelpkg::pack((dir / "seeded.ansmodel").string(), req, err) &&
err.find("tests only") != std::string::npos,
"a build without the test switch refuses to pack from a seed");
}
static void testOpenForAnEngine(const fs::path& dataDir) { static void testOpenForAnEngine(const fs::path& dataDir) {
const std::string path = (dataDir / "interop.ansmodel").string(); const std::string path = (dataDir / "interop.ansmodel").string();
if (!fs::exists(path)) { if (!fs::exists(path)) {
@@ -448,6 +376,7 @@ static void testShapeFromTheGraph(const fs::path& dir) {
req.keyId = "ansk-v1-base"; req.keyId = "ansk-v1-base";
req.useX25519 = true; req.useX25519 = true;
req.basePublic = id.basePub; req.basePublic = id.basePub;
req.nonceSeed = 21;
req.signerId = "anssig-2026-09"; req.signerId = "anssig-2026-09";
req.signerSecret = id.signerSk; req.signerSecret = id.signerSk;
req.signerPublic = id.signerPk; req.signerPublic = id.signerPk;
@@ -543,7 +472,6 @@ int main(int argc, char** argv) {
testVectors(); testVectors();
testInterop(dataDir); testInterop(dataDir);
testRoundTripAndRefusals(tmp); testRoundTripAndRefusals(tmp);
testEachPackIsSealedAfresh(tmp);
testOpenForAnEngine(dataDir); testOpenForAnEngine(dataDir);
testShapeFromTheGraph(tmp); testShapeFromTheGraph(tmp);