The reader has been able to open packages for a while and has held no key, so it refused all of them. This is the seam that hands it one. The base private key is not in this repository and never will be. It is written by the ceremony (ANSMINDCORE's ansmodel-keys) into a generated header that is gitignored and produced from the key vault on the machine that builds a release. ANSModelKeys.h uses that file when it is there and says "no keys" when it is not, so a developer's checkout builds and runs with no secret in it: every zip still loads, and a package is refused with a sentence that says why. Keys are settled once — what the build carries, plus a developer's environment — unless a caller sets a keyring outright, which now means exactly that rather than depending on which call ran first. Checked with the real keys against the packages ANSMINDCORE's side produced: a package sealed by a box that holds only the public half opens here byte for byte, and one signed by the new release key verifies against the built-in root. If those two products disagreed, a model that runs in one would not run in the other, which is the failure this format exists to avoid. 40 checks in the package test, no failures. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
239 lines
9.4 KiB
C++
239 lines
9.4 KiB
C++
#ifndef ANSMODELOPEN_H
|
|
#define ANSMODELOPEN_H
|
|
#pragma once
|
|
// ANSModelOpen — the one place that knows how a model package reaches an engine.
|
|
//
|
|
// An engine's "model" has always been a path: a password-protected zip that
|
|
// ANSODBase inflates into a folder, after which the engine picks out
|
|
// train_last.onnx, classes.names and model_config.json. A .ansmodel package
|
|
// replaces all four with one sealed file, and this header is what turns it
|
|
// back into the things the engines already understand — the model's bytes, the
|
|
// class list, the task and the input size.
|
|
//
|
|
// THE PLAINTEXT NEVER REACHES THE DISK. That is the whole reason the format
|
|
// exists. OpenModelPackage decrypts into a buffer the caller owns; the caller
|
|
// hands that buffer to a session and wipes it. Nothing here writes a file.
|
|
//
|
|
// WHERE THE KEY COMES FROM. Three places, in this order: what the build ships
|
|
// with (ANSModelKeys.h, filled in by the key ceremony), what the environment
|
|
// offers a developer, and whatever a caller sets with SetProcessModelKeys().
|
|
// Nothing here reads a key out of the package or off the disk. With no keys at
|
|
// all, a package is refused with a sentence that says so, and every existing
|
|
// zip still loads — a build made without the ceremony runs everything except
|
|
// packages.
|
|
//
|
|
// The base private key must never sit in the binary as one contiguous
|
|
// constant; assembling it is the job of whatever calls SetProcessModelKeys,
|
|
// not of this header, which is why this header has no key material in it.
|
|
//
|
|
// The development fallback (ANSCORE_MODEL_KEY_ID / ANSCORE_MODEL_BASE_PRIVATE
|
|
// / ANSCORE_MODEL_ROOT_ID / ANSCORE_MODEL_ROOT_PUBLIC, all base64) exists so
|
|
// the path can be exercised before that ceremony exists. It is read once and
|
|
// announced, because a key that came from the environment is not the key a
|
|
// shipped product should be using.
|
|
|
|
#include <cstdio>
|
|
#include <cstdlib>
|
|
#include <cstring>
|
|
#include <mutex>
|
|
#include <string>
|
|
#include <vector>
|
|
|
|
#include "ANSModelKeys.h"
|
|
#include "ANSModelPackage.h"
|
|
#include "ANSOnnxShape.h"
|
|
|
|
namespace ANSCENTER {
|
|
namespace modelpkg {
|
|
|
|
// ── the keys this process opens packages with ─────────────────────────────
|
|
|
|
namespace detail {
|
|
|
|
inline std::mutex& keysMutex() {
|
|
static std::mutex m;
|
|
return m;
|
|
}
|
|
|
|
inline Keyring& keyringStorage() {
|
|
static Keyring ring;
|
|
return ring;
|
|
}
|
|
|
|
inline bool keyFromB64(const char* b64, Key& out) {
|
|
if (!b64 || !*b64) return false;
|
|
std::vector<uint8_t> raw;
|
|
if (!bits::base64Decode(b64, raw) || raw.size() != out.size()) return false;
|
|
std::memcpy(out.data(), raw.data(), out.size());
|
|
return true;
|
|
}
|
|
|
|
// True once this process's keyring is settled — either because the build's own
|
|
// keys were installed, or because a caller replaced them outright. A caller
|
|
// that sets keys means exactly that: what it set is what this process holds.
|
|
inline bool& keysSettled() {
|
|
static bool settled = false;
|
|
return settled;
|
|
}
|
|
|
|
// Development only: keys from the environment, and said out loud.
|
|
inline void loadEnvKeys(Keyring& ring) {
|
|
const char* id = std::getenv("ANSCORE_MODEL_KEY_ID");
|
|
const char* priv = std::getenv("ANSCORE_MODEL_BASE_PRIVATE");
|
|
Key k{};
|
|
if (id && *id && keyFromB64(priv, k)) {
|
|
ring.addX25519(id, k);
|
|
std::fprintf(stderr,
|
|
"[modelpkg] base key %s taken from the environment - for "
|
|
"development, not for a shipped build\n",
|
|
id);
|
|
}
|
|
const char* rootId = std::getenv("ANSCORE_MODEL_ROOT_ID");
|
|
const char* rootPub = std::getenv("ANSCORE_MODEL_ROOT_PUBLIC");
|
|
Key r{};
|
|
if (rootId && *rootId && keyFromB64(rootPub, r)) {
|
|
ring.addRoot(rootId, r);
|
|
std::fprintf(stderr,
|
|
"[modelpkg] root key %s taken from the environment\n",
|
|
rootId);
|
|
}
|
|
}
|
|
|
|
} // namespace detail
|
|
|
|
inline void SetProcessModelKeys(Keyring ring) {
|
|
std::lock_guard<std::mutex> lk(detail::keysMutex());
|
|
detail::keyringStorage() = std::move(ring);
|
|
detail::keysSettled() = true;
|
|
}
|
|
|
|
inline Keyring ProcessModelKeys() {
|
|
std::lock_guard<std::mutex> lk(detail::keysMutex());
|
|
Keyring& ring = detail::keyringStorage();
|
|
if (!detail::keysSettled()) {
|
|
detail::keysSettled() = true;
|
|
AddBuiltInKeys(ring); // what this build was given at the ceremony
|
|
detail::loadEnvKeys(ring); // and what a developer's environment adds
|
|
}
|
|
return ring; // by value: the caller may open on another thread
|
|
}
|
|
|
|
// True once this build can open anything at all. A product that has not had
|
|
// keys installed should say "this build cannot open model packages" rather
|
|
// than let every load fail one at a time.
|
|
inline bool HaveProcessModelKeys() {
|
|
const Keyring ring = ProcessModelKeys();
|
|
return !ring.x25519.empty() || !ring.keys.empty();
|
|
}
|
|
|
|
// ── recognising one ───────────────────────────────────────────────────────
|
|
|
|
inline bool IsPackagePath(const std::string& path) {
|
|
return path.size() > 9 &&
|
|
path.compare(path.size() - 9, 9, ".ansmodel") == 0;
|
|
}
|
|
|
|
// ── what an engine needs out of one ───────────────────────────────────────
|
|
|
|
// Everything the inflated folder used to carry, in the shape the engines read
|
|
// it: the model itself, the classes in order, and the geometry that
|
|
// model_config.json held.
|
|
struct OpenedModel {
|
|
std::vector<uint8_t> model; // decrypted; wipe it once loaded
|
|
std::vector<std::string> classes;
|
|
std::string task; // detection | classification | segmentation | obb | pose
|
|
std::string family; // yolo | yolo10 | rfdetr
|
|
std::string runtime; // onnx | openvino-ir
|
|
std::string legacyMarker; // what the zip would have carried
|
|
std::string name; // what to call this model in a log
|
|
int inputWidth = 0; // 0 when the package does not say
|
|
int inputHeight = 0;
|
|
std::string warning; // show it; the model still loaded
|
|
|
|
void wipe() {
|
|
if (!model.empty()) std::memset(model.data(), 0, model.size());
|
|
model.clear();
|
|
model.shrink_to_fit();
|
|
}
|
|
};
|
|
|
|
// Read what a package says about itself without decrypting the model. Cheap
|
|
// enough to call on a folder full of packages.
|
|
inline bool DescribeModelPackage(const std::string& path, OpenedModel& out,
|
|
std::string& err) {
|
|
Package pkg;
|
|
if (!open(path, ProcessModelKeys(), pkg, err)) return false;
|
|
const Json& m = pkg.manifest;
|
|
out.task = m.value("task", std::string("detection"));
|
|
out.family = m.value("family", std::string());
|
|
out.runtime = m.value("runtime", std::string("onnx"));
|
|
out.legacyMarker =
|
|
m.value("legacy", Json::object()).value("marker", std::string());
|
|
out.warning = pkg.warning;
|
|
out.classes.clear();
|
|
for (const auto& c : m.value("classes", Json::array()))
|
|
out.classes.push_back(c.value("name", std::string()));
|
|
const Json in = m.value("input", Json::object());
|
|
out.inputWidth = in.value("width", 0);
|
|
out.inputHeight = in.value("height", 0);
|
|
out.name = m.value("project", std::string());
|
|
if (out.name.empty()) out.name = path;
|
|
return true;
|
|
}
|
|
|
|
// Open one all the way: the description above, plus the model's bytes.
|
|
//
|
|
// The caller owns those bytes and should wipe them (OpenedModel::wipe) the
|
|
// moment the session has copied what it needs.
|
|
inline bool OpenModelPackage(const std::string& path, OpenedModel& out,
|
|
std::string& err) {
|
|
if (!HaveProcessModelKeys()) {
|
|
err = "this build has no model-package key, so it cannot open " + path;
|
|
return false;
|
|
}
|
|
Package pkg;
|
|
if (!open(path, ProcessModelKeys(), pkg, err)) return false;
|
|
|
|
const std::string entry = pkg.entryPayloadId();
|
|
if (entry.empty()) {
|
|
err = "the package names no model to run";
|
|
return false;
|
|
}
|
|
if (!readPayload(pkg, entry, out.model, err)) return false;
|
|
|
|
const Json& m = pkg.manifest;
|
|
out.task = m.value("task", std::string("detection"));
|
|
out.family = m.value("family", std::string());
|
|
out.runtime = m.value("runtime", std::string("onnx"));
|
|
out.legacyMarker =
|
|
m.value("legacy", Json::object()).value("marker", std::string());
|
|
out.warning = pkg.warning;
|
|
out.classes.clear();
|
|
for (const auto& c : m.value("classes", Json::array()))
|
|
out.classes.push_back(c.value("name", std::string()));
|
|
const Json in = m.value("input", Json::object());
|
|
out.inputWidth = in.value("width", 0);
|
|
out.inputHeight = in.value("height", 0);
|
|
out.name = m.value("project", std::string());
|
|
if (out.name.empty()) out.name = path;
|
|
|
|
// A package written before manifests carried an input size still has to
|
|
// answer, and the model in hand is the thing that knows. It costs nothing
|
|
// here: the bytes are already decrypted.
|
|
if (out.inputWidth <= 0 || out.inputHeight <= 0) {
|
|
onnxshape::InputShape shape;
|
|
std::string swhy;
|
|
if (onnxshape::fromMemory(out.model.data(), out.model.size(), shape,
|
|
swhy)) {
|
|
out.inputWidth = shape.width;
|
|
out.inputHeight = shape.height;
|
|
}
|
|
}
|
|
return true;
|
|
}
|
|
|
|
} // namespace modelpkg
|
|
} // namespace ANSCENTER
|
|
|
|
#endif // ANSMODELOPEN_H
|