Files
ANSCORE/core/ANSModelPackage/ANSModelPackage.h
T

769 lines
31 KiB
C++
Raw Normal View History

#ifndef ANSMODELPACKAGE_H
#define ANSMODELPACKAGE_H
#pragma once
// ANSModelPackage — reading (and, for tools and tests, writing) an ANS model
// package: one encrypted file carrying a model, its classes and how to run it.
//
// KEPT IN STEP WITH ANSMINDCORE's src/model/ansmodel.hpp. The two are the same
// format and must stay byte-compatible: the unit test here opens a package
// written by THAT implementation, which is what makes "compatible" a fact
// rather than an intention. Change them together.
//
// WHAT THIS REPLACES. A model shipped as a zip under a password every copy of
// the software knew — a password that has since been recovered — holding
// Categories/<name>.<id>, classes.names, a zero-length backend marker and
// train_last.onnx. Opening one meant inflating it to a directory, so the
// unprotected model sat on disk, was optimised, and was zipped back into a
// cache. This format keeps every fact that bundle carried and changes how it
// is carried: the manifest names the classes and the model family, and the
// bytes go straight from decryption into the runtime (engine/ibackend's
// ModelSource) without touching the filesystem.
//
// LAYOUT (all integers little-endian, offsets absolute):
//
// 0 8 magic "ANSMDL\0\0"
// 8 2 format version (1)
// 10 2 flags, bit 0 = more than one model
// 12 4 header length
// 16 n cleartext header, UTF-8 JSON — readable without any key
// … manifest, sealed — classes, task, family, models
// … payloads, sealed in chunks — the model bytes
//
// WHAT IS IN THE CLEAR, and why so little. The header carries only what a
// reader needs before it can decrypt — which key opens the file, where the
// sealed parts are — plus an optional card of counts for a store listing.
// Class names, metrics and the model's shape are in the manifest, sealed.
//
// SEALING. ChaCha20-Poly1305 (core/chacha20poly1305) with a content key that
// is random per package. Each chunk's nonce is a random per-payload base
// followed by the chunk index, and the associated data binds the key id, the
// payload id and that index — so a chunk cannot be moved between payloads or
// between packages, and a reordered file fails on the first chunk.
//
// WRAPPING. The content key is sealed to the base key with X25519: an
// ephemeral key pair per package, a shared secret against the base public key,
// and a key derived from it with HMAC-SHA256. Packaging tools therefore hold
// only the PUBLIC half and cannot open what they produce — which is the point
// of doing it asymmetrically when a symmetric key would have been simpler. A
// package may carry several wraps; a reader takes the first one it holds a key
// for. The older symmetric wrap is still read, so packages written before this
// keep opening.
//
// SIGNING. Ed25519 (RFC 8032, via the vendored monocypher-ed25519) over the
// header with its own `sig` removed. The signature is made by a RELEASE key
// whose public half travels in the package inside a certificate the ROOT key
// signed — so an engine that has never seen that release still verifies it,
// from the one root key it was built with. That is what lets signing keys
// rotate without stranding an engine that cannot be updated.
//
// EXPIRY AND REVOCATION. An expired certificate WARNS and loads: the date says
// when a key was in service, not whether the package is genuine, and an
// air-gapped box's clock drifts. A revoked key refuses outright. Strict mode
// turns expiry into a refusal for sites that keep accurate time.
#include <algorithm>
#include <array>
#include <chrono>
#include <cstdint>
#include <cstring>
#include <ctime>
#include <fstream>
#include <map>
#include <string>
#include <vector>
#include <nlohmann/json.hpp>
#include "ANSAead.h"
#include "ANSPackageBits.h"
extern "C" {
#include "monocypher-ed25519.h"
#include "monocypher.h"
}
namespace ANSCENTER {
namespace modelpkg {
using Json = nlohmann::json;
inline constexpr char kMagic[8] = {'A', 'N', 'S', 'M', 'D', 'L', '\0', '\0'};
inline constexpr uint16_t kVersion = 1;
inline constexpr uint16_t kFlagMultiModel = 1u << 0;
inline constexpr const char* kWrapSymmetric = "chacha20poly1305-key";
inline constexpr const char* kWrapX25519 = "x25519-hkdf-sha256-chacha20poly1305";
inline constexpr const char* kSigEd25519 = "ed25519";
inline constexpr uint32_t kDefaultChunk = 4u * 1024 * 1024;
using Key = std::array<uint8_t, aead::kKeyBytes>;
// What this build can open and what it trusts.
//
// A reader holds every base key it has ever shipped, because the base key does
// not rotate: a package names the key it was sealed to, and the answer to an
// unknown id is "this package needs a newer engine", not a decryption failure.
// It holds root public keys, which is how a release signing key it has never
// seen is still verified. And it holds the list of signing keys that have been
// burned, which is the only thing that refuses a validly signed package.
struct Keyring {
std::map<std::string, Key> keys; // symmetric base keys (legacy wrap)
std::map<std::string, Key> x25519; // base PRIVATE keys, by id
std::map<std::string, Key> roots; // root PUBLIC keys, by id
std::vector<std::string> revokedSigners;
bool strictExpiry = false; // off: an expired cert warns and loads
int64_t nowSec = 0; // 0 = ask the clock
void add(std::string id, const Key& k) { keys[std::move(id)] = k; }
void addX25519(std::string id, const Key& priv) { x25519[std::move(id)] = priv; }
void addRoot(std::string id, const Key& pub) { roots[std::move(id)] = pub; }
bool revoked(const std::string& signerId) const {
for (const auto& r : revokedSigners)
if (r == signerId) return true;
return false;
}
const Key* find(const std::string& id) const {
const auto it = keys.find(id);
return it == keys.end() ? nullptr : &it->second;
}
};
// One model inside a package: where its bytes are and how they are sealed.
struct PayloadRef {
std::string id;
uint64_t off = 0;
uint64_t len = 0; // sealed length, tags included
uint32_t chunk = kDefaultChunk;
std::array<uint8_t, 4> nonceBase{};
};
// An opened package: everything but the model bytes, which are read on demand
// so a two-gigabyte model is never decrypted to answer "what is this?".
struct Package {
std::string path;
uint16_t version = 0;
uint16_t flags = 0;
std::string keyId;
Json header; // the cleartext header, as read
Json manifest; // decrypted
std::vector<PayloadRef> payloads;
Key contentKey{};
std::string signerId; // the release key that signed it
bool certExpired = false; // loaded anyway; see the header comment
std::string warning; // what a caller should show, if anything
bool multiModel() const { return (flags & kFlagMultiModel) != 0; }
// The model a reader should run when it is given no choice: the manifest's
// entry model, or the first one.
std::string entryPayloadId() const {
const auto models = manifest.value("models", Json::array());
for (const auto& m : models)
if (m.value("entry", false)) return m.value("payload", std::string());
if (!models.empty()) return models[0].value("payload", std::string());
return payloads.empty() ? std::string() : payloads.front().id;
}
};
namespace detail {
inline uint16_t rd16(const uint8_t* p) { return uint16_t(p[0] | (p[1] << 8)); }
inline uint32_t rd32(const uint8_t* p) {
return uint32_t(p[0]) | (uint32_t(p[1]) << 8) | (uint32_t(p[2]) << 16) |
(uint32_t(p[3]) << 24);
}
inline void wr16(std::string& out, uint16_t v) {
out.push_back(char(v & 0xff));
out.push_back(char(v >> 8));
}
inline void wr32(std::string& out, uint32_t v) {
for (int i = 0; i < 4; ++i) out.push_back(char((v >> (8 * i)) & 0xff));
}
inline std::string b64(const std::vector<uint8_t>& v) {
return bits::base64Encode(std::string(v.begin(), v.end()));
}
inline std::vector<uint8_t> unb64(const std::string& s) {
std::vector<uint8_t> out;
if (!bits::base64Decode(s, out)) out.clear();
return out;
}
// The nonce for one chunk: a random per-payload base, then the index. Unique
// per (payload, chunk) under a content key that is unique per package.
inline std::array<uint8_t, aead::kNonceBytes> chunkNonce(
const std::array<uint8_t, 4>& base, uint64_t index) {
std::array<uint8_t, aead::kNonceBytes> n{};
std::memcpy(n.data(), base.data(), 4);
for (int i = 0; i < 8; ++i) n[4 + i] = uint8_t(index >> (8 * i));
return n;
}
// What each chunk is bound to. Moving a chunk to another payload, another
// index or another package changes this, and the tag then fails.
inline std::string chunkAad(const std::string& keyId, const std::string& payloadId,
uint64_t index, uint64_t chunkCount) {
return "ansmodel/v1|" + keyId + "|" + payloadId + "|" + std::to_string(index) +
"|" + std::to_string(chunkCount);
}
// The wrapping key from an X25519 shared secret. HMAC-SHA256 over the secret
// with a fixed label, which is HKDF-Extract: the raw secret is not uniform and
// must never be used as a key directly.
inline Key deriveWrapKey(const uint8_t shared[32], const std::string& keyId) {
const std::string salt = "ansmodel/v1|wrap|" + keyId;
const auto mac = bits::hmacSha256(
reinterpret_cast<const uint8_t*>(salt.data()), salt.size(), shared, 32);
Key k{};
std::memcpy(k.data(), mac.data(), k.size());
return k;
}
// The bytes a signature covers: the header with its own `sig` taken out, as
// canonical JSON. nlohmann orders object keys, so a reader that removes the
// same field and re-serialises reproduces exactly what the packer signed.
inline std::string signedBytes(Json header) {
header.erase("sig");
return header.dump();
}
// What the root vouches for: this release key, this id, until this date.
inline std::string certBytes(const Json& cert) {
return "ansmodel/v1|cert|" + cert.value("keyId", std::string()) + "|" +
cert.value("pub", std::string()) + "|" +
cert.value("notAfter", std::string());
}
inline int64_t nowSeconds() {
return int64_t(std::chrono::duration_cast<std::chrono::seconds>(
std::chrono::system_clock::now().time_since_epoch())
.count());
}
// "2027-09-01T00:00:00Z" → seconds. Returns 0 when it cannot be read, which
// is treated as "no expiry stated" rather than "expired".
inline int64_t parseIso8601(const std::string& t) {
std::tm tm{};
if (t.size() < 19) return 0;
tm.tm_year = std::atoi(t.substr(0, 4).c_str()) - 1900;
tm.tm_mon = std::atoi(t.substr(5, 2).c_str()) - 1;
tm.tm_mday = std::atoi(t.substr(8, 2).c_str());
tm.tm_hour = std::atoi(t.substr(11, 2).c_str());
tm.tm_min = std::atoi(t.substr(14, 2).c_str());
tm.tm_sec = std::atoi(t.substr(17, 2).c_str());
if (tm.tm_year <= 0 || tm.tm_mon < 0 || tm.tm_mon > 11) return 0;
#if defined(_WIN32)
return int64_t(_mkgmtime(&tm));
#else
return int64_t(timegm(&tm));
#endif
}
inline uint64_t chunkCountFor(uint64_t sealedLen, uint32_t chunk) {
const uint64_t perChunk = chunk + aead::kTagBytes;
return (sealedLen + perChunk - 1) / perChunk;
}
} // namespace detail
// ── reading ───────────────────────────────────────────────────────────────
// Open the header and manifest. Does not read a single model byte.
inline bool open(const std::string& path, const Keyring& ring, Package& out,
std::string& err) {
out = Package{};
out.path = path;
std::ifstream f(path, std::ios::binary);
if (!f) {
err = "cannot open " + path;
return false;
}
uint8_t fixed[16];
f.read(reinterpret_cast<char*>(fixed), sizeof(fixed));
if (f.gcount() != std::streamsize(sizeof(fixed))) {
err = "not a model package: file is too short";
return false;
}
if (std::memcmp(fixed, kMagic, sizeof(kMagic)) != 0) {
err = "not a model package: wrong magic";
return false;
}
out.version = detail::rd16(fixed + 8);
out.flags = detail::rd16(fixed + 10);
if (out.version > kVersion) {
err = "this package is format version " + std::to_string(out.version) +
", which needs a newer ANS engine";
return false;
}
const uint32_t headerLen = detail::rd32(fixed + 12);
if (headerLen == 0 || headerLen > 8u * 1024 * 1024) {
err = "header length is not plausible";
return false;
}
std::string headerText(headerLen, '\0');
f.read(headerText.data(), headerLen);
if (f.gcount() != std::streamsize(headerLen)) {
err = "truncated: the header is not all there";
return false;
}
out.header = Json::parse(headerText, nullptr, false);
if (out.header.is_discarded()) {
err = "the header is not readable JSON";
return false;
}
// ── the signature, before anything is decrypted ────────────────────────
const Json sig = out.header.value("sig", Json::object());
if (sig.is_null() || sig.empty()) {
err = "the package is not signed";
return false;
}
const std::string sigAlg = sig.value("alg", std::string());
if (sigAlg == kSigEd25519) {
const Json cert = sig.value("cert", Json::object());
const std::string signerId = cert.value("keyId", std::string());
const std::string rootId = cert.value("rootKeyId", std::string());
const auto signerPub = detail::unb64(cert.value("pub", std::string()));
const auto rootSig = detail::unb64(cert.value("rootSig", std::string()));
const auto headerSig = detail::unb64(sig.value("value", std::string()));
if (signerPub.size() != 32 || rootSig.size() != 64 || headerSig.size() != 64) {
err = "the signature block is malformed";
return false;
}
if (ring.revoked(signerId)) {
err = "this package was signed with a key that has been revoked (" +
signerId + ")";
return false;
}
const auto rootIt = ring.roots.find(rootId);
if (rootIt == ring.roots.end()) {
err = "the package is vouched for by an unknown root (" + rootId + ")";
return false;
}
// The root vouches for the release key; the release key signs the header.
const std::string cb = detail::certBytes(cert);
if (crypto_ed25519_check(rootSig.data(), rootIt->second.data(),
reinterpret_cast<const uint8_t*>(cb.data()),
cb.size()) != 0) {
err = "the signing key is not vouched for by the root it names";
return false;
}
const std::string sb = detail::signedBytes(out.header);
if (crypto_ed25519_check(headerSig.data(), signerPub.data(),
reinterpret_cast<const uint8_t*>(sb.data()),
sb.size()) != 0) {
err = "the package signature does not verify — it has been altered";
return false;
}
out.signerId = signerId;
// Expiry is a fact about the key's service, not about this package's
// authenticity, and an air-gapped clock drifts. Warn and load; strict
// mode is for sites that would rather fail closed.
const int64_t notAfter = detail::parseIso8601(cert.value("notAfter", std::string()));
const int64_t now = ring.nowSec > 0 ? ring.nowSec : detail::nowSeconds();
if (notAfter > 0 && now > notAfter) {
out.certExpired = true;
out.warning = "signed with a certificate that expired on " +
cert.value("notAfter", std::string()) + " (key " + signerId + ")";
if (ring.strictExpiry) {
err = out.warning + " — refused because strict expiry is on";
return false;
}
}
} else if (sigAlg != "none") {
err = "the package is signed in a way this build does not know (" + sigAlg + ")";
return false;
}
// Which key opens it. Every wrap is tried, so a package sealed to several
// keys opens on any engine holding one of them.
const auto wraps = out.header.value("wraps", Json::array());
if (wraps.empty()) {
err = "the package names no key";
return false;
}
std::vector<std::string> unknown;
bool unwrapped = false;
for (const auto& w : wraps) {
const std::string keyId = w.value("keyId", std::string());
const std::string alg = w.value("alg", std::string());
const auto nonceBytes = detail::unb64(w.value("nonce", std::string()));
const auto ct = detail::unb64(w.value("ct", std::string()));
if (nonceBytes.size() != aead::kNonceBytes ||
ct.size() != aead::kKeyBytes + aead::kTagBytes) {
err = "the wrapped key is malformed";
return false;
}
std::array<uint8_t, aead::kNonceBytes> nonce{};
std::memcpy(nonce.data(), nonceBytes.data(), nonce.size());
// The key that opens THIS wrap: either an X25519 shared secret against the
// ephemeral public key the packer left here, or — for packages written
// before the curve was vendored — the symmetric base key.
Key unwrapKey{};
if (alg == kWrapX25519) {
const auto it = ring.x25519.find(keyId);
if (it == ring.x25519.end()) {
unknown.push_back(keyId);
continue;
}
const auto epk = detail::unb64(w.value("epk", std::string()));
if (epk.size() != 32) {
err = "the wrap carries no usable ephemeral key";
return false;
}
uint8_t shared[32];
crypto_x25519(shared, it->second.data(), epk.data());
unwrapKey = detail::deriveWrapKey(shared, keyId);
crypto_wipe(shared, sizeof(shared));
} else if (alg == kWrapSymmetric) {
const Key* k = ring.find(keyId);
if (!k) {
unknown.push_back(keyId);
continue;
}
unwrapKey = *k;
} else {
err = "the package uses an unsupported wrap (" + alg + ")";
return false;
}
std::vector<uint8_t> plain;
if (!aead::open(unwrapKey, nonce, "ansmodel/v1|wrap|" + keyId, ct, plain)) {
err = "the wrapped key does not open with " + keyId +
" — the file is damaged or was sealed to a different key";
return false;
}
std::memcpy(out.contentKey.data(), plain.data(), out.contentKey.size());
out.keyId = keyId;
unwrapped = true;
break;
}
if (!unwrapped) {
err = "this package needs a newer ANS engine: it is sealed to key " +
(unknown.empty() ? std::string("(unnamed)") : unknown.front());
return false;
}
// Where the sealed parts are.
for (const auto& p : out.header.value("payloads", Json::array())) {
PayloadRef r;
r.id = p.value("id", std::string());
r.off = p.value("off", uint64_t(0));
r.len = p.value("len", uint64_t(0));
r.chunk = p.value("chunk", kDefaultChunk);
const auto nb = detail::unb64(p.value("nonceBase", std::string()));
if (r.id.empty() || r.len == 0 || r.chunk == 0 || nb.size() != 4) {
err = "a payload entry is malformed";
return false;
}
std::memcpy(r.nonceBase.data(), nb.data(), 4);
out.payloads.push_back(r);
}
if (out.payloads.empty()) {
err = "the package carries no model";
return false;
}
// The manifest, sealed as a single chunk under the payload id "manifest".
const Json m = out.header.value("manifest", Json::object());
const uint64_t moff = m.value("off", uint64_t(0));
const uint64_t mlen = m.value("len", uint64_t(0));
const auto mnb = detail::unb64(m.value("nonceBase", std::string()));
if (mlen == 0 || mnb.size() != 4) {
err = "the manifest entry is malformed";
return false;
}
std::vector<uint8_t> sealed(static_cast<size_t>(mlen), 0);
f.seekg(std::streamoff(moff));
f.read(reinterpret_cast<char*>(sealed.data()), std::streamsize(mlen));
if (f.gcount() != std::streamsize(mlen)) {
err = "truncated: the manifest is not all there";
return false;
}
std::array<uint8_t, 4> mbase{};
std::memcpy(mbase.data(), mnb.data(), 4);
std::vector<uint8_t> manifestText;
if (!aead::open(out.contentKey, detail::chunkNonce(mbase, 0),
detail::chunkAad(out.keyId, "manifest", 0, 1), sealed,
manifestText)) {
err = "the manifest does not verify — the package has been altered";
return false;
}
out.manifest =
Json::parse(std::string(manifestText.begin(), manifestText.end()), nullptr,
false);
if (out.manifest.is_discarded()) {
err = "the manifest is not readable JSON";
return false;
}
return true;
}
// Decrypt one model into memory. Chunk by chunk, each verified as it is read,
// so a damaged file fails on the chunk that is wrong rather than after the
// whole thing has been decrypted.
inline bool readPayload(const Package& pkg, const std::string& payloadId,
std::vector<uint8_t>& out, std::string& err) {
const PayloadRef* ref = nullptr;
for (const auto& p : pkg.payloads)
if (p.id == payloadId) ref = &p;
if (!ref) {
err = "no model called " + payloadId + " in this package";
return false;
}
std::ifstream f(pkg.path, std::ios::binary);
if (!f) {
err = "cannot reopen " + pkg.path;
return false;
}
f.seekg(std::streamoff(ref->off));
const uint64_t chunks = detail::chunkCountFor(ref->len, ref->chunk);
out.clear();
out.reserve(size_t(ref->len));
std::vector<uint8_t> sealed;
std::vector<uint8_t> plain;
uint64_t remaining = ref->len;
for (uint64_t i = 0; i < chunks; ++i) {
const uint64_t take =
remaining < uint64_t(ref->chunk) + aead::kTagBytes
? remaining
: uint64_t(ref->chunk) + aead::kTagBytes;
sealed.assign(size_t(take), 0);
f.read(reinterpret_cast<char*>(sealed.data()), std::streamsize(take));
if (f.gcount() != std::streamsize(take)) {
err = "truncated: model " + payloadId + " stops at chunk " +
std::to_string(i);
return false;
}
if (!aead::open(pkg.contentKey, detail::chunkNonce(ref->nonceBase, i),
detail::chunkAad(pkg.keyId, ref->id, i, chunks), sealed,
plain)) {
err = "chunk " + std::to_string(i) + " of " + payloadId +
" does not verify — the package has been altered";
out.clear();
return false;
}
out.insert(out.end(), plain.begin(), plain.end());
remaining -= take;
}
return true;
}
// ── writing (tools and tests) ─────────────────────────────────────────────
//
// The product's packages are written by the training box. This exists so the
// reader can be tested against files built by something other than itself,
// and so a converter can turn today's zip bundles into packages.
struct PackModel {
std::string id; // payload id, e.g. "p0"
std::vector<uint8_t> bytes;
};
struct PackRequest {
std::string keyId;
Key key{}; // symmetric base key (legacy wrap)
// Wrap to a base PUBLIC key instead, which is what a real packer does: it
// can then produce packages it cannot itself open.
bool useX25519 = false;
Key basePublic{};
// Sign as a release key the root vouches for. Empty signerSecret writes an
// unsigned package, which only tests should ever want.
std::string signerId;
std::vector<uint8_t> signerSecret; // 64 bytes, RFC 8032
std::vector<uint8_t> signerPublic; // 32 bytes
std::string rootKeyId;
std::vector<uint8_t> rootSig; // 64 bytes over the certificate
std::string notAfter; // ISO-8601, may be empty
Json manifest;
Json card; // optional, counts only
std::vector<PackModel> models;
uint32_t chunk = kDefaultChunk; // smaller in tests, to exercise chunking
// Deterministic nonces are for TESTS ONLY: a real package draws them from
// the platform CSPRNG, because a repeated (key, nonce) pair is fatal.
uint8_t nonceSeed = 0;
};
inline bool pack(const std::string& path, const PackRequest& req,
std::string& err) {
if (req.models.empty()) {
err = "a package with no model describes nothing";
return false;
}
// A content key per package. Tests pass a seed; a real packer must use the
// platform CSPRNG here.
Key contentKey{};
for (size_t i = 0; i < contentKey.size(); ++i)
contentKey[i] = uint8_t(req.nonceSeed * 31 + i * 7 + 11);
auto sealChunked = [&](const std::vector<uint8_t>& plain,
const std::string& payloadId,
const std::array<uint8_t, 4>& base,
std::string& blob) {
const uint64_t chunks =
plain.empty() ? 1 : (plain.size() + req.chunk - 1) / req.chunk;
for (uint64_t i = 0; i < chunks; ++i) {
const size_t from = size_t(i * req.chunk);
const size_t take = std::min<size_t>(req.chunk, plain.size() - from);
std::vector<uint8_t> part(plain.begin() + from, plain.begin() + from + take);
const auto sealed =
aead::seal(contentKey, detail::chunkNonce(base, i),
detail::chunkAad(req.keyId, payloadId, i, chunks), part);
blob.append(reinterpret_cast<const char*>(sealed.data()), sealed.size());
}
};
// The manifest first, then each model, so the header can name their offsets.
const std::string manifestText = req.manifest.dump();
const std::vector<uint8_t> manifestBytes(manifestText.begin(),
manifestText.end());
std::array<uint8_t, 4> manifestBase{uint8_t(req.nonceSeed + 1), 2, 3, 4};
std::string manifestBlob;
{
const auto sealed =
aead::seal(contentKey, detail::chunkNonce(manifestBase, 0),
detail::chunkAad(req.keyId, "manifest", 0, 1),
manifestBytes);
manifestBlob.assign(reinterpret_cast<const char*>(sealed.data()),
sealed.size());
}
std::vector<std::string> blobs;
std::vector<std::array<uint8_t, 4>> bases;
for (size_t i = 0; i < req.models.size(); ++i) {
std::array<uint8_t, 4> base{uint8_t(req.nonceSeed + 10 + i), 5, 6, 7};
std::string blob;
sealChunked(req.models[i].bytes, req.models[i].id, base, blob);
blobs.push_back(std::move(blob));
bases.push_back(base);
}
// Wrap the content key to the base key. One wrap here; the header's shape
// takes several, which is how a package opens on more than one key.
std::array<uint8_t, aead::kNonceBytes> wrapNonce{};
for (size_t i = 0; i < wrapNonce.size(); ++i)
wrapNonce[i] = uint8_t(req.nonceSeed + 100 + i);
const std::vector<uint8_t> keyPlain(contentKey.begin(), contentKey.end());
Key wrapKey = req.key;
std::vector<uint8_t> epk;
if (req.useX25519) {
// An ephemeral key pair per package: its public half travels in the
// header, its private half dies here, and the packer never holds anything
// that opens the result.
Key esk{};
for (size_t i = 0; i < esk.size(); ++i)
esk[i] = uint8_t(req.nonceSeed * 13 + i * 5 + 3);
epk.assign(32, 0);
crypto_x25519_public_key(epk.data(), esk.data());
uint8_t shared[32];
crypto_x25519(shared, esk.data(), req.basePublic.data());
wrapKey = detail::deriveWrapKey(shared, req.keyId);
crypto_wipe(shared, sizeof(shared));
crypto_wipe(esk.data(), esk.size());
}
const auto wrapped = aead::seal(wrapKey, wrapNonce,
"ansmodel/v1|wrap|" + req.keyId, keyPlain);
// Header offsets are known once the sizes are: the header is written first,
// so its own length has to be settled before the offsets inside it. Build it
// with placeholder offsets, measure, then fill them in — the size cannot
// change, because the numbers are written with a fixed width.
auto buildHeader = [&](uint64_t headerLen) {
Json h;
h["format"] = "ansmodel";
h["version"] = int(kVersion);
Json wrap{{"keyId", req.keyId},
{"alg", req.useX25519 ? kWrapX25519 : kWrapSymmetric},
{"nonce", detail::b64(std::vector<uint8_t>(wrapNonce.begin(),
wrapNonce.end()))},
{"ct", detail::b64(wrapped)}};
if (req.useX25519) wrap["epk"] = detail::b64(epk);
h["wraps"] = Json::array({wrap});
uint64_t at = 16 + headerLen;
h["manifest"] = Json{{"off", at},
{"len", uint64_t(manifestBlob.size())},
{"nonceBase", detail::b64(std::vector<uint8_t>(
manifestBase.begin(), manifestBase.end()))}};
at += manifestBlob.size();
Json payloads = Json::array();
for (size_t i = 0; i < blobs.size(); ++i) {
payloads.push_back(Json{
{"id", req.models[i].id},
{"off", at},
{"len", uint64_t(blobs[i].size())},
{"chunk", req.chunk},
{"nonceBase", detail::b64(std::vector<uint8_t>(bases[i].begin(),
bases[i].end()))},
{"sha256", bits::sha256Hex(std::string(blobs[i].begin(), blobs[i].end()))}});
at += blobs[i].size();
}
h["payloads"] = payloads;
if (!req.card.is_null()) h["card"] = req.card;
// The signature is written last, over everything above: `sig` is added
// with a placeholder of the exact length the real one takes, so signing
// cannot change the header's size and the offsets stay true.
if (req.signerSecret.size() == 64 && req.signerPublic.size() == 32) {
h["sig"] = Json{{"alg", kSigEd25519},
{"value", std::string(88, 'A')},
{"cert", Json{{"keyId", req.signerId},
{"pub", detail::b64(req.signerPublic)},
{"notAfter", req.notAfter},
{"rootKeyId", req.rootKeyId},
{"rootSig", detail::b64(req.rootSig)}}}};
} else {
h["sig"] = Json{{"alg", "none"}, {"value", ""}};
}
return h;
};
// Two passes: the first measures, the second is written. Offsets are plain
// JSON numbers, so a longer offset could lengthen the header — hence the
// loop, which settles in one extra step.
std::string headerText = buildHeader(0).dump();
for (int i = 0; i < 4; ++i) {
const std::string next = buildHeader(headerText.size()).dump();
if (next.size() == headerText.size()) {
headerText = next;
break;
}
headerText = next;
}
Json header = buildHeader(headerText.size());
if (req.signerSecret.size() == 64) {
// Sign the header as the reader will read it: without `sig`, canonically
// serialised. The placeholder above keeps the length identical, so putting
// the real signature in changes no offset.
const std::string sb = detail::signedBytes(header);
std::vector<uint8_t> value(64, 0);
crypto_ed25519_sign(value.data(), req.signerSecret.data(),
reinterpret_cast<const uint8_t*>(sb.data()), sb.size());
header["sig"]["value"] = detail::b64(value);
}
headerText = header.dump();
std::string file(kMagic, sizeof(kMagic));
detail::wr16(file, kVersion);
detail::wr16(file, req.models.size() > 1 ? kFlagMultiModel : uint16_t(0));
detail::wr32(file, uint32_t(headerText.size()));
file += headerText;
file += manifestBlob;
for (const auto& b : blobs) file += b;
std::ofstream o(path, std::ios::binary | std::ios::trunc);
if (!o) {
err = "cannot write " + path;
return false;
}
o.write(file.data(), std::streamsize(file.size()));
if (!o) {
err = "failed while writing " + path;
return false;
}
return true;
}
} // namespace modelpkg
} // namespace ANSCENTER
#endif // ANSMODELPACKAGE_H