feat(ANSODEngine): run a sealed model on TensorRT without unsealing it
A .ansmodel reaching the GPU path went to ModelOptimizer, which knows one way to open a model — unzip it — and failed on the first byte. Even past that, the TensorRT engine only ever parsed an ONNX file, so the one thing the format exists to prevent, a decrypted model on disk, was the only way it could have run. The builder now takes its bytes from memory when it has them. The path it is given still names the plan that gets cached beside it, because a plan is a built artefact and carries no model: the second run loads it and never needs the ONNX at all. For a package the plan lands in the folder the zip flow used, so a model converted from a zip reuses the plan that zip already built. OptimizeModelStr recognises a package and skips the unzip-and-optimise leg entirely: the handle below it opens the package in memory and builds the plan from the decrypted bytes, which is the same work by a shorter road. Still on the file path: the other TensorRT handlers (segmentation, pose, classification). Only the YOLO detector reads a package today. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
@@ -316,7 +316,7 @@ bool Engine<T>::buildLoadNetwork(std::string onnxModelPath, const std::array<flo
|
||||
return false;
|
||||
}
|
||||
}
|
||||
if (!FileExist(onnxModelPath)) {
|
||||
if (!hasOnnxBytes() && !FileExist(onnxModelPath)) {
|
||||
// ONNX model does not exist, try to find alternative precision engine
|
||||
std::cout << "Searching for alternative precision engine..." << std::endl;
|
||||
|
||||
@@ -390,12 +390,9 @@ bool Engine<T>::buildLoadNetwork(std::string onnxModelPath, const std::array<flo
|
||||
auto tempNetwork = std::unique_ptr<nvinfer1::INetworkDefinition>(TRT_CREATE_NETWORK(tempBuilder));
|
||||
auto tempParser = std::unique_ptr<nvonnxparser::IParser>(nvonnxparser::createParser(*tempNetwork, m_logger));
|
||||
|
||||
std::ifstream onnxFile(onnxModelPath, std::ios::binary | std::ios::ate);
|
||||
std::streamsize onnxSize = onnxFile.tellg();
|
||||
onnxFile.seekg(0, std::ios::beg);
|
||||
std::vector<char> onnxBuffer(onnxSize);
|
||||
if (!onnxFile.read(onnxBuffer.data(), onnxSize)) {
|
||||
std::cout << "Error: Failed to read ONNX file" << std::endl;
|
||||
std::vector<char> onnxBuffer;
|
||||
if (!readOnnxBytes(onnxModelPath, onnxBuffer)) {
|
||||
std::cout << "Error: Failed to read ONNX model" << std::endl;
|
||||
return false;
|
||||
}
|
||||
|
||||
@@ -1465,7 +1462,7 @@ bool Engine<T>::build(std::string onnxModelPath, const std::array<float, 3>& sub
|
||||
std::cout << "Engine file already exists: " << engineName << std::endl;
|
||||
return true;
|
||||
}
|
||||
if (!FileExist(onnxModelPath)) {
|
||||
if (!hasOnnxBytes() && !FileExist(onnxModelPath)) {
|
||||
std::cout << "Error: ONNX model file does not exist: " << onnxModelPath << std::endl;
|
||||
return false;
|
||||
}
|
||||
@@ -1502,16 +1499,14 @@ bool Engine<T>::build(std::string onnxModelPath, const std::array<float, 3>& sub
|
||||
return false;
|
||||
}
|
||||
|
||||
// Read the onnx file into memory
|
||||
std::ifstream file(onnxModelPath, std::ios::binary | std::ios::ate);
|
||||
std::streamsize size = file.tellg();
|
||||
file.seekg(0, std::ios::beg);
|
||||
|
||||
std::vector<char> buffer(size);
|
||||
if (!file.read(buffer.data(), size)) {
|
||||
std::cout << "Error: Unable to read ONNX file" << std::endl;
|
||||
// Get the onnx bytes: from the in-memory buffer of a sealed package, or
|
||||
// from the file on disk.
|
||||
std::vector<char> buffer;
|
||||
if (!readOnnxBytes(onnxModelPath, buffer)) {
|
||||
std::cout << "Error: Unable to read ONNX model" << std::endl;
|
||||
return false;
|
||||
}
|
||||
const std::streamsize size = static_cast<std::streamsize>(buffer.size());
|
||||
|
||||
std::cout << "ONNX model size: " << size / (1024 * 1024) << " MiB" << std::endl;
|
||||
|
||||
@@ -2471,16 +2466,14 @@ bool Engine<T>::buildWithRetry(std::string onnxModelPath,
|
||||
auto tempParser = std::unique_ptr<nvonnxparser::IParser>(
|
||||
nvonnxparser::createParser(*tempNetwork, m_logger));
|
||||
|
||||
std::ifstream onnxFile(onnxModelPath, std::ios::binary | std::ios::ate);
|
||||
if (!onnxFile.good()) {
|
||||
std::vector<char> onnxBuffer;
|
||||
const bool onnxReadable = readOnnxBytes(onnxModelPath, onnxBuffer);
|
||||
if (!onnxReadable) {
|
||||
ANS_DBG("TRT_Build", "buildWithRetry WARN cannot open ONNX for pre-parse path=%s",
|
||||
onnxModelPath.c_str());
|
||||
}
|
||||
if (onnxFile.good()) {
|
||||
std::streamsize onnxSize = onnxFile.tellg();
|
||||
onnxFile.seekg(0, std::ios::beg);
|
||||
std::vector<char> onnxBuffer(onnxSize);
|
||||
if (onnxFile.read(onnxBuffer.data(), onnxSize)) {
|
||||
if (onnxReadable) {
|
||||
{
|
||||
unsigned long sehRetryParse = 0;
|
||||
bool retryParsed = parseOnnxModelSafe(tempParser.get(),
|
||||
onnxBuffer.data(), onnxBuffer.size(), &sehRetryParse);
|
||||
@@ -2508,9 +2501,6 @@ bool Engine<T>::buildWithRetry(std::string onnxModelPath,
|
||||
(int)retryParsed,
|
||||
retryParsed ? tempNetwork->getNbInputs() : -1);
|
||||
}
|
||||
} else {
|
||||
ANS_DBG("TRT_Build", "buildWithRetry WARN ONNX read failed bytes=%zd",
|
||||
(long long)onnxSize);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
|
||||
@@ -88,8 +88,20 @@ public:
|
||||
const std::array<float, 3>& subVals,
|
||||
const std::array<float, 3>& divVals,
|
||||
bool normalize,
|
||||
int maxSlotsPerGpu)
|
||||
int maxSlotsPerGpu,
|
||||
const void* onnxData = nullptr,
|
||||
size_t onnxSize = 0)
|
||||
{
|
||||
// A sealed .ansmodel is decrypted in memory and never written out, so
|
||||
// the caller hands the bytes down here: every Engine this call builds
|
||||
// parses them instead of opening modelPath, which only names the
|
||||
// TensorRT plan that gets cached.
|
||||
auto newEngine = [&]() {
|
||||
auto e = std::make_shared<Engine<T>>(options);
|
||||
if (onnxData != nullptr && onnxSize > 0) e->setOnnxBytes(onnxData, onnxSize);
|
||||
return e;
|
||||
};
|
||||
|
||||
// Optimizer / temporary engines: maxSlotsPerGpu==0 means the caller
|
||||
// only needs a lightweight, non-shared engine (e.g., OptimizeModelStr).
|
||||
// Bypass the pool cache entirely:
|
||||
@@ -99,7 +111,7 @@ public:
|
||||
// Note: maxSlotsPerGpu==1 is now the normal "1 slot per GPU" multi-GPU
|
||||
// round-robin mode, so it goes through the pool path below.
|
||||
if (maxSlotsPerGpu == 0) {
|
||||
auto engine = std::make_shared<Engine<T>>(options);
|
||||
auto engine = newEngine();
|
||||
bool ok = engine->buildLoadNetwork(modelPath, subVals, divVals, normalize);
|
||||
return ok ? engine : nullptr;
|
||||
}
|
||||
@@ -141,13 +153,13 @@ public:
|
||||
}
|
||||
|
||||
// Cache miss — create new Engine pool
|
||||
auto engine = std::make_shared<Engine<T>>(options);
|
||||
auto engine = newEngine();
|
||||
bool ok = engine->buildLoadNetwork(modelPath, subVals, divVals, normalize, maxSlotsPerGpu);
|
||||
if (!ok) {
|
||||
// Step 1: Force-evict all pools with refcount=0 to reclaim VRAM
|
||||
int evicted = forceEvictPending();
|
||||
if (evicted > 0) {
|
||||
engine = std::make_shared<Engine<T>>(options);
|
||||
engine = newEngine();
|
||||
ok = engine->buildLoadNetwork(modelPath, subVals, divVals, normalize, maxSlotsPerGpu);
|
||||
}
|
||||
|
||||
@@ -156,7 +168,7 @@ public:
|
||||
// consumes ~300-500 MB vs ~50-100 MB for a simple loadNetwork.
|
||||
// Lightweight mode: tasks queue for a single shared slot — slower but works.
|
||||
if (!ok) {
|
||||
engine = std::make_shared<Engine<T>>(options);
|
||||
engine = newEngine();
|
||||
ok = engine->buildLoadNetwork(modelPath, subVals, divVals, normalize);
|
||||
}
|
||||
|
||||
@@ -183,7 +195,7 @@ public:
|
||||
}
|
||||
|
||||
// Final retry — try lightweight again after delay
|
||||
engine = std::make_shared<Engine<T>>(options);
|
||||
engine = newEngine();
|
||||
ok = engine->buildLoadNetwork(modelPath, subVals, divVals, normalize);
|
||||
}
|
||||
|
||||
|
||||
@@ -4,6 +4,48 @@
|
||||
#include <NvInferVersion.h> // also defines TRT version macros
|
||||
#include <cudnn_version.h> // CUDNN_MAJOR/MINOR/PATCHLEVEL
|
||||
#include <cuda_runtime.h> // cudaRuntimeGetVersion
|
||||
#include <cstring> // memset for wiping decrypted model bytes
|
||||
#include <fstream>
|
||||
|
||||
// ============================================================================
|
||||
// ONNX bytes in memory
|
||||
//
|
||||
// A sealed .ansmodel is decrypted into RAM and stays there: handing the
|
||||
// parser a buffer is what keeps the plaintext model off the disk. The build
|
||||
// path still takes a path, because the path names the TensorRT plan that is
|
||||
// cached beside it — a plan is a built artefact and carries no model.
|
||||
// ============================================================================
|
||||
template <typename T>
|
||||
void Engine<T>::setOnnxBytes(const void* data, size_t size) {
|
||||
clearOnnxBytes();
|
||||
if (data == nullptr || size == 0) return;
|
||||
const char* p = static_cast<const char*>(data);
|
||||
m_onnxBytes.assign(p, p + size);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
void Engine<T>::clearOnnxBytes() {
|
||||
if (m_onnxBytes.empty()) return;
|
||||
std::memset(m_onnxBytes.data(), 0, m_onnxBytes.size());
|
||||
m_onnxBytes.clear();
|
||||
m_onnxBytes.shrink_to_fit();
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
bool Engine<T>::readOnnxBytes(const std::string& onnxModelPath,
|
||||
std::vector<char>& buffer) const {
|
||||
if (!m_onnxBytes.empty()) {
|
||||
buffer = m_onnxBytes;
|
||||
return true;
|
||||
}
|
||||
std::ifstream file(onnxModelPath, std::ios::binary | std::ios::ate);
|
||||
if (!file.good()) return false;
|
||||
const std::streamsize size = file.tellg();
|
||||
if (size <= 0) return false;
|
||||
file.seekg(0, std::ios::beg);
|
||||
buffer.resize(static_cast<size_t>(size));
|
||||
return static_cast<bool>(file.read(buffer.data(), size));
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
void Engine<T>::transformOutput(std::vector<std::vector<std::vector<T>>> &input, std::vector<std::vector<T>> &output) {
|
||||
|
||||
Reference in New Issue
Block a user