ANSCENTER/ANSCORE
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor project structure

Browse Source
This commit is contained in:
Tuan Nghia Nguyen
2026-03-28 19:56:39 +11:00
parent 1d267378b2
commit 8a2e721058
511 changed files with 59 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files

106
core/ANSLibsLoader/ANSLibsLoader.cpp Normal file
View File

@@ -0,0 +1,106 @@
// ============================================================================
// ANSLibsLoader.cpp — Master orchestrator for all dynamic library loading
//
// Coordinates sub-loaders in the correct dependency order:
// 1. DLL search path injection (shared directory)
// 2. OpenCV (CvLoader) — pre-loads opencv_world + companions
// 3. OpenVINO (OvLoader) — pre-loads TBB + openvino.dll
// 4. NVIDIA stack (NvDynLoader) — TRT, cuDNN, CUDA
// 5. ONNX Runtime (EPLoader) — with execution provider selection
//
// Shutdown releases everything in reverse order.
// ============================================================================
#include "ANSLibsLoader.h"
#include "DynLibUtils.h"
#include "CvLoader.h"
#include "OvLoader.h"
#include "NvDynLoader.h"
#include "EPLoader.h"
#include <iostream>
#include <mutex>
namespace ANSCENTER {
// ─────────────────────────────────────────────────────────────────────────────
// Static member definitions
// ─────────────────────────────────────────────────────────────────────────────
bool ANSLibsLoader::s_initialized = false;
static std::mutex g_masterMutex;
// ─────────────────────────────────────────────────────────────────────────────
// Initialize
// ─────────────────────────────────────────────────────────────────────────────
void ANSLibsLoader::Initialize(const std::string& shared_dir, EngineType preferred)
{
std::lock_guard<std::mutex> lock(g_masterMutex);
if (s_initialized) return;
std::cout << "==========================================================\n"
<< "ANSLibsLoader: Initializing all dynamic libraries...\n"
<< " Shared dir : " << shared_dir << "\n"
<< " Engine pref : " << static_cast<int>(preferred) << "\n"
<< "==========================================================" << std::endl;
// ── STEP 1: Inject shared dir into search path ───────────────────────────
// This must happen before any DLL loading so all libraries in the shared
// directory are discoverable.
DynLib::InjectDllSearchPath(shared_dir);
// ── STEP 2: Pre-load OpenCV ──────────────────────────────────────────────
// Must happen before consuming DLLs (ANSODEngine etc.) are loaded, because
// those are import-linked against opencv_world and need the DLL available.
CvLoader::Initialize(shared_dir);
// ── STEP 3: Pre-load OpenVINO + TBB ──────────────────────────────────────
// OpenVINO is used directly (ov::Core) by many projects (ANSODEngine,
// ANSLPR, ANSFR, etc.). Pre-loading ensures the DLLs from the shared
// directory are resolved before any import-linked consumer is loaded.
OvLoader::Initialize(shared_dir);
// ── STEP 4: Load NVIDIA stack ────────────────────────────────────────────
// Only attempt if the preferred engine is NVIDIA or AUTO_DETECT.
if (preferred == EngineType::NVIDIA_GPU ||
preferred == EngineType::AUTO_DETECT)
{
NvDynLoader::Initialize();
}
// ── STEP 5: Load ONNX Runtime + select execution provider ────────────────
EPLoader::Initialize(shared_dir, preferred);
s_initialized = true;
std::cout << "==========================================================\n"
<< "ANSLibsLoader: Initialization complete.\n"
<< "==========================================================" << std::endl;
}
// ─────────────────────────────────────────────────────────────────────────────
// Shutdown
// ─────────────────────────────────────────────────────────────────────────────
void ANSLibsLoader::Shutdown()
{
std::lock_guard<std::mutex> lock(g_masterMutex);
// Shut down in reverse initialization order.
// Each sub-loader is idempotent and safe to call even if not initialized.
EPLoader::Shutdown();
NvDynLoader::Shutdown();
OvLoader::Shutdown();
CvLoader::Shutdown();
s_initialized = false;
}
// ─────────────────────────────────────────────────────────────────────────────
// IsInitialized
// ─────────────────────────────────────────────────────────────────────────────
bool ANSLibsLoader::IsInitialized()
{
return s_initialized;
}
} // namespace ANSCENTER

28
core/ANSLibsLoader/ANSLibsLoader.def Normal file
View File

@@ -0,0 +1,28 @@
; ============================================================================
; ANSLibsLoader.def — Module definition file
;
; Exports the extern "C" TRT / ONNX-parser factory stubs so consuming
; projects can link ANSLibsLoader.lib instead of nvinfer_XX.lib /
; nvonnxparser_XX.lib.
;
; WHY A .DEF FILE?
; ----------------
; NvInfer.h declares these functions with TENSORRTAPI, and our stub
; definitions in NvDynLoader.cpp must match that declaration exactly.
; Any mismatch in __declspec attributes or noexcept causes C2375.
; By using a .def file the linker exports the symbols without requiring
; __declspec(dllexport) in the source code at all.
; ============================================================================
LIBRARY ANSLibsLoader
EXPORTS
; TensorRT factory stubs (called by NvInfer.h inline wrappers)
createInferBuilder_INTERNAL
createInferRuntime_INTERNAL
createInferRefitter_INTERNAL
; ONNX parser factory stubs (called by NvOnnxParser.h inline wrappers)
createNvOnnxParser_INTERNAL
createNvOnnxParserRefitter_INTERNAL
getNvOnnxParserVersion

38
core/ANSLibsLoader/CMakeLists.txt Normal file
View File

@@ -0,0 +1,38 @@
# ANSLibsLoader — DLL that dynamically loads all ANS modules
add_library(ANSLibsLoader SHARED
ANSLibsLoader.cpp
CvLoader.cpp
DynLibUtils.cpp
EPLoader.cpp
NvDynLoader.cpp
OvLoader.cpp
dllmain.cpp
pch.cpp
pch.h
framework.h
include/ANSLibsLoader.h
include/CvLoader.h
include/DynLibUtils.h
include/EPLoader.h
include/NvDynLoader.h
include/OvLoader.h
)
target_include_directories(ANSLibsLoader PUBLIC
${CMAKE_CURRENT_SOURCE_DIR}
${CMAKE_CURRENT_SOURCE_DIR}/include
)
target_link_libraries(ANSLibsLoader
PRIVATE ANSLicensingSystem
PRIVATE labview
PRIVATE spdlog_dep
PRIVATE opencv
PRIVATE onnxruntime
PRIVATE tensorrt
PRIVATE openvino
PRIVATE CUDA::cudart_static
)
target_compile_definitions(ANSLibsLoader PRIVATE UNICODE _UNICODE ANSLIBSLOADER_EXPORTS NOMINMAX)
target_precompile_headers(ANSLibsLoader PRIVATE pch.h)

211
core/ANSLibsLoader/CvLoader.cpp Normal file
View File

@@ -0,0 +1,211 @@
// ============================================================================
// CvLoader.cpp — OpenCV DLL discovery and pre-loading
// ============================================================================
#include "CvLoader.h"
#include "DynLibUtils.h"
#include <iostream>
#include <mutex>
using namespace ANSCENTER::DynLib;
// ─────────────────────────────────────────────────────────────────────────────
// Static member definitions
// ─────────────────────────────────────────────────────────────────────────────
std::mutex CvLoader::s_mutex;
bool CvLoader::s_initialized = false;
CvInfo CvLoader::s_info;
LibHandle CvLoader::s_hCvWorld = nullptr;
LibHandle CvLoader::s_hCvImgHash = nullptr;
// ─────────────────────────────────────────────────────────────────────────────
// Helper — extract version string from version code (e.g. 4130 → "4.13.0")
// ─────────────────────────────────────────────────────────────────────────────
static std::string VersionFromCode(int code)
{
// OpenCV version encoding: XYZZ where X=major, Y=minor, ZZ=patch
// 4130 → major=4, minor=13, patch=0
// 4100 → major=4, minor=10, patch=0
// 490 → major=4, minor=9, patch=0
int major = code / 1000;
int minor = (code / 10) % 100;
int patch = code % 10;
return std::to_string(major) + "." +
std::to_string(minor) + "." +
std::to_string(patch);
}
// ─────────────────────────────────────────────────────────────────────────────
// Helper — extract version code from DLL filename
// "opencv_world4130.dll" → 4130
// "opencv_world4100.dll" → 4100
// "libopencv_world.so.413"→ 413 (Linux)
// ─────────────────────────────────────────────────────────────────────────────
static int ExtractVersionCode(const std::string& path)
{
#ifdef _WIN32
// Windows: opencv_world<XYZZ>.dll — digits between "opencv_world" and ".dll"
const std::string marker = "opencv_world";
size_t pos = path.find(marker);
if (pos == std::string::npos) return 0;
pos += marker.size();
size_t dot = path.find('.', pos);
if (dot == std::string::npos) return 0;
try { return std::stoi(path.substr(pos, dot - pos)); }
catch (...) { return 0; }
#else
// Linux: libopencv_world.so.<digits>
size_t dot = path.find_last_of('.');
if (dot == std::string::npos || dot + 1 >= path.size()) return 0;
try { return std::stoi(path.substr(dot + 1)); }
catch (...) { return 0; }
#endif
}
// ─────────────────────────────────────────────────────────────────────────────
// Candidate lists — search ANSCENTER shared dir first, then system PATH
// ─────────────────────────────────────────────────────────────────────────────
std::vector<std::string> CvLoader::CvWorldCandidates(const std::string& shared_dir)
{
std::vector<std::string> v;
// Versioned candidates, newest first (covers OpenCV 4.5 through 4.20+)
// Version codes: 4200, 4190, 4180, ..., 4130, ..., 4100, ..., 450
static const int versions[] = {
4200, 4190, 4180, 4170, 4160, 4150, 4140, 4130, 4120, 4110, 4100,
490, 480, 470, 460, 450
};
// ── Priority 1: ANSCENTER shared directory ────────────────────────────────
for (int ver : versions) {
#ifdef _WIN32
v.push_back(JoinPath(shared_dir,
"opencv_world" + std::to_string(ver) + ".dll"));
#else
v.push_back(JoinPath(shared_dir,
"libopencv_world.so." + std::to_string(ver)));
#endif
}
// ── Priority 2: System PATH / LD_LIBRARY_PATH ─────────────────────────────
for (int ver : versions) {
#ifdef _WIN32
v.push_back("opencv_world" + std::to_string(ver) + ".dll");
#else
v.push_back("libopencv_world.so." + std::to_string(ver));
#endif
}
return v;
}
std::vector<std::string> CvLoader::CvImgHashCandidates(const std::string& shared_dir)
{
std::vector<std::string> v;
static const int versions[] = {
4200, 4190, 4180, 4170, 4160, 4150, 4140, 4130, 4120, 4110, 4100,
490, 480, 470, 460, 450
};
// ── Priority 1: ANSCENTER shared directory ────────────────────────────────
for (int ver : versions) {
#ifdef _WIN32
v.push_back(JoinPath(shared_dir,
"opencv_img_hash" + std::to_string(ver) + ".dll"));
#else
v.push_back(JoinPath(shared_dir,
"libopencv_img_hash.so." + std::to_string(ver)));
#endif
}
// ── Priority 2: System PATH / LD_LIBRARY_PATH ─────────────────────────────
for (int ver : versions) {
#ifdef _WIN32
v.push_back("opencv_img_hash" + std::to_string(ver) + ".dll");
#else
v.push_back("libopencv_img_hash.so." + std::to_string(ver));
#endif
}
return v;
}
// ─────────────────────────────────────────────────────────────────────────────
// Initialize
// ─────────────────────────────────────────────────────────────────────────────
const CvInfo& CvLoader::Initialize(const std::string& shared_dir, bool verbose)
{
std::lock_guard<std::mutex> lock(s_mutex);
if (s_initialized) return s_info;
// Clean up any partially-loaded handles from a previous failed attempt
if (s_hCvWorld) { FreeLib(s_hCvWorld); s_hCvWorld = nullptr; }
if (s_hCvImgHash) { FreeLib(s_hCvImgHash); s_hCvImgHash = nullptr; }
if (verbose) {
std::cout << "==================================================" << std::endl;
std::cout << "[CvLoader] Discovering OpenCV libraries..." << std::endl;
std::cout << " Shared dir : " << shared_dir << std::endl;
std::cout << "==================================================" << std::endl;
}
// ── STEP 1: Inject shared dir into search path ───────────────────────────
// This ensures dependent DLLs (e.g. opencv_videoio_ffmpeg) can also be
// found when they are co-located with opencv_world in the shared dir.
InjectDllSearchPath(shared_dir);
// ── STEP 2: Pre-load opencv_world ────────────────────────────────────────
std::string worldPath;
s_hCvWorld = FindAndLoad(CvWorldCandidates(shared_dir), worldPath, verbose);
if (s_hCvWorld) {
s_info.dllPath = worldPath;
s_info.versionCode = ExtractVersionCode(worldPath);
s_info.version = VersionFromCode(s_info.versionCode);
s_info.loaded = true;
if (verbose) {
std::cout << "[CvLoader] OpenCV version = " << s_info.version
<< " (code " << s_info.versionCode << ")" << std::endl;
}
} else {
std::cout << "[CvLoader] WARNING: No opencv_world DLL found in "
<< shared_dir << " or system PATH." << std::endl;
std::cout << "[CvLoader] OpenCV will rely on import-lib resolution at load time."
<< std::endl;
}
// ── STEP 3: Pre-load companion modules ───────────────────────────────────
// These are optional — missing is a warning, not a failure.
std::string imgHashPath;
s_hCvImgHash = FindAndLoad(CvImgHashCandidates(shared_dir), imgHashPath, verbose);
if (!s_hCvImgHash && verbose) {
std::cout << "[CvLoader] opencv_img_hash not found (optional)." << std::endl;
}
s_initialized = true;
if (verbose) {
std::cout << "[CvLoader] "
<< (s_info.loaded ? "initialised successfully." : "WARNING — opencv_world not pre-loaded.")
<< std::endl;
std::cout << "==================================================" << std::endl;
}
return s_info;
}
// ─────────────────────────────────────────────────────────────────────────────
// Shutdown
// ─────────────────────────────────────────────────────────────────────────────
void CvLoader::Shutdown()
{
std::lock_guard<std::mutex> lock(s_mutex);
// Release in reverse order
if (s_hCvImgHash) { FreeLib(s_hCvImgHash); s_hCvImgHash = nullptr; }
if (s_hCvWorld) { FreeLib(s_hCvWorld); s_hCvWorld = nullptr; }
s_info = CvInfo{};
s_initialized = false;
}

125
core/ANSLibsLoader/DynLibUtils.cpp Normal file
View File

@@ -0,0 +1,125 @@
// ============================================================================
// DynLibUtils.cpp — Shared dynamic library loading utilities
// ============================================================================
#include "DynLibUtils.h"
#ifndef _WIN32
# include <sys/stat.h>
#endif
namespace ANSCENTER {
namespace DynLib {
// ── FileExists ───────────────────────────────────────────────────────────
bool FileExists(const std::string& path)
{
#ifdef _WIN32
DWORD attr = GetFileAttributesA(path.c_str());
return (attr != INVALID_FILE_ATTRIBUTES) &&
!(attr & FILE_ATTRIBUTE_DIRECTORY);
#else
struct stat st {};
return (::stat(path.c_str(), &st) == 0) && S_ISREG(st.st_mode);
#endif
}
// ── JoinPath ─────────────────────────────────────────────────────────────
std::string JoinPath(const std::string& base, const std::string& component)
{
#ifdef _WIN32
const char sep = '\\';
#else
const char sep = '/';
#endif
if (base.empty()) return component;
if (base.back() == sep || base.back() == '/')
return base + component;
return base + sep + component;
}
// ── InjectDllSearchPath ──────────────────────────────────────────────────
void InjectDllSearchPath(const std::string& dir)
{
#ifdef _WIN32
// Safe UTF-8 → wide-string conversion (handles non-ASCII paths correctly)
int wlen = MultiByteToWideChar(CP_UTF8, 0, dir.c_str(), -1, nullptr, 0);
std::wstring wdir(static_cast<size_t>(wlen > 0 ? wlen - 1 : 0), L'\0');
if (wlen > 0)
MultiByteToWideChar(CP_UTF8, 0, dir.c_str(), -1, wdir.data(), wlen);
DLL_DIRECTORY_COOKIE cookie = AddDllDirectory(wdir.c_str());
if (!cookie)
std::cerr << "[DynLib] WARNING: AddDllDirectory failed for: "
<< dir << " (error " << GetLastError() << ")" << std::endl;
char existing_path[32767] = {};
GetEnvironmentVariableA("PATH", existing_path, sizeof(existing_path));
std::string new_path = dir + ";" + existing_path;
if (!SetEnvironmentVariableA("PATH", new_path.c_str()))
std::cerr << "[DynLib] WARNING: SetEnvironmentVariable PATH failed."
<< std::endl;
#else
const char* existing = getenv("LD_LIBRARY_PATH");
std::string new_path = dir +
(existing ? (":" + std::string(existing)) : "");
setenv("LD_LIBRARY_PATH", new_path.c_str(), 1);
#endif
std::cout << "[DynLib] DLL search path injected: " << dir << std::endl;
}
// ── LoadLib ──────────────────────────────────────────────────────────────
LibHandle LoadLib(const std::string& path, bool globalLoad)
{
#ifdef _WIN32
(void)globalLoad;
return LoadLibraryExA(path.c_str(), nullptr,
LOAD_WITH_ALTERED_SEARCH_PATH);
#else
return globalLoad
? ::dlopen(path.c_str(), RTLD_LAZY | RTLD_GLOBAL)
: ::dlopen(path.c_str(), RTLD_LAZY | RTLD_LOCAL);
#endif
}
// ── FreeLib ──────────────────────────────────────────────────────────────
void FreeLib(LibHandle h)
{
if (!h) return;
#ifdef _WIN32
::FreeLibrary(h);
#else
::dlclose(h);
#endif
}
// ── GetSymbol ────────────────────────────────────────────────────────────
void* GetSymbol(LibHandle h, const char* sym)
{
if (!h) return nullptr;
#ifdef _WIN32
return reinterpret_cast<void*>(::GetProcAddress(h, sym));
#else
return ::dlsym(h, sym);
#endif
}
// ── FindAndLoad ──────────────────────────────────────────────────────────
LibHandle FindAndLoad(const std::vector<std::string>& candidates,
std::string& loadedPath,
bool verbose,
bool globalLoad)
{
for (const auto& name : candidates) {
LibHandle h = LoadLib(name, globalLoad);
if (h) {
loadedPath = name;
if (verbose)
std::cout << "[DynLib] Loaded: " << name << std::endl;
return h;
}
}
return LIB_INVALID;
}
} // namespace DynLib
} // namespace ANSCENTER

492
core/ANSLibsLoader/EPLoader.cpp Normal file
View File

@@ -0,0 +1,492 @@
// EPLoader.cpp
// Dynamic ONNX Runtime EP loader.
// Loads onnxruntime.dll at runtime — no onnxruntime.lib linkage required.
//
// Moved from ONNXEngine/ to ANSLibsLoader/.
// Compile this file in EXACTLY ONE project (ANSLibsLoader.dll).
// That project MUST define ANSLIBSLOADER_EXPORTS in its Preprocessor Definitions.
//
// Windows: LoadLibraryExW + AddDllDirectory + GetProcAddress
// Linux: dlopen (RTLD_NOW | RTLD_GLOBAL) + dlsym
#include "EPLoader.h"
#include "DynLibUtils.h"
// ORT C++ headers — included ONLY in this translation unit.
// ORT_API_MANUAL_INIT must be defined project-wide (Preprocessor Definitions)
// in every project that includes ORT headers, so all translation units see
// Ort::Global<void>::api_ as an extern rather than a default-constructed object.
#include <onnxruntime_cxx_api.h>
#include <algorithm>
#include <iostream>
#include <stdexcept>
#include <string>
#include <vector>
#ifdef _WIN32
# ifndef WIN32_LEAN_AND_MEAN
# define WIN32_LEAN_AND_MEAN
# endif
# include <windows.h>
#else
# include <dlfcn.h>
#endif
namespace ANSCENTER {
// ── Static member definitions ────────────────────────────────────────────
#ifdef ANSLIBSLOADER_EXPORTS
std::mutex EPLoader::s_mutex;
bool EPLoader::s_initialized = false;
EPInfo EPLoader::s_info;
# ifdef _WIN32
std::string EPLoader::s_temp_ort_path;
std::string EPLoader::s_temp_dir;
# endif
#endif
// ── File-scope state ─────────────────────────────────────────────────────
#ifdef _WIN32
static HMODULE s_ort_module = nullptr;
#else
static void* s_ort_module = nullptr;
#endif
static const OrtApi* s_ort_api = nullptr;
// ════════════════════════════════════════════════════════════════════════
// File-scope helpers (anonymous namespace — not exported)
// ════════════════════════════════════════════════════════════════════════
namespace {
// ── GetOrtApi ────────────────────────────────────────────────────────
const OrtApi* GetOrtApi()
{
if (!s_ort_module)
throw std::runtime_error(
"[EPLoader] ORT DLL not loaded — call EPLoader::Current() first.");
if (s_ort_api)
return s_ort_api;
#ifdef _WIN32
using Fn = const OrtApiBase* (ORT_API_CALL*)();
auto fn = reinterpret_cast<Fn>(
GetProcAddress(s_ort_module, "OrtGetApiBase"));
#else
using Fn = const OrtApiBase* (ORT_API_CALL*)();
auto fn = reinterpret_cast<Fn>(
dlsym(s_ort_module, "OrtGetApiBase"));
#endif
if (!fn)
throw std::runtime_error(
"[EPLoader] OrtGetApiBase symbol not found in loaded ORT DLL.");
const OrtApiBase* base = fn();
if (!base)
throw std::runtime_error(
"[EPLoader] OrtGetApiBase() returned null.");
int dllMaxApi = ORT_API_VERSION;
{
const char* verStr = base->GetVersionString();
int major = 0, minor = 0;
if (verStr && sscanf(verStr, "%d.%d", &major, &minor) == 2)
dllMaxApi = minor;
}
int targetApi = std::min(ORT_API_VERSION, dllMaxApi);
const OrtApi* api = base->GetApi(targetApi);
if (!api)
throw std::runtime_error(
"[EPLoader] No compatible ORT API version found in loaded DLL.");
s_ort_api = api;
return s_ort_api;
}
} // anonymous namespace
// ════════════════════════════════════════════════════════════════════════
// EPLoader public / private methods
// ════════════════════════════════════════════════════════════════════════
const char* OrtDllName()
{
#ifdef _WIN32
return "onnxruntime.dll";
#elif defined(__APPLE__)
return "libonnxruntime.dylib";
#else
return "libonnxruntime.so";
#endif
}
const char* EPLoader::SubdirName(EngineType type)
{
switch (type) {
case EngineType::NVIDIA_GPU: return "cuda";
case EngineType::AMD_GPU: return "directml";
case EngineType::OPENVINO_GPU: return "openvino";
case EngineType::CPU: return "cpu";
default: return "cpu";
}
}
const char* EPLoader::EngineTypeName(EngineType type)
{
switch (type) {
case EngineType::NVIDIA_GPU: return "NVIDIA_GPU";
case EngineType::AMD_GPU: return "AMD_GPU";
case EngineType::OPENVINO_GPU: return "OPENVINO_GPU";
case EngineType::CPU: return "CPU";
case EngineType::AUTO_DETECT: return "AUTO_DETECT";
default: return "UNKNOWN";
}
}
/*static*/
std::string EPLoader::ResolveEPDir(const std::string& shared_dir,
EngineType type)
{
std::string ep_base = DynLib::JoinPath(shared_dir, "ep");
std::string subdir = DynLib::JoinPath(ep_base, SubdirName(type));
std::string dll_probe = DynLib::JoinPath(subdir, OrtDllName());
if (DynLib::FileExists(dll_probe)) {
std::cout << "[EPLoader] EP subdir found: " << subdir << std::endl;
return subdir;
}
std::string flat_probe = DynLib::JoinPath(shared_dir, OrtDllName());
if (DynLib::FileExists(flat_probe)) {
std::cout << "[EPLoader] EP subdir not found — "
"using flat Shared/ (backward compat): "
<< shared_dir << std::endl;
return shared_dir;
}
std::cerr << "[EPLoader] WARNING: " << OrtDllName() << " not found in:\n"
<< " " << subdir << "\n"
<< " " << shared_dir << "\n"
<< " LoadOrtDll will fail with a clear error."
<< std::endl;
return subdir;
}
EngineType EPLoader::AutoDetect()
{
std::cout << "[EPLoader] Auto-detecting hardware..." << std::endl;
ANSLicenseHelper helper;
EngineType detected = helper.CheckHardwareInformation();
std::cout << "[EPLoader] Detected: " << EngineTypeName(detected) << std::endl;
return detected;
}
/*static*/
const EPInfo& EPLoader::Initialize(const std::string& shared_dir,
EngineType preferred)
{
if (s_initialized) return s_info;
std::lock_guard<std::mutex> lock(s_mutex);
if (s_initialized) return s_info;
std::cout << "[EPLoader] Initializing..." << std::endl;
std::cout << "[EPLoader] Shared dir : " << shared_dir << std::endl;
std::cout << "[EPLoader] Preferred EP : " << EngineTypeName(preferred) << std::endl;
EngineType type = (preferred == EngineType::AUTO_DETECT)
? AutoDetect() : preferred;
std::string ep_dir = ResolveEPDir(shared_dir, type);
// When the EP lives in a subdirectory (e.g. ep/openvino/), provider
// DLLs may depend on runtime libraries that live in the parent
// shared_dir (e.g. openvino.dll). Inject shared_dir into the DLL
// search path so Windows can resolve those dependencies.
if (ep_dir != shared_dir)
DynLib::InjectDllSearchPath(shared_dir);
LoadOrtDll(ep_dir);
s_info.type = type;
s_info.libraryDir = ep_dir;
s_info.fromSubdir = (ep_dir != shared_dir);
s_initialized = true;
std::cout << "[EPLoader] Ready. EP=" << EngineTypeName(type)
<< " dir=" << ep_dir << std::endl;
return s_info;
}
/*static*/
const EPInfo& EPLoader::Current()
{
if (!s_initialized)
return Initialize(DEFAULT_SHARED_DIR, EngineType::AUTO_DETECT);
return s_info;
}
/*static*/
bool EPLoader::IsInitialized()
{
return s_initialized;
}
void* EPLoader::GetOrtApiRaw()
{
Current();
const OrtApi* api = GetOrtApi();
if (!api)
throw std::runtime_error(
"[EPLoader] GetOrtApiRaw: OrtApi not available.");
return static_cast<void*>(const_cast<OrtApi*>(api));
}
#ifdef _WIN32
// ── MakeTempDir ──────────────────────────────────────────────────────────
static std::string MakeTempDir()
{
char tmp[MAX_PATH] = {};
GetTempPathA(MAX_PATH, tmp);
std::string dir = std::string(tmp)
+ "anscenter_ort_"
+ std::to_string(GetCurrentProcessId());
CreateDirectoryA(dir.c_str(), nullptr);
return dir;
}
// ── CopyDirToTemp ─────────────────────────────────────────────────────────
static void CopyDirToTemp(const std::string& ep_dir,
const std::string& temp_dir)
{
std::string pattern = ep_dir + "\\*.dll";
WIN32_FIND_DATAA fd{};
HANDLE hFind = FindFirstFileA(pattern.c_str(), &fd);
if (hFind == INVALID_HANDLE_VALUE) {
std::cerr << "[EPLoader] WARNING: No DLLs found in ep_dir: "
<< ep_dir << std::endl;
return;
}
int copied = 0, skipped = 0;
do {
std::string src = ep_dir + "\\" + fd.cFileName;
std::string dst = temp_dir + "\\" + fd.cFileName;
if (CopyFileA(src.c_str(), dst.c_str(), /*bFailIfExists=*/FALSE)) {
++copied;
std::cout << "[EPLoader] copied: " << fd.cFileName << std::endl;
}
else {
DWORD err = GetLastError();
if (err != ERROR_FILE_EXISTS && err != ERROR_ALREADY_EXISTS) {
std::cerr << "[EPLoader] WARNING: could not copy "
<< fd.cFileName
<< " (err=" << err << ") — skipping." << std::endl;
}
++skipped;
}
} while (FindNextFileA(hFind, &fd));
FindClose(hFind);
std::cout << "[EPLoader] Staged " << copied << " DLL(s) to temp dir"
<< (skipped ? " (" + std::to_string(skipped) + " already present)" : "")
<< std::endl;
}
// ── DeleteTempDir ─────────────────────────────────────────────────────────
static void DeleteTempDir(const std::string& dir)
{
if (dir.empty()) return;
std::string pattern = dir + "\\*";
WIN32_FIND_DATAA fd{};
HANDLE hFind = FindFirstFileA(pattern.c_str(), &fd);
if (hFind != INVALID_HANDLE_VALUE) {
do {
if (strcmp(fd.cFileName, ".") == 0 ||
strcmp(fd.cFileName, "..") == 0) continue;
std::string f = dir + "\\" + fd.cFileName;
DeleteFileA(f.c_str());
} while (FindNextFileA(hFind, &fd));
FindClose(hFind);
}
RemoveDirectoryA(dir.c_str());
std::cout << "[EPLoader] Temp staging dir deleted: " << dir << std::endl;
}
#endif // _WIN32
// ── LoadOrtDll ───────────────────────────────────────────────────────────
void EPLoader::LoadOrtDll(const std::string& ep_dir)
{
if (s_ort_module) {
std::cout << "[EPLoader] ORT DLL already loaded — skipping." << std::endl;
return;
}
// Inject ep_dir into the DLL search path so cudart64_*.dll and other
// CUDA runtime DLLs that are NOT copied to temp are still found.
DynLib::InjectDllSearchPath(ep_dir);
std::string src_path = DynLib::JoinPath(ep_dir, OrtDllName());
std::cout << "[EPLoader] ORT source : " << src_path << std::endl;
#ifdef _WIN32
// ── Windows: stage ALL ep_dir DLLs into a process-unique temp folder ────
std::string temp_dir = MakeTempDir();
std::cout << "[EPLoader] Staging dir : " << temp_dir << std::endl;
CopyDirToTemp(ep_dir, temp_dir);
std::string ort_alias = temp_dir + "\\anscenter_ort_"
+ std::to_string(GetCurrentProcessId()) + ".dll";
std::string ort_in_temp = temp_dir + "\\" + OrtDllName();
if (!CopyFileA(ort_in_temp.c_str(), ort_alias.c_str(), /*bFailIfExists=*/FALSE)) {
DWORD err = GetLastError();
if (err != ERROR_FILE_EXISTS && err != ERROR_ALREADY_EXISTS) {
throw std::runtime_error(
"[EPLoader] Failed to create ORT alias in temp dir.\n"
" src : " + ort_in_temp + "\n"
" dst : " + ort_alias + "\n"
" err : " + std::to_string(err));
}
}
std::cout << "[EPLoader] ORT alias : " << ort_alias << std::endl;
DynLib::InjectDllSearchPath(temp_dir);
HMODULE hExisting = GetModuleHandleA("onnxruntime.dll");
if (hExisting) {
char existingPath[MAX_PATH] = {};
GetModuleFileNameA(hExisting, existingPath, MAX_PATH);
std::string existingStr(existingPath);
std::transform(existingStr.begin(), existingStr.end(),
existingStr.begin(), ::tolower);
if (existingStr.find("system32") != std::string::npos ||
existingStr.find("syswow64") != std::string::npos) {
std::cerr << "[EPLoader] WARNING: System ORT is resident ("
<< existingPath << ") - our alias overrides it." << std::endl;
}
}
// Safe UTF-8 → wide-string conversion (handles non-ASCII paths correctly)
int wlen = MultiByteToWideChar(CP_UTF8, 0, ort_alias.c_str(), -1, nullptr, 0);
std::wstring walias(static_cast<size_t>(wlen > 0 ? wlen - 1 : 0), L'\0');
if (wlen > 0)
MultiByteToWideChar(CP_UTF8, 0, ort_alias.c_str(), -1, walias.data(), wlen);
s_ort_module = LoadLibraryExW(
walias.c_str(),
nullptr,
LOAD_WITH_ALTERED_SEARCH_PATH);
if (!s_ort_module) {
DWORD err = GetLastError();
DeleteTempDir(temp_dir);
throw std::runtime_error(
"[EPLoader] LoadLibraryExW failed: " + ort_alias +
"\n Windows error code: " + std::to_string(err) +
"\n Ensure all CUDA runtime DLLs (cudart64_*.dll etc.)"
"\n exist in: " + ep_dir);
}
s_temp_ort_path = ort_alias;
s_temp_dir = temp_dir;
#else
// ── Linux / macOS ─────────────────────────────────────────────────────
s_ort_module = dlopen(src_path.c_str(), RTLD_NOW | RTLD_GLOBAL);
if (!s_ort_module) {
throw std::runtime_error(
std::string("[EPLoader] dlopen failed: ") + src_path +
"\n " + dlerror());
}
#endif
// ── Bootstrap ORT C++ API ─────────────────────────────────────────────
using OrtGetApiBase_fn = const OrtApiBase* (ORT_API_CALL*)();
#ifdef _WIN32
auto fn = reinterpret_cast<OrtGetApiBase_fn>(
GetProcAddress(s_ort_module, "OrtGetApiBase"));
#else
auto fn = reinterpret_cast<OrtGetApiBase_fn>(
dlsym(s_ort_module, "OrtGetApiBase"));
#endif
if (!fn)
throw std::runtime_error(
"[EPLoader] OrtGetApiBase not exported — is this a genuine onnxruntime build?\n"
" path: " + src_path);
const OrtApiBase* base = fn();
if (!base)
throw std::runtime_error(
"[EPLoader] OrtGetApiBase() returned null from: " + src_path);
// ── Version negotiation ───────────────────────────────────────────────
int dllMaxApi = ORT_API_VERSION;
{
const char* verStr = base->GetVersionString();
int major = 0, minor = 0;
if (verStr && sscanf(verStr, "%d.%d", &major, &minor) == 2)
dllMaxApi = minor;
}
int targetApi = std::min(ORT_API_VERSION, dllMaxApi);
if (targetApi < ORT_API_VERSION) {
std::cerr << "[EPLoader] WARNING: ORT DLL version "
<< base->GetVersionString()
<< " supports up to API " << dllMaxApi
<< " but headers expect API " << ORT_API_VERSION << ".\n"
<< " Using API " << targetApi
<< ". Consider upgrading onnxruntime.dll in ep/ to match SDK headers."
<< std::endl;
}
const OrtApi* api = base->GetApi(targetApi);
if (!api)
throw std::runtime_error(
"[EPLoader] GetApi(" + std::to_string(targetApi) +
") returned null — the DLL may be corrupt.");
s_ort_api = api;
Ort::Global<void>::api_ = api;
std::cout << "[EPLoader] ORT loaded successfully." << std::endl;
std::cout << "[EPLoader] ORT DLL version : " << base->GetVersionString() << std::endl;
std::cout << "[EPLoader] ORT header API : " << ORT_API_VERSION << std::endl;
std::cout << "[EPLoader] ORT active API : " << targetApi << std::endl;
}
// ── Shutdown ──────────────────────────────────────────────────────────────
void EPLoader::Shutdown()
{
std::lock_guard<std::mutex> lock(s_mutex);
if (s_ort_module) {
std::cout << "[EPLoader] Unloading ORT DLL..." << std::endl;
#ifdef _WIN32
FreeLibrary(s_ort_module);
if (!s_temp_dir.empty()) {
DeleteTempDir(s_temp_dir);
s_temp_dir.clear();
}
s_temp_ort_path.clear();
#else
dlclose(s_ort_module);
#endif
s_ort_module = nullptr;
}
s_ort_api = nullptr;
s_initialized = false;
s_info = EPInfo{};
std::cout << "[EPLoader] Shutdown complete." << std::endl;
}
} // namespace ANSCENTER

646
core/ANSLibsLoader/NvDynLoader.cpp Normal file
View File

@@ -0,0 +1,646 @@
// ============================================================================
// NvDynLoader.cpp
//
// Moved from TensorRTAPI/ to ANSLibsLoader/ for centralized library management.
// Now compiled into ANSLibsLoader.dll — all extern "C" stubs are exported via
// ANSLIBS_API so consuming projects can link ANSLibsLoader.lib instead of
// nvinfer_XX.lib / nvonnxparser_XX.lib.
//
// Two responsibilities:
//
// 1. Provide extern "C" stub DEFINITIONS for every symbol that TRT / ONNX
// parser inline wrappers reference. This satisfies the linker without
// linking nvinfer_XX.lib / nvonnxparser_XX.lib. Each stub forwards the
// call through a function pointer that Initialize() populates at runtime.
//
// 2. Implement NvDynLoader::Initialize() which searches for installed NVIDIA
// DLLs (trying multiple major versions, newest first) and binds all
// required function pointers.
// ============================================================================
#include "NvDynLoader.h"
#include "DynLibUtils.h"
#include <iostream>
#include <cassert>
#include <mutex>
using namespace ANSCENTER::DynLib;
// ─────────────────────────────────────────────────────────────────────────────
// Static member definitions
// ─────────────────────────────────────────────────────────────────────────────
bool NvDynLoader::s_initialized = false;
int NvDynLoader::s_trtMajor = 0;
std::string NvDynLoader::s_trtPath;
std::string NvDynLoader::s_onnxPath;
std::string NvDynLoader::s_cudaPath;
std::string NvDynLoader::s_cudnnPath;
LibHandle NvDynLoader::s_hTrt = nullptr;
LibHandle NvDynLoader::s_hOnnx = nullptr;
LibHandle NvDynLoader::s_hCuda = nullptr;
LibHandle NvDynLoader::s_hCudnn = nullptr;
NvDynLoader::PfnBuilder* NvDynLoader::pfn_createInferBuilder_INTERNAL = nullptr;
NvDynLoader::PfnRuntime* NvDynLoader::pfn_createInferRuntime_INTERNAL = nullptr;
NvDynLoader::PfnRefitter* NvDynLoader::pfn_createInferRefitter_INTERNAL = nullptr;
NvDynLoader::PfnParser* NvDynLoader::pfn_createNvOnnxParser_INTERNAL = nullptr;
NvDynLoader::PfnParserRefitter* NvDynLoader::pfn_createNvOnnxParserRefitter_INTERNAL = nullptr;
NvDynLoader::PfnGetParserVersion* NvDynLoader::pfn_getNvOnnxParserVersion = nullptr;
#ifdef NV_DYNAMIC_CUDA
cudaError_t (*NvDynLoader::pfn_cudaGetDeviceCount) (int*) = nullptr;
cudaError_t (*NvDynLoader::pfn_cudaSetDevice) (int) = nullptr;
cudaError_t (*NvDynLoader::pfn_cudaGetDeviceProperties) (cudaDeviceProp*, int) = nullptr;
cudaError_t (*NvDynLoader::pfn_cudaDeviceSetLimit) (cudaLimit, size_t) = nullptr;
cudaError_t (*NvDynLoader::pfn_cudaDeviceSynchronize) () = nullptr;
cudaError_t (*NvDynLoader::pfn_cudaDeviceGetStreamPriorityRange)(int*, int*) = nullptr;
cudaError_t (*NvDynLoader::pfn_cudaMalloc) (void**, size_t) = nullptr;
cudaError_t (*NvDynLoader::pfn_cudaFree) (void*) = nullptr;
cudaError_t (*NvDynLoader::pfn_cudaMemset) (void*, int, size_t) = nullptr;
cudaError_t (*NvDynLoader::pfn_cudaMemGetInfo) (size_t*, size_t*) = nullptr;
cudaError_t (*NvDynLoader::pfn_cudaMemcpy) (void*, const void*, size_t, cudaMemcpyKind) = nullptr;
cudaError_t (*NvDynLoader::pfn_cudaMemcpyAsync) (void*, const void*, size_t, cudaMemcpyKind, cudaStream_t) = nullptr;
cudaError_t (*NvDynLoader::pfn_cudaStreamCreate) (cudaStream_t*) = nullptr;
cudaError_t (*NvDynLoader::pfn_cudaStreamCreateWithPriority) (cudaStream_t*, unsigned int, int) = nullptr;
cudaError_t (*NvDynLoader::pfn_cudaStreamDestroy) (cudaStream_t) = nullptr;
cudaError_t (*NvDynLoader::pfn_cudaStreamSynchronize) (cudaStream_t) = nullptr;
cudaError_t (*NvDynLoader::pfn_cudaStreamWaitEvent) (cudaStream_t, cudaEvent_t, unsigned int) = nullptr;
cudaError_t (*NvDynLoader::pfn_cudaEventCreate) (cudaEvent_t*) = nullptr;
cudaError_t (*NvDynLoader::pfn_cudaEventCreateWithFlags) (cudaEvent_t*, unsigned int) = nullptr;
cudaError_t (*NvDynLoader::pfn_cudaEventRecord) (cudaEvent_t, cudaStream_t) = nullptr;
cudaError_t (*NvDynLoader::pfn_cudaEventDestroy) (cudaEvent_t) = nullptr;
const char* (*NvDynLoader::pfn_cudaGetErrorString) (cudaError_t) = nullptr;
cudaError_t (*NvDynLoader::pfn_cudaGetLastError) () = nullptr;
#endif // NV_DYNAMIC_CUDA
// ─────────────────────────────────────────────────────────────────────────────
// extern "C" stubs (exported from ANSLibsLoader.dll via .def file)
//
// These provide the linker symbols that TRT / ONNX-parser inline wrappers
// reference. TENSORRTAPI / NVONNXPARSER_API are overridden to empty in
// NvDynLoader.h, so the NvInfer.h / NvOnnxParser.h declarations are plain
// extern "C" (no __declspec attribute). Our definitions below are also
// plain extern "C" — no __declspec attribute — so the linkage matches.
//
// The symbols are exported from the DLL via ANSLibsLoader.def, NOT via
// __declspec(dllexport). This avoids the C2375 "different linkage" error
// that occurs when NvInfer.h and our definitions carry different attributes.
// ─────────────────────────────────────────────────────────────────────────────
extern "C" {
void* createInferBuilder_INTERNAL(void* logger, int32_t version) noexcept
{
assert(NvDynLoader::pfn_createInferBuilder_INTERNAL &&
"NvDynLoader::Initialize() has not been called before the first TRT use.");
return NvDynLoader::pfn_createInferBuilder_INTERNAL(logger, version);
}
void* createInferRuntime_INTERNAL(void* logger, int32_t version) noexcept
{
assert(NvDynLoader::pfn_createInferRuntime_INTERNAL &&
"NvDynLoader::Initialize() has not been called before the first TRT use.");
return NvDynLoader::pfn_createInferRuntime_INTERNAL(logger, version);
}
// createInferRefitter_INTERNAL — only needed if engine refitting is used;
// guarded so a missing symbol is a no-op rather than an assert failure.
void* createInferRefitter_INTERNAL(void* engine, void* logger, int32_t version) noexcept
{
if (!NvDynLoader::pfn_createInferRefitter_INTERNAL) return nullptr;
return NvDynLoader::pfn_createInferRefitter_INTERNAL(engine, logger, version);
}
void* createNvOnnxParser_INTERNAL(void* network, void* logger, int32_t version) noexcept
{
assert(NvDynLoader::pfn_createNvOnnxParser_INTERNAL &&
"NvDynLoader::Initialize() has not been called before the first ONNX-parser use.");
return NvDynLoader::pfn_createNvOnnxParser_INTERNAL(network, logger, version);
}
// createNvOnnxParserRefitter_INTERNAL — TRT 10+ symbol; absent on TRT 8/9.
// Guarded rather than asserted so older TRT versions continue to work.
void* createNvOnnxParserRefitter_INTERNAL(void* refitter, void* logger, int32_t version) noexcept
{
if (!NvDynLoader::pfn_createNvOnnxParserRefitter_INTERNAL) return nullptr;
return NvDynLoader::pfn_createNvOnnxParserRefitter_INTERNAL(refitter, logger, version);
}
// getNvOnnxParserVersion — optional version query; returns 0 if not available.
int getNvOnnxParserVersion() noexcept
{
if (!NvDynLoader::pfn_getNvOnnxParserVersion) return 0;
return NvDynLoader::pfn_getNvOnnxParserVersion();
}
} // extern "C"
// ─────────────────────────────────────────────────────────────────────────────
// CUDA stubs (only when NV_DYNAMIC_CUDA is defined)
// Removes the cudart_static.lib dependency; CUDA RT is loaded dynamically.
//
// WHY extern "C":
// cuda_runtime.h wraps every public API in `extern "C"` when compiled as C++,
// giving each function C linkage (no name mangling). Our stub DEFINITIONS
// must use the same linkage — otherwise the linker sees a C-linkage declaration
// (from the header) paired with a C++-mangled definition (from our .cpp), which
// is an unresolved-symbol error on both MSVC and GCC/Clang.
//
// WHY cudaDeviceSynchronize / cudaGetLastError are NOT in the macro list:
// CUDA_STUB_N macros require at least one parameter. These two functions take
// zero arguments; they are defined manually in the block below. Do NOT add
// them back to the macro invocation list — it would create duplicate definitions.
// ─────────────────────────────────────────────────────────────────────────────
#ifdef NV_DYNAMIC_CUDA
extern "C" { // ← must match the extern "C" in cuda_runtime.h
#define CUDA_STUB_1(ret, name, t0, a0) \
ret name(t0 a0) { \
assert(NvDynLoader::pfn_##name); return NvDynLoader::pfn_##name(a0); }
#define CUDA_STUB_2(ret, name, t0,a0, t1,a1) \
ret name(t0 a0, t1 a1) { \
assert(NvDynLoader::pfn_##name); return NvDynLoader::pfn_##name(a0,a1); }
#define CUDA_STUB_3(ret, name, t0,a0, t1,a1, t2,a2) \
ret name(t0 a0, t1 a1, t2 a2) { \
assert(NvDynLoader::pfn_##name); return NvDynLoader::pfn_##name(a0,a1,a2); }
#define CUDA_STUB_4(ret, name, t0,a0, t1,a1, t2,a2, t3,a3) \
ret name(t0 a0, t1 a1, t2 a2, t3 a3) { \
assert(NvDynLoader::pfn_##name); return NvDynLoader::pfn_##name(a0,a1,a2,a3); }
#define CUDA_STUB_5(ret, name, t0,a0, t1,a1, t2,a2, t3,a3, t4,a4) \
ret name(t0 a0, t1 a1, t2 a2, t3 a3, t4 a4) { \
assert(NvDynLoader::pfn_##name); return NvDynLoader::pfn_##name(a0,a1,a2,a3,a4); }
CUDA_STUB_1(cudaError_t, cudaGetDeviceCount, int*, count)
CUDA_STUB_1(cudaError_t, cudaSetDevice, int, dev)
CUDA_STUB_2(cudaError_t, cudaGetDeviceProperties, cudaDeviceProp*, prop, int, dev)
CUDA_STUB_2(cudaError_t, cudaDeviceSetLimit, cudaLimit, limit, size_t, value)
CUDA_STUB_2(cudaError_t, cudaDeviceGetStreamPriorityRange, int*, leastPriority, int*, greatestPriority)
CUDA_STUB_2(cudaError_t, cudaMalloc, void**, devPtr, size_t, size)
CUDA_STUB_1(cudaError_t, cudaFree, void*, devPtr)
CUDA_STUB_3(cudaError_t, cudaMemset, void*, devPtr, int, value, size_t, count)
CUDA_STUB_2(cudaError_t, cudaMemGetInfo, size_t*, free, size_t*, total)
CUDA_STUB_4(cudaError_t, cudaMemcpy, void*, dst, const void*, src, size_t, count, cudaMemcpyKind, kind)
CUDA_STUB_5(cudaError_t, cudaMemcpyAsync, void*, dst, const void*, src, size_t, count, cudaMemcpyKind, kind, cudaStream_t, stream)
CUDA_STUB_1(cudaError_t, cudaStreamCreate, cudaStream_t*, stream)
CUDA_STUB_3(cudaError_t, cudaStreamCreateWithPriority, cudaStream_t*, stream, unsigned int, flags, int, priority)
CUDA_STUB_1(cudaError_t, cudaStreamDestroy, cudaStream_t, stream)
CUDA_STUB_1(cudaError_t, cudaStreamSynchronize, cudaStream_t, stream)
CUDA_STUB_3(cudaError_t, cudaStreamWaitEvent, cudaStream_t, stream, cudaEvent_t, event, unsigned int, flags)
CUDA_STUB_1(cudaError_t, cudaEventCreate, cudaEvent_t*, event)
CUDA_STUB_2(cudaError_t, cudaEventCreateWithFlags, cudaEvent_t*, event, unsigned int, flags)
CUDA_STUB_2(cudaError_t, cudaEventRecord, cudaEvent_t, event, cudaStream_t, stream)
CUDA_STUB_1(cudaError_t, cudaEventDestroy, cudaEvent_t, event)
CUDA_STUB_1(const char*, cudaGetErrorString, cudaError_t, error)
// 0-argument stubs — defined manually; NOT in the macro list above:
cudaError_t cudaDeviceSynchronize() {
assert(NvDynLoader::pfn_cudaDeviceSynchronize);
return NvDynLoader::pfn_cudaDeviceSynchronize();
}
cudaError_t cudaGetLastError() {
assert(NvDynLoader::pfn_cudaGetLastError);
return NvDynLoader::pfn_cudaGetLastError();
}
#undef CUDA_STUB_1
#undef CUDA_STUB_2
#undef CUDA_STUB_3
#undef CUDA_STUB_4
#undef CUDA_STUB_5
} // extern "C"
#endif // NV_DYNAMIC_CUDA
// ─────────────────────────────────────────────────────────────────────────────
// Helper — shared directory path with trailing separator
// ─────────────────────────────────────────────────────────────────────────────
static std::string SharedDirWithSep()
{
std::string dir = DEFAULT_SHARED_DIR;
#ifdef _WIN32
if (!dir.empty() && dir.back() != '\\') dir += '\\';
#else
if (!dir.empty() && dir.back() != '/') dir += '/';
#endif
return dir;
}
// ─────────────────────────────────────────────────────────────────────────────
// DLL candidate lists
// Priority 1 — ANSCENTER shared directory (full absolute path, newest first)
// Priority 2 — System PATH / LD_LIBRARY_PATH fallback (bare name, newest first)
// ─────────────────────────────────────────────────────────────────────────────
std::vector<std::string> NvDynLoader::TrtCandidates()
{
const std::string ansDir = SharedDirWithSep();
std::vector<std::string> v;
// ── Priority 1: ANSCENTER shared directory ────────────────────────────────
for (int major = 20; major >= 8; --major) {
#ifdef _WIN32
v.push_back(ansDir + "nvinfer_" + std::to_string(major) + ".dll");
#else
v.push_back(ansDir + "libnvinfer.so." + std::to_string(major));
#endif
}
// ── Priority 2: System PATH / LD_LIBRARY_PATH ─────────────────────────────
for (int major = 20; major >= 8; --major) {
#ifdef _WIN32
v.push_back("nvinfer_" + std::to_string(major) + ".dll");
#else
v.push_back("libnvinfer.so." + std::to_string(major));
#endif
}
return v;
}
std::vector<std::string> NvDynLoader::OnnxCandidates()
{
const std::string ansDir = SharedDirWithSep();
std::vector<std::string> v;
// ── Priority 1: ANSCENTER shared directory ────────────────────────────────
for (int major = 20; major >= 8; --major) {
#ifdef _WIN32
v.push_back(ansDir + "nvonnxparser_" + std::to_string(major) + ".dll");
#else
v.push_back(ansDir + "libnvonnxparser.so." + std::to_string(major));
#endif
}
// ── Priority 2: System PATH / LD_LIBRARY_PATH ─────────────────────────────
for (int major = 20; major >= 8; --major) {
#ifdef _WIN32
v.push_back("nvonnxparser_" + std::to_string(major) + ".dll");
#else
v.push_back("libnvonnxparser.so." + std::to_string(major));
#endif
}
return v;
}
std::vector<std::string> NvDynLoader::CudnnCandidates()
{
const std::string ansDir = SharedDirWithSep();
std::vector<std::string> v;
// ── cuDNN DLL/SO naming conventions ───────────────────────────────────────
// Windows: cudnn64_<major>.dll (e.g. cudnn64_8.dll, cudnn64_9.dll)
// Linux: libcudnn.so.<major> (e.g. libcudnn.so.8, libcudnn.so.9)
//
// Note: cuDNN 9 on Windows ships as multiple split DLLs
// (cudnn_ops.dll, cudnn_cnn.dll, etc.) but cudnn64_9.dll remains the
// main entry-point / compatibility shim. The split DLLs are resolved
// automatically once the ANSCENTER directory is added to the search path
// via LOAD_WITH_ALTERED_SEARCH_PATH (Windows) or RTLD_GLOBAL (Linux).
//
// ── Priority 1: ANSCENTER shared directory ────────────────────────────────
for (int major = 12; major >= 7; --major) {
#ifdef _WIN32
v.push_back(ansDir + "cudnn64_" + std::to_string(major) + ".dll");
#else
v.push_back(ansDir + "libcudnn.so." + std::to_string(major));
#endif
}
// ── Priority 2: System PATH / LD_LIBRARY_PATH ─────────────────────────────
for (int major = 12; major >= 7; --major) {
#ifdef _WIN32
v.push_back("cudnn64_" + std::to_string(major) + ".dll");
#else
v.push_back("libcudnn.so." + std::to_string(major));
#endif
}
return v;
}
#ifdef NV_DYNAMIC_CUDA
std::vector<std::string> NvDynLoader::CudaRtCandidates()
{
const std::string ansDir = SharedDirWithSep();
std::vector<std::string> v;
// ── CUDA runtime DLL/SO naming conventions ────────────────────────────────
//
// Windows — NVIDIA changed naming at CUDA 12:
// CUDA 11.x : cudart64_110.dll … cudart64_118.dll (major + minor digit)
// CUDA 12+ : cudart64_12.dll, cudart64_13.dll … (major-only)
//
// Linux — major-only symlink is the stable name on all versions:
// libcudart.so.11, libcudart.so.12, libcudart.so.13 …
// (major.minor symlinks also tried as fallback for CUDA 11.x)
//
// ── Priority 1: ANSCENTER shared directory ────────────────────────────────
#ifdef _WIN32
for (int major = 15; major >= 12; --major) // CUDA 12+ : major-only
v.push_back(ansDir + "cudart64_" + std::to_string(major) + ".dll");
for (int minor = 9; minor >= 0; --minor) // CUDA 11.x: major+minor
v.push_back(ansDir + "cudart64_11" + std::to_string(minor) + ".dll");
v.push_back(ansDir + "cudart64.dll"); // non-versioned alias
#else
for (int major = 15; major >= 11; --major) // all versions: major-only
v.push_back(ansDir + "libcudart.so." + std::to_string(major));
for (int minor = 9; minor >= 0; --minor) // CUDA 11.x: major.minor fallback
v.push_back(ansDir + "libcudart.so.11." + std::to_string(minor));
#endif
// ── Priority 2: System PATH / LD_LIBRARY_PATH ─────────────────────────────
#ifdef _WIN32
for (int major = 15; major >= 12; --major)
v.push_back("cudart64_" + std::to_string(major) + ".dll");
for (int minor = 9; minor >= 0; --minor)
v.push_back("cudart64_11" + std::to_string(minor) + ".dll");
v.push_back("cudart64.dll");
#else
for (int major = 15; major >= 11; --major)
v.push_back("libcudart.so." + std::to_string(major));
for (int minor = 9; minor >= 0; --minor)
v.push_back("libcudart.so.11." + std::to_string(minor));
#endif
return v;
}
#endif
// ─────────────────────────────────────────────────────────────────────────────
// Initialize
// ─────────────────────────────────────────────────────────────────────────────
static std::mutex g_initMutex;
bool NvDynLoader::Initialize(bool verbose)
{
std::lock_guard<std::mutex> lock(g_initMutex);
if (s_initialized) return true;
const std::string ansDir = SharedDirWithSep();
// If a previous Initialize() call partially loaded some libraries before
// failing (s_initialized stays false but handles may be non-null), release
// them now so we can retry cleanly without leaking handles.
if (s_hOnnx) { FreeLib(s_hOnnx); s_hOnnx = nullptr; }
if (s_hTrt) { FreeLib(s_hTrt); s_hTrt = nullptr; }
if (s_hCudnn) { FreeLib(s_hCudnn); s_hCudnn = nullptr; }
#ifdef NV_DYNAMIC_CUDA
if (s_hCuda) { FreeLib(s_hCuda); s_hCuda = nullptr; }
#endif
bool ok = true;
if (verbose) {
std::cout << "==================================================" << std::endl;
std::cout << "NvDynLoader: discovering NVIDIA libraries..." << std::endl;
std::cout << " Shared dir : " << ansDir << std::endl;
std::cout << "==================================================" << std::endl;
}
// ── STEP 1: CUDA Runtime (NV_DYNAMIC_CUDA only) ───────────────────────────
// Loaded first so cuDNN and TRT find CUDA symbols already resident in the
// process image when they are opened.
// Linux: globalLoad=true (RTLD_GLOBAL) makes cudart symbols process-wide.
// Windows: LoadLibraryExA is always process-global; globalLoad is a no-op.
#ifdef NV_DYNAMIC_CUDA
s_hCuda = FindAndLoad(CudaRtCandidates(), s_cudaPath, verbose, /*globalLoad=*/true);
if (!s_hCuda) {
std::cout << "NvDynLoader ERROR: No CUDA runtime DLL found. "
"Searched " << ansDir << " and system PATH." << std::endl;
ok = false;
} else {
bool b = true;
b &= Bind(s_hCuda, "cudaGetDeviceCount", pfn_cudaGetDeviceCount);
b &= Bind(s_hCuda, "cudaSetDevice", pfn_cudaSetDevice);
// cudaGetDeviceProperties: CUDA 12 renamed the export to '_v2'; CUDA 11
// uses the plain name. Try the CUDA 12+ symbol first, fall back to
// the CUDA 11 name. IMPORTANT: do NOT `b &=` the first attempt before
// the fallback — that would set b=false on CUDA 11 even though the
// plain-name fallback succeeds, causing a spurious init failure.
Bind(s_hCuda, "cudaGetDeviceProperties_v2", pfn_cudaGetDeviceProperties); // CUDA 12+
if (!pfn_cudaGetDeviceProperties)
Bind(s_hCuda, "cudaGetDeviceProperties", pfn_cudaGetDeviceProperties); // CUDA 11
b &= (pfn_cudaGetDeviceProperties != nullptr);
b &= Bind(s_hCuda, "cudaDeviceSetLimit", pfn_cudaDeviceSetLimit);
b &= Bind(s_hCuda, "cudaDeviceSynchronize", pfn_cudaDeviceSynchronize);
b &= Bind(s_hCuda, "cudaDeviceGetStreamPriorityRange", pfn_cudaDeviceGetStreamPriorityRange);
b &= Bind(s_hCuda, "cudaMalloc", pfn_cudaMalloc);
b &= Bind(s_hCuda, "cudaFree", pfn_cudaFree);
b &= Bind(s_hCuda, "cudaMemset", pfn_cudaMemset);
b &= Bind(s_hCuda, "cudaMemGetInfo", pfn_cudaMemGetInfo);
b &= Bind(s_hCuda, "cudaMemcpy", pfn_cudaMemcpy);
b &= Bind(s_hCuda, "cudaMemcpyAsync", pfn_cudaMemcpyAsync);
b &= Bind(s_hCuda, "cudaStreamCreate", pfn_cudaStreamCreate);
b &= Bind(s_hCuda, "cudaStreamCreateWithPriority", pfn_cudaStreamCreateWithPriority);
b &= Bind(s_hCuda, "cudaStreamDestroy", pfn_cudaStreamDestroy);
b &= Bind(s_hCuda, "cudaStreamSynchronize", pfn_cudaStreamSynchronize);
b &= Bind(s_hCuda, "cudaStreamWaitEvent", pfn_cudaStreamWaitEvent);
b &= Bind(s_hCuda, "cudaEventCreate", pfn_cudaEventCreate);
b &= Bind(s_hCuda, "cudaEventCreateWithFlags", pfn_cudaEventCreateWithFlags);
b &= Bind(s_hCuda, "cudaEventRecord", pfn_cudaEventRecord);
b &= Bind(s_hCuda, "cudaEventDestroy", pfn_cudaEventDestroy);
b &= Bind(s_hCuda, "cudaGetErrorString", pfn_cudaGetErrorString);
b &= Bind(s_hCuda, "cudaGetLastError", pfn_cudaGetLastError);
if (!b) {
std::cout << "NvDynLoader ERROR: One or more CUDA entry points not found in "
<< s_cudaPath << std::endl;
ok = false;
}
}
#endif // NV_DYNAMIC_CUDA
// ── STEP 2: cuDNN pre-load ────────────────────────────────────────────────
// Must be loaded BEFORE TRT so the OS can resolve TRT's cuDNN dependency:
//
// Windows – LoadLibraryExA + LOAD_WITH_ALTERED_SEARCH_PATH causes Windows
// to use the DLL's own directory when resolving its transitive
// dependencies. Pre-loading cuDNN from ANSCENTER dir also puts
// it in the process DLL table so TRT's import loader finds it.
//
// Linux – RTLD_GLOBAL exports cuDNN symbols process-wide so TRT's
// DT_NEEDED resolution succeeds without LD_LIBRARY_PATH changes.
// cuDNN 9 also ships split DLLs (libcudnn_ops.so.9, etc.); those
// are found automatically once libcudnn.so.9 is globally loaded.
//
// Not finding cuDNN is a WARNING, not a fatal error — TRT may still locate
// it via LD_LIBRARY_PATH / system PATH at load time.
s_hCudnn = FindAndLoad(CudnnCandidates(), s_cudnnPath, verbose, /*globalLoad=*/true);
if (!s_hCudnn && verbose) {
std::cout << "NvDynLoader: WARNING — cuDNN not found in " << ansDir
<< "; TRT will search system PATH / LD_LIBRARY_PATH." << std::endl;
}
// ── STEP 3: TensorRT core ─────────────────────────────────────────────────
s_hTrt = FindAndLoad(TrtCandidates(), s_trtPath, verbose);
if (!s_hTrt) {
std::cout << "NvDynLoader ERROR: No TensorRT DLL found. "
"Searched " << ansDir << " and system PATH." << std::endl;
ok = false;
} else {
// Extract major version from the loaded library path.
// Windows : "C:\...\nvinfer_10.dll" → digits between last '_' and last '.'
// Linux : ".../libnvinfer.so.10" → digits after last '.'
#ifdef _WIN32
{
const size_t under = s_trtPath.find_last_of('_');
const size_t dot = s_trtPath.find_last_of('.');
if (under != std::string::npos && dot != std::string::npos && dot > under) {
try { s_trtMajor = std::stoi(s_trtPath.substr(under + 1, dot - under - 1)); }
catch (...) { /* non-numeric — leave s_trtMajor at 0 */ }
}
}
#else
{
const size_t dot = s_trtPath.find_last_of('.');
if (dot != std::string::npos && dot + 1 < s_trtPath.size()) {
try { s_trtMajor = std::stoi(s_trtPath.substr(dot + 1)); }
catch (...) { /* non-numeric — leave s_trtMajor at 0 */ }
}
}
#endif
bool b = true;
b &= Bind(s_hTrt, "createInferBuilder_INTERNAL", pfn_createInferBuilder_INTERNAL);
b &= Bind(s_hTrt, "createInferRuntime_INTERNAL", pfn_createInferRuntime_INTERNAL);
Bind(s_hTrt, "createInferRefitter_INTERNAL", pfn_createInferRefitter_INTERNAL); // optional
if (!b) {
std::cout << "NvDynLoader ERROR: Required TRT entry points not found in "
<< s_trtPath << std::endl;
ok = false;
} else if (verbose) {
std::cout << "NvDynLoader: TRT major version = " << s_trtMajor << std::endl;
}
}
// ── STEP 4: ONNX parser ───────────────────────────────────────────────────
s_hOnnx = FindAndLoad(OnnxCandidates(), s_onnxPath, verbose);
if (!s_hOnnx) {
std::cout << "NvDynLoader ERROR: No nvonnxparser DLL found. "
"Searched " << ansDir << " and system PATH." << std::endl;
ok = false;
} else {
if (!Bind(s_hOnnx, "createNvOnnxParser_INTERNAL", pfn_createNvOnnxParser_INTERNAL)) {
std::cout << "NvDynLoader ERROR: createNvOnnxParser_INTERNAL not found in "
<< s_onnxPath << std::endl;
ok = false;
} else {
// TRT 10+ optional symbols — absent on TRT 8/9; missing is not an error.
Bind(s_hOnnx, "createNvOnnxParserRefitter_INTERNAL", pfn_createNvOnnxParserRefitter_INTERNAL);
Bind(s_hOnnx, "getNvOnnxParserVersion", pfn_getNvOnnxParserVersion);
// Cross-check: TRT and ONNX-parser must share the same major version.
// Mismatched majors (e.g. nvinfer_10 + nvonnxparser_11) compile fine
// but crash at runtime when the ONNX parser tries to populate a TRT
// network object built by a different major version.
int onnxMajor = 0;
#ifdef _WIN32
{
const size_t under = s_onnxPath.find_last_of('_');
const size_t dot = s_onnxPath.find_last_of('.');
if (under != std::string::npos && dot != std::string::npos && dot > under)
try { onnxMajor = std::stoi(s_onnxPath.substr(under + 1, dot - under - 1)); }
catch (...) {}
}
#else
{
const size_t dot = s_onnxPath.find_last_of('.');
if (dot != std::string::npos && dot + 1 < s_onnxPath.size())
try { onnxMajor = std::stoi(s_onnxPath.substr(dot + 1)); }
catch (...) {}
}
#endif
if (onnxMajor > 0 && s_trtMajor > 0 && onnxMajor != s_trtMajor) {
std::cout << "NvDynLoader ERROR: TRT major version (" << s_trtMajor
<< ") != ONNX-parser major version (" << onnxMajor << ").\n"
<< " TRT : " << s_trtPath << "\n"
<< " ONNX : " << s_onnxPath << "\n"
<< " Replace both DLLs with matching versions in "
<< ansDir << std::endl;
ok = false;
} else if (verbose && onnxMajor > 0) {
std::cout << "NvDynLoader: ONNX-parser major version = " << onnxMajor << std::endl;
}
}
}
s_initialized = ok;
if (verbose) {
std::cout << "NvDynLoader: "
<< (ok ? "initialised successfully." : "FAILED — check errors above.")
<< std::endl;
std::cout << "==================================================" << std::endl;
}
return ok;
}
// ─────────────────────────────────────────────────────────────────────────────
// Shutdown
// ─────────────────────────────────────────────────────────────────────────────
void NvDynLoader::Shutdown()
{
std::lock_guard<std::mutex> lock(g_initMutex);
// NOTE: No early-return guard on s_initialized here.
//
// Shutdown() must always release whatever handles are currently open,
// including handles left behind by a *partially*-successful Initialize()
// call (where s_initialized == false but some DLLs were already opened).
// Every FreeLib call is guarded by if(h), so this function is safe to call
// on a fully- or partially-initialised instance, and also safe to call
// multiple times (idempotent).
// ── Null all entry-point pointers first ───────────────────────────────────
// Prevents stubs from jumping into freed library memory in the window
// between Shutdown() and a subsequent Initialize() call.
pfn_createInferBuilder_INTERNAL = nullptr;
pfn_createInferRuntime_INTERNAL = nullptr;
pfn_createInferRefitter_INTERNAL = nullptr;
pfn_createNvOnnxParser_INTERNAL = nullptr;
pfn_createNvOnnxParserRefitter_INTERNAL = nullptr;
pfn_getNvOnnxParserVersion = nullptr;
#ifdef NV_DYNAMIC_CUDA
// Null every CUDA function pointer so the assert() in each stub fires
// immediately if a CUDA call is made after Shutdown() without a new
// Initialize(), rather than silently calling into freed DLL memory.
pfn_cudaGetDeviceCount = nullptr;
pfn_cudaSetDevice = nullptr;
pfn_cudaGetDeviceProperties = nullptr;
pfn_cudaDeviceSetLimit = nullptr;
pfn_cudaDeviceSynchronize = nullptr;
pfn_cudaDeviceGetStreamPriorityRange = nullptr;
pfn_cudaMalloc = nullptr;
pfn_cudaFree = nullptr;
pfn_cudaMemset = nullptr;
pfn_cudaMemGetInfo = nullptr;
pfn_cudaMemcpy = nullptr;
pfn_cudaMemcpyAsync = nullptr;
pfn_cudaStreamCreate = nullptr;
pfn_cudaStreamCreateWithPriority = nullptr;
pfn_cudaStreamDestroy = nullptr;
pfn_cudaStreamSynchronize = nullptr;
pfn_cudaStreamWaitEvent = nullptr;
pfn_cudaEventCreate = nullptr;
pfn_cudaEventCreateWithFlags = nullptr;
pfn_cudaEventRecord = nullptr;
pfn_cudaEventDestroy = nullptr;
pfn_cudaGetErrorString = nullptr;
pfn_cudaGetLastError = nullptr;
#endif // NV_DYNAMIC_CUDA
// ── Release handles in reverse load order ─────────────────────────────────
// ONNX parser → depends on TRT
// TRT → depends on cuDNN + CUDA RT
// cuDNN → depends on CUDA RT
// CUDA RT → base dependency
if (s_hOnnx) { FreeLib(s_hOnnx); s_hOnnx = nullptr; }
if (s_hTrt) { FreeLib(s_hTrt); s_hTrt = nullptr; }
if (s_hCudnn) { FreeLib(s_hCudnn); s_hCudnn = nullptr; }
#ifdef NV_DYNAMIC_CUDA
if (s_hCuda) { FreeLib(s_hCuda); s_hCuda = nullptr; }
#endif
// ── Reset version / path state ────────────────────────────────────────────
// A subsequent Initialize() call will re-discover fresh paths and versions.
s_trtMajor = 0;
s_trtPath.clear();
s_onnxPath.clear();
s_cudaPath.clear();
s_cudnnPath.clear();
s_initialized = false;
}

252
core/ANSLibsLoader/OvLoader.cpp Normal file
View File

@@ -0,0 +1,252 @@
// ============================================================================
// OvLoader.cpp — OpenVINO DLL discovery and pre-loading
//
// Loading order (dependency chain):
// 1. Intel TBB (tbb12.dll / tbbmalloc.dll) — required by openvino.dll
// 2. openvino.dll — core runtime
// 3. openvino_c.dll — C API (optional, version query)
//
// Plugins (openvino_intel_cpu_plugin, openvino_intel_gpu_plugin, etc.) and
// frontends (openvino_ir_frontend, openvino_onnx_frontend, etc.) are loaded
// on-demand by OpenVINO's internal plugin system. They are discovered
// automatically because the shared directory is injected into the DLL
// search path by ANSLibsLoader before OvLoader::Initialize() runs.
// ============================================================================
#include "OvLoader.h"
#include "DynLibUtils.h"
#include <iostream>
#include <mutex>
using namespace ANSCENTER::DynLib;
// ─────────────────────────────────────────────────────────────────────────────
// Static member definitions
// ─────────────────────────────────────────────────────────────────────────────
std::mutex OvLoader::s_mutex;
bool OvLoader::s_initialized = false;
OvInfo OvLoader::s_info;
LibHandle OvLoader::s_hTbb = nullptr;
LibHandle OvLoader::s_hTbbMalloc = nullptr;
LibHandle OvLoader::s_hOvCore = nullptr;
LibHandle OvLoader::s_hOvC = nullptr;
// ─────────────────────────────────────────────────────────────────────────────
// Candidate lists — search ANSCENTER shared dir first, then system PATH
// ─────────────────────────────────────────────────────────────────────────────
std::vector<std::string> OvLoader::TbbCandidates(const std::string& shared_dir)
{
std::vector<std::string> v;
// ── Priority 1: ANSCENTER shared directory ────────────────────────────────
#ifdef _WIN32
v.push_back(JoinPath(shared_dir, "tbb12.dll"));
v.push_back(JoinPath(shared_dir, "tbb.dll"));
#else
v.push_back(JoinPath(shared_dir, "libtbb.so.12"));
v.push_back(JoinPath(shared_dir, "libtbb.so.2"));
v.push_back(JoinPath(shared_dir, "libtbb.so"));
#endif
// ── Priority 2: System PATH / LD_LIBRARY_PATH ─────────────────────────────
#ifdef _WIN32
v.push_back("tbb12.dll");
v.push_back("tbb.dll");
#else
v.push_back("libtbb.so.12");
v.push_back("libtbb.so.2");
v.push_back("libtbb.so");
#endif
return v;
}
std::vector<std::string> OvLoader::TbbMallocCandidates(const std::string& shared_dir)
{
std::vector<std::string> v;
#ifdef _WIN32
v.push_back(JoinPath(shared_dir, "tbbmalloc.dll"));
v.push_back("tbbmalloc.dll");
#else
v.push_back(JoinPath(shared_dir, "libtbbmalloc.so.2"));
v.push_back(JoinPath(shared_dir, "libtbbmalloc.so"));
v.push_back("libtbbmalloc.so.2");
v.push_back("libtbbmalloc.so");
#endif
return v;
}
std::vector<std::string> OvLoader::OvCoreCandidates(const std::string& shared_dir)
{
std::vector<std::string> v;
#ifdef _WIN32
v.push_back(JoinPath(shared_dir, "openvino.dll"));
v.push_back("openvino.dll");
#else
v.push_back(JoinPath(shared_dir, "libopenvino.so"));
v.push_back("libopenvino.so");
#endif
return v;
}
std::vector<std::string> OvLoader::OvCApiCandidates(const std::string& shared_dir)
{
std::vector<std::string> v;
#ifdef _WIN32
v.push_back(JoinPath(shared_dir, "openvino_c.dll"));
v.push_back("openvino_c.dll");
#else
v.push_back(JoinPath(shared_dir, "libopenvino_c.so"));
v.push_back("libopenvino_c.so");
#endif
return v;
}
// ─────────────────────────────────────────────────────────────────────────────
// Version extraction via OpenVINO C API
//
// The C API exports ov_get_openvino_version() which fills an ov_version_t
// struct with a buildNumber string (e.g. "2024.6.0-17404-4c0f47d2335").
// We replicate the struct layout locally to avoid including OpenVINO headers.
// ─────────────────────────────────────────────────────────────────────────────
std::string OvLoader::TryGetVersion(LibHandle hOvC)
{
if (!hOvC) return {};
// Replicate the ov_version_t layout from openvino/c/ov_common.h
struct ov_version_local {
const char* buildNumber;
const char* description;
};
using fn_get_version = int (*)(ov_version_local*);
using fn_free_version = void (*)(ov_version_local*);
fn_get_version pfnGet = nullptr;
fn_free_version pfnFree = nullptr;
Bind(hOvC, "ov_get_openvino_version", pfnGet);
Bind(hOvC, "ov_version_free", pfnFree);
if (!pfnGet) return {};
ov_version_local ver = {};
if (pfnGet(&ver) == 0 && ver.buildNumber) {
std::string result = ver.buildNumber;
if (pfnFree) pfnFree(&ver);
return result;
}
return {};
}
// ─────────────────────────────────────────────────────────────────────────────
// Initialize
// ─────────────────────────────────────────────────────────────────────────────
const OvInfo& OvLoader::Initialize(const std::string& shared_dir, bool verbose)
{
std::lock_guard<std::mutex> lock(s_mutex);
if (s_initialized) return s_info;
// Clean up any partially-loaded handles from a previous failed attempt
if (s_hOvC) { FreeLib(s_hOvC); s_hOvC = nullptr; }
if (s_hOvCore) { FreeLib(s_hOvCore); s_hOvCore = nullptr; }
if (s_hTbbMalloc) { FreeLib(s_hTbbMalloc); s_hTbbMalloc = nullptr; }
if (s_hTbb) { FreeLib(s_hTbb); s_hTbb = nullptr; }
if (verbose) {
std::cout << "==================================================" << std::endl;
std::cout << "[OvLoader] Discovering OpenVINO libraries..." << std::endl;
std::cout << " Shared dir : " << shared_dir << std::endl;
std::cout << "==================================================" << std::endl;
}
// ── STEP 1: Inject shared dir into search path ───────────────────────────
// Ensures plugins/frontends are discoverable by OpenVINO's internal loader.
InjectDllSearchPath(shared_dir);
// ── STEP 2: Pre-load Intel TBB (dependency of openvino.dll) ──────────────
std::string tbbPath;
s_hTbb = FindAndLoad(TbbCandidates(shared_dir), tbbPath, verbose);
if (s_hTbb) {
if (verbose)
std::cout << "[OvLoader] TBB loaded: " << tbbPath << std::endl;
} else if (verbose) {
std::cout << "[OvLoader] WARNING: TBB not found "
<< "(openvino.dll may fail to load)." << std::endl;
}
std::string tbbMallocPath;
s_hTbbMalloc = FindAndLoad(TbbMallocCandidates(shared_dir),
tbbMallocPath, verbose);
if (!s_hTbbMalloc && verbose) {
std::cout << "[OvLoader] tbbmalloc not found (optional)." << std::endl;
}
// ── STEP 3: Pre-load openvino.dll (core runtime) ─────────────────────────
std::string corePath;
s_hOvCore = FindAndLoad(OvCoreCandidates(shared_dir), corePath, verbose);
if (s_hOvCore) {
s_info.dllPath = corePath;
s_info.loaded = true;
if (verbose)
std::cout << "[OvLoader] OpenVINO core loaded: " << corePath
<< std::endl;
} else {
std::cout << "[OvLoader] WARNING: openvino.dll not found in "
<< shared_dir << " or system PATH." << std::endl;
std::cout << "[OvLoader] OpenVINO will rely on import-lib resolution "
<< "at load time." << std::endl;
}
// ── STEP 4: Pre-load openvino_c.dll + query version ──────────────────────
std::string cApiPath;
s_hOvC = FindAndLoad(OvCApiCandidates(shared_dir), cApiPath, verbose);
if (s_hOvC) {
s_info.version = TryGetVersion(s_hOvC);
if (!s_info.version.empty() && verbose) {
std::cout << "[OvLoader] OpenVINO version = " << s_info.version
<< std::endl;
}
} else if (verbose) {
std::cout << "[OvLoader] openvino_c.dll not found "
<< "(version detection skipped)." << std::endl;
}
s_initialized = true;
if (verbose) {
std::cout << "[OvLoader] "
<< (s_info.loaded
? "initialised successfully."
: "WARNING — openvino.dll not pre-loaded.")
<< std::endl;
std::cout << "==================================================" << std::endl;
}
return s_info;
}
// ─────────────────────────────────────────────────────────────────────────────
// Shutdown
// ─────────────────────────────────────────────────────────────────────────────
void OvLoader::Shutdown()
{
std::lock_guard<std::mutex> lock(s_mutex);
// Release in reverse load order
if (s_hOvC) { FreeLib(s_hOvC); s_hOvC = nullptr; }
if (s_hOvCore) { FreeLib(s_hOvCore); s_hOvCore = nullptr; }
if (s_hTbbMalloc) { FreeLib(s_hTbbMalloc); s_hTbbMalloc = nullptr; }
if (s_hTbb) { FreeLib(s_hTbb); s_hTbb = nullptr; }
s_info = OvInfo{};
s_initialized = false;
}

23
core/ANSLibsLoader/dllmain.cpp Normal file
View File

@@ -0,0 +1,23 @@
// dllmain.cpp : Defines the entry point for the DLL application.
#ifndef WIN32_LEAN_AND_MEAN
#define WIN32_LEAN_AND_MEAN
#endif
#define NOMINMAX // Prevent windows.h from defining min/max macros
#include <windows.h>
BOOL APIENTRY DllMain( HMODULE hModule,
DWORD ul_reason_for_call,
LPVOID lpReserved
)
{
switch (ul_reason_for_call)
{
case DLL_PROCESS_ATTACH:
case DLL_THREAD_ATTACH:
case DLL_THREAD_DETACH:
case DLL_PROCESS_DETACH:
break;
}
return TRUE;
}

5
core/ANSLibsLoader/framework.h Normal file
View File

@@ -0,0 +1,5 @@
#pragma once
#define WIN32_LEAN_AND_MEAN // Exclude rarely-used stuff from Windows headers
// Windows Header Files
#include <windows.h>

89
core/ANSLibsLoader/include/ANSLibsLoader.h Normal file
View File

@@ -0,0 +1,89 @@
#pragma once
#ifndef ANSLIBSLOADER_H
#define ANSLIBSLOADER_H
// ============================================================================
// ANSLibsLoader — Centralized dynamic library loader for ANSCENTER
//
// Consolidates all runtime DLL loading:
// - ONNX Runtime (via EPLoader)
// - TensorRT / cuDNN / CUDA (via NvDynLoader)
// - OpenCV (via CvLoader)
// - OpenVINO + TBB (via OvLoader)
//
// USAGE
// -----
// // At application startup:
// ANSCENTER::ANSLibsLoader::Initialize();
//
// // At application exit (after all engine objects are destroyed):
// ANSCENTER::ANSLibsLoader::Shutdown();
//
// EXPORT MACRO
// ------------
// ANSLIBS_API replaces both ANSCORE_API and ENGINE_API for loader classes.
// Define ANSLIBSLOADER_EXPORTS when building ANSLibsLoader.dll.
// All other projects importing from ANSLibsLoader get __declspec(dllimport).
// ============================================================================
#ifdef _WIN32
# ifdef ANSLIBSLOADER_EXPORTS
# define ANSLIBS_API __declspec(dllexport)
# else
# define ANSLIBS_API __declspec(dllimport)
# endif
#else
# define ANSLIBS_API
#endif
#include "ANSLicense.h" // for ANSCENTER::EngineType enum
#include <string>
namespace ANSCENTER {
class ANSLIBS_API ANSLibsLoader
{
public:
// ── Platform defaults ────────────────────────────────────────────────
#ifdef _WIN32
static constexpr const char* DEFAULT_SHARED_DIR =
"C:\\ProgramData\\ANSCENTER\\Shared";
#else
static constexpr const char* DEFAULT_SHARED_DIR =
"/opt/anscenter/shared";
#endif
// ── Master initialization ────────────────────────────────────────────
/// Calls all sub-loaders in the correct order:
/// 1. Injects shared_dir into DLL search path
/// 2. CvLoader — pre-loads OpenCV
/// 3. OvLoader — pre-loads OpenVINO + TBB
/// 4. NvDynLoader — loads TRT / cuDNN / CUDA (if NVIDIA)
/// 5. EPLoader — loads ONNX Runtime with selected EP
///
/// Thread-safe. Subsequent calls are no-ops.
static void Initialize(
const std::string& shared_dir = DEFAULT_SHARED_DIR,
EngineType preferred = EngineType::AUTO_DETECT);
// ── Master shutdown ──────────────────────────────────────────────────
/// Shuts down all sub-loaders in reverse order.
/// Must be called AFTER all engine/session objects are destroyed.
/// Safe to call multiple times.
static void Shutdown();
// ── Status ───────────────────────────────────────────────────────────
[[nodiscard]] static bool IsInitialized();
// ── Non-copyable / non-movable ───────────────────────────────────────
ANSLibsLoader() = delete;
ANSLibsLoader(const ANSLibsLoader&) = delete;
ANSLibsLoader& operator=(const ANSLibsLoader&) = delete;
private:
static bool s_initialized;
};
} // namespace ANSCENTER
#endif // ANSLIBSLOADER_H

62
core/ANSLibsLoader/include/CvLoader.h Normal file
View File

@@ -0,0 +1,62 @@
#pragma once
// ============================================================================
// CvLoader -- Runtime discovery and pre-loading of OpenCV DLLs
//
// OpenCV is normally import-linked (opencv_world4130.lib), but the DLLs
// must be discoverable at process load time. CvLoader ensures the correct
// DLL is found by:
// 1. Searching the ANSCENTER shared directory for versioned candidates
// 2. Injecting the directory into the DLL search path
// 3. Pre-loading the DLL so dependent modules resolve it correctly
//
// Companion modules (opencv_img_hash, opencv_videoio_ffmpeg) are also
// pre-loaded when found alongside the main opencv_world DLL.
// ============================================================================
#include "ANSLibsLoader.h" // ANSLIBS_API
#include "DynLibUtils.h" // LibHandle
#include <string>
#include <vector>
#include <mutex>
// ============================================================================
struct ANSLIBS_API CvInfo
{
std::string dllPath; // Full path to loaded opencv_world DLL
std::string version; // e.g. "4.13.0"
int versionCode = 0;// e.g. 4130
bool loaded = false;
};
// ============================================================================
class ANSLIBS_API CvLoader
{
public:
/// Discover and pre-load OpenCV DLLs.
/// Safe to call multiple times -- subsequent calls are no-ops.
/// @param shared_dir Directory to search first (default: ANSCENTER shared).
/// @param verbose Print discovery results to stdout.
/// @returns Reference to the discovery result.
[[nodiscard]] static const CvInfo& Initialize(
const std::string& shared_dir = ANSCENTER::DynLib::DEFAULT_SHARED_DIR,
bool verbose = true);
/// Release pre-loaded library handles.
static void Shutdown();
/// Query current state.
[[nodiscard]] static const CvInfo& Current() noexcept { return s_info; }
[[nodiscard]] static bool IsInitialized() noexcept { return s_initialized; }
private:
/// Build candidate DLL names, newest first.
static std::vector<std::string> CvWorldCandidates(const std::string& shared_dir);
static std::vector<std::string> CvImgHashCandidates(const std::string& shared_dir);
static std::mutex s_mutex;
static bool s_initialized;
static CvInfo s_info;
static LibHandle s_hCvWorld;
static LibHandle s_hCvImgHash;
};

95
core/ANSLibsLoader/include/DynLibUtils.h Normal file
View File

@@ -0,0 +1,95 @@
#pragma once
#ifndef DYNLIBUTILS_H
#define DYNLIBUTILS_H
// ============================================================================
// DynLibUtils — Shared dynamic library loading utilities
//
// Common infrastructure used by EPLoader, NvDynLoader, CvLoader, OvLoader, etc.
// Consolidates duplicate utility functions that were previously in each loader.
// ============================================================================
#ifdef _WIN32
# ifndef WIN32_LEAN_AND_MEAN
# define WIN32_LEAN_AND_MEAN
# endif
# ifndef NOMINMAX
# define NOMINMAX
# endif
# include <windows.h>
using LibHandle = HMODULE;
# define LIB_INVALID nullptr
#else
# include <dlfcn.h>
using LibHandle = void*;
# define LIB_INVALID nullptr
#endif
#include <string>
#include <vector>
#include <iostream>
namespace ANSCENTER {
namespace DynLib {
// ── Platform defaults ────────────────────────────────────────────────────
#ifdef _WIN32
static constexpr const char* DEFAULT_SHARED_DIR =
"C:\\ProgramData\\ANSCENTER\\Shared";
#else
static constexpr const char* DEFAULT_SHARED_DIR =
"/opt/anscenter/shared";
#endif
// ── File / path helpers ──────────────────────────────────────────────────
/// Returns true if @p path is a regular file.
[[nodiscard]] bool FileExists(const std::string& path);
/// Joins base + component with the platform separator.
[[nodiscard]] std::string JoinPath(const std::string& base,
const std::string& component);
// ── DLL search-path injection ────────────────────────────────────────────
/// Adds @p dir to the DLL search path.
/// - Windows: AddDllDirectory() + prepend to %PATH%
/// - Linux: prepend to $LD_LIBRARY_PATH
void InjectDllSearchPath(const std::string& dir);
// ── Library loading ──────────────────────────────────────────────────────
/// Iterates @p candidates, returns the first successfully loaded library.
/// @param loadedPath On success, receives the path/name that was loaded.
/// @param verbose Print discovery results to stdout.
/// @param globalLoad Linux only: use RTLD_GLOBAL (ignored on Windows).
[[nodiscard]] LibHandle FindAndLoad(const std::vector<std::string>& candidates,
std::string& loadedPath,
bool verbose = true,
bool globalLoad = false);
/// Load a single library by path.
/// @param globalLoad Linux only: use RTLD_GLOBAL.
[[nodiscard]] LibHandle LoadLib(const std::string& path, bool globalLoad = false);
/// Free a loaded library handle.
void FreeLib(LibHandle h);
/// Get a symbol address from a loaded library.
[[nodiscard]] void* GetSymbol(LibHandle h, const char* sym);
// ── Type-safe symbol binding ─────────────────────────────────────────────
/// Bind a function pointer from a loaded library handle.
/// @returns true if the symbol was found and bound.
template<typename Fn>
[[nodiscard]] bool Bind(LibHandle h, const char* sym, Fn*& pfn)
{
pfn = reinterpret_cast<Fn*>(GetSymbol(h, sym));
return pfn != nullptr;
}
} // namespace DynLib
} // namespace ANSCENTER
#endif // DYNLIBUTILS_H

116
core/ANSLibsLoader/include/EPLoader.h Normal file
View File

@@ -0,0 +1,116 @@
#pragma once
#ifndef EPLOADER_H
#define EPLOADER_H
// ============================================================================
// EPLoader — Dynamic ONNX Runtime loader
//
// Moved from ONNXEngine/ to ANSLibsLoader/ for centralized library management.
// Now exported via ANSLIBS_API from ANSLibsLoader.dll.
// ============================================================================
#include "ANSLibsLoader.h" // ANSLIBS_API, EngineType
#include <string>
#include <mutex>
namespace ANSCENTER {
struct EPInfo {
EngineType type = EngineType::CPU;
std::string name; // "CUDA", "OpenVINO", "DirectML", "CPU"
std::string ortProviderName; // exact ORT provider string
std::string libraryDir; // resolved absolute path to EP DLLs
bool fromSubdir = false; // true = ep/<name>/, false = flat Shared/
};
class ANSLIBS_API EPLoader
{
public:
// ── Platform defaults ────────────────────────────────────────────────
#ifdef _WIN32
static constexpr const char* DEFAULT_SHARED_DIR =
"C:\\ProgramData\\ANSCENTER\\Shared";
#else
static constexpr const char* DEFAULT_SHARED_DIR =
"/opt/anscenter/shared";
#endif
// ── Initialize ───────────────────────────────────────────────────────
/// Call once at application startup (optional — Current() lazy-inits).
///
/// @param shared_dir Root deployment directory.
/// Expected subdirectory layout:
/// <shared_dir>/ep/cuda/onnxruntime.dll
/// <shared_dir>/ep/directml/onnxruntime.dll
/// <shared_dir>/ep/openvino/onnxruntime.dll
/// <shared_dir>/ep/cpu/onnxruntime.dll
/// Falls back to <shared_dir>/onnxruntime.dll if
/// subdirectories are absent (backward compat).
///
/// @param preferred Force a specific EP.
/// Pass EngineType::AUTO_DETECT to let AutoDetect()
/// query ANSLicenseHelper (default).
///
/// @return Const reference to the populated EPInfo.
/// Valid for the lifetime of the process.
///
/// @throws std::runtime_error if onnxruntime.dll cannot be loaded or
/// OrtGetApiBase is not found in the DLL.
[[nodiscard]] static const EPInfo& Initialize(
const std::string& shared_dir = DEFAULT_SHARED_DIR,
EngineType preferred = EngineType::AUTO_DETECT);
// ── Current ──────────────────────────────────────────────────────────
/// Returns the active EPInfo. Calls Initialize() with default arguments
/// on the first invocation (lazy auto-init). Thread-safe.
[[nodiscard]] static const EPInfo& Current();
// ── AutoDetect ───────────────────────────────────────────────────────
/// Delegates to ANSLicenseHelper::CheckHardwareInformation().
/// Priority: NVIDIA > AMD > Intel (OpenVINO) > CPU.
[[nodiscard]] static EngineType AutoDetect();
// ── IsInitialized ────────────────────────────────────────────────────
[[nodiscard]] static bool IsInitialized();
// ── EngineTypeName ───────────────────────────────────────────────────
[[nodiscard]] static const char* EngineTypeName(EngineType type);
// ── SubdirName ───────────────────────────────────────────────────────
/// Returns the ep/ subdirectory name for a given EngineType.
/// e.g. EngineType::NVIDIA_GPU → "cuda"
[[nodiscard]] static const char* SubdirName(EngineType type);
[[nodiscard]] static void* GetOrtApiRaw();
// ── Shutdown ─────────────────────────────────────────────────────────
/// Releases the onnxruntime.dll handle.
/// Call at application exit AFTER all Ort::Session objects are destroyed.
/// Safe to call multiple times.
static void Shutdown();
// ── Non-copyable / non-movable ───────────────────────────────────────
EPLoader() = delete;
EPLoader(const EPLoader&) = delete;
EPLoader& operator=(const EPLoader&) = delete;
private:
static std::string ResolveEPDir(const std::string& shared_dir,
EngineType type);
static void LoadOrtDll(const std::string& ep_dir);
// ── State ────────────────────────────────────────────────────────────
// Defined in EPLoader.cpp only when ANSLIBSLOADER_EXPORTS is set.
static std::mutex s_mutex;
static bool s_initialized;
static EPInfo s_info;
#ifdef _WIN32
static std::string s_temp_ort_path;
static std::string s_temp_dir;
#endif
};
} // namespace ANSCENTER
#endif // EPLOADER_H

142
core/ANSLibsLoader/include/NvDynLoader.h Normal file
View File

@@ -0,0 +1,142 @@
#pragma once
// ============================================================================
// NvDynLoader -- Runtime discovery and loading of NVIDIA TensorRT / CUDA DLLs
//
// Moved from TensorRTAPI/ to ANSLibsLoader/ for centralized library management.
// Now exported via ANSLIBS_API from ANSLibsLoader.dll.
//
// PROBLEM SOLVED
// --------------
// Linking against nvinfer_10.lib / nvonnxparser_10.lib hard-codes the major
// version into the binary's import table. Upgrading TRT 10 -> 11 then forces
// every project to update its .lib references and relink.
//
// SOLUTION
// --------
// NvDynLoader provides the three extern "C" symbols that TRT / ONNX-parser
// inline wrappers call (createInferBuilder_INTERNAL, etc.) as thin stubs
// compiled directly into ANSLibsLoader.dll. At runtime the stubs call through
// function pointers into whichever DLL version is actually installed.
//
// All C++ vtable dispatch (IBuilder, IRuntime, IParser methods) continues to
// work correctly because the objects are created by -- and owned by the vtable
// of -- the DLL that was dynamically loaded.
//
// REQUIRED PROJECT CHANGES
// ------------------------
// Consuming projects:
// REMOVE: nvinfer_10.lib / nvonnxparser_10.lib from linker input
// ADD: ANSLibsLoader.lib
// KEEP: cudart_static.lib (or use NV_DYNAMIC_CUDA for dynamic CUDA RT)
// ============================================================================
#include "ANSLibsLoader.h" // ANSLIBS_API
#include "DynLibUtils.h" // LibHandle
// -- TRT / ONNX-parser API decoration override --------------------------------
// Must appear BEFORE including NvInfer.h / NvOnnxParser.h.
// By default these macros expand to __declspec(dllimport), which would conflict
// with our extern "C" stub definitions in NvDynLoader.cpp. Setting them to
// empty makes all TRT inline-wrapper calls direct, so the linker resolves them
// against our stubs rather than against nvinfer_XX.lib.
//
// The stubs are exported from ANSLibsLoader.dll via the .def file
// (ANSLibsLoader.def), NOT via __declspec(dllexport), to avoid C2375
// linkage conflicts between the NvInfer.h declarations and our definitions.
#ifndef TENSORRTAPI
# define TENSORRTAPI
#endif
#ifndef NVONNXPARSER_API
# define NVONNXPARSER_API
#endif
#include <cuda_runtime.h> // CUDA types (cudaStream_t, cudaDeviceProp, ...)
#include <NvInfer.h> // TRT types (IBuilder, IRuntime, ...)
#include <NvOnnxParser.h> // nvonnxparser types
#include <string>
#include <vector>
// ============================================================================
class ANSLIBS_API NvDynLoader
{
public:
// -- Lifecycle -------------------------------------------------------------
/// Discover and load NVIDIA DLLs at runtime.
/// Safe to call multiple times -- subsequent calls are no-ops.
/// @param verbose Print discovery results to stdout.
/// @returns false if a critical library (TRT or CUDA) could not be loaded.
[[nodiscard]] static bool Initialize(bool verbose = true);
/// Release all loaded library handles. Call at application exit.
static void Shutdown();
[[nodiscard]] static bool IsInitialized() noexcept { return s_initialized; }
// -- Informational ---------------------------------------------------------
[[nodiscard]] static const std::string& TrtDllPath() noexcept { return s_trtPath; }
[[nodiscard]] static const std::string& OnnxDllPath() noexcept { return s_onnxPath; }
[[nodiscard]] static const std::string& CudaDllPath() noexcept { return s_cudaPath; }
[[nodiscard]] static const std::string& CudnnDllPath()noexcept { return s_cudnnPath; }
[[nodiscard]] static int TrtMajor() noexcept { return s_trtMajor; }
// -- TRT factory pointers ---------------------------------------------------
using PfnBuilder = void*(void* logger, int32_t version) noexcept;
using PfnRuntime = void*(void* logger, int32_t version) noexcept;
using PfnRefitter = void*(void* engine, void* logger, int32_t version) noexcept;
using PfnParser = void*(void* network, void* logger, int32_t version) noexcept;
using PfnParserRefitter = void*(void* refitter, void* logger, int32_t version) noexcept;
using PfnGetParserVersion = int() noexcept;
static PfnBuilder* pfn_createInferBuilder_INTERNAL;
static PfnRuntime* pfn_createInferRuntime_INTERNAL;
static PfnRefitter* pfn_createInferRefitter_INTERNAL;
static PfnParser* pfn_createNvOnnxParser_INTERNAL;
static PfnParserRefitter* pfn_createNvOnnxParserRefitter_INTERNAL;
static PfnGetParserVersion* pfn_getNvOnnxParserVersion;
// -- CUDA function pointers (populated only with NV_DYNAMIC_CUDA) -----------
#ifdef NV_DYNAMIC_CUDA
static cudaError_t (*pfn_cudaGetDeviceCount) (int*);
static cudaError_t (*pfn_cudaSetDevice) (int);
static cudaError_t (*pfn_cudaGetDeviceProperties) (cudaDeviceProp*, int);
static cudaError_t (*pfn_cudaDeviceSetLimit) (cudaLimit, size_t);
static cudaError_t (*pfn_cudaDeviceSynchronize) ();
static cudaError_t (*pfn_cudaDeviceGetStreamPriorityRange)(int*, int*);
static cudaError_t (*pfn_cudaMalloc) (void**, size_t);
static cudaError_t (*pfn_cudaFree) (void*);
static cudaError_t (*pfn_cudaMemset) (void*, int, size_t);
static cudaError_t (*pfn_cudaMemGetInfo) (size_t*, size_t*);
static cudaError_t (*pfn_cudaMemcpy) (void*, const void*, size_t, cudaMemcpyKind);
static cudaError_t (*pfn_cudaMemcpyAsync) (void*, const void*, size_t, cudaMemcpyKind, cudaStream_t);
static cudaError_t (*pfn_cudaStreamCreate) (cudaStream_t*);
static cudaError_t (*pfn_cudaStreamCreateWithPriority) (cudaStream_t*, unsigned int, int);
static cudaError_t (*pfn_cudaStreamDestroy) (cudaStream_t);
static cudaError_t (*pfn_cudaStreamSynchronize) (cudaStream_t);
static cudaError_t (*pfn_cudaStreamWaitEvent) (cudaStream_t, cudaEvent_t, unsigned int);
static cudaError_t (*pfn_cudaEventCreate) (cudaEvent_t*);
static cudaError_t (*pfn_cudaEventCreateWithFlags) (cudaEvent_t*, unsigned int);
static cudaError_t (*pfn_cudaEventRecord) (cudaEvent_t, cudaStream_t);
static cudaError_t (*pfn_cudaEventDestroy) (cudaEvent_t);
static const char* (*pfn_cudaGetErrorString) (cudaError_t);
static cudaError_t (*pfn_cudaGetLastError) ();
#endif // NV_DYNAMIC_CUDA
private:
static bool s_initialized;
static int s_trtMajor;
static std::string s_trtPath;
static std::string s_onnxPath;
static std::string s_cudaPath;
static std::string s_cudnnPath;
static LibHandle s_hTrt;
static LibHandle s_hOnnx;
static LibHandle s_hCuda;
static LibHandle s_hCudnn;
// Candidate DLL / SO name lists.
static std::vector<std::string> TrtCandidates();
static std::vector<std::string> OnnxCandidates();
static std::vector<std::string> CudnnCandidates();
static std::vector<std::string> CudaRtCandidates();
};

69
core/ANSLibsLoader/include/OvLoader.h Normal file
View File

@@ -0,0 +1,69 @@
#pragma once
// ============================================================================
// OvLoader -- Runtime discovery and pre-loading of OpenVINO DLLs
//
// OpenVINO is normally import-linked (openvino.lib), but the DLLs
// must be discoverable at process load time. OvLoader ensures the correct
// DLLs are found by:
// 1. Pre-loading Intel TBB (dependency of openvino.dll)
// 2. Pre-loading openvino.dll from the ANSCENTER shared directory
// 3. Optionally querying the OpenVINO version via the C API
//
// Plugins (CPU, GPU, NPU) and frontends (IR, ONNX, etc.) are discovered
// automatically by OpenVINO's internal plugin loader from the same directory
// once the shared directory has been injected into the DLL search path.
// ============================================================================
#include "ANSLibsLoader.h" // ANSLIBS_API
#include "DynLibUtils.h" // LibHandle
#include <string>
#include <vector>
#include <mutex>
// ============================================================================
struct ANSLIBS_API OvInfo
{
std::string dllPath; // Full path to loaded openvino.dll
std::string version; // e.g. "2024.6.0" (build number from C API)
bool loaded = false;
};
// ============================================================================
class ANSLIBS_API OvLoader
{
public:
/// Discover and pre-load OpenVINO + TBB DLLs.
/// Safe to call multiple times -- subsequent calls are no-ops.
/// @param shared_dir Directory to search first (default: ANSCENTER shared).
/// @param verbose Print discovery results to stdout.
/// @returns Reference to the discovery result.
[[nodiscard]] static const OvInfo& Initialize(
const std::string& shared_dir = ANSCENTER::DynLib::DEFAULT_SHARED_DIR,
bool verbose = true);
/// Release pre-loaded library handles.
static void Shutdown();
/// Query current state.
[[nodiscard]] static const OvInfo& Current() noexcept { return s_info; }
[[nodiscard]] static bool IsInitialized() noexcept { return s_initialized; }
private:
/// Build candidate DLL names for each component.
static std::vector<std::string> TbbCandidates(const std::string& shared_dir);
static std::vector<std::string> TbbMallocCandidates(const std::string& shared_dir);
static std::vector<std::string> OvCoreCandidates(const std::string& shared_dir);
static std::vector<std::string> OvCApiCandidates(const std::string& shared_dir);
/// Try to extract version string via the ov_get_openvino_version C API.
static std::string TryGetVersion(LibHandle hOvC);
static std::mutex s_mutex;
static bool s_initialized;
static OvInfo s_info;
static LibHandle s_hTbb;
static LibHandle s_hTbbMalloc;
static LibHandle s_hOvCore;
static LibHandle s_hOvC;
};

5
core/ANSLibsLoader/pch.cpp Normal file
View File

@@ -0,0 +1,5 @@
// pch.cpp: source file corresponding to the pre-compiled header
#include "pch.h"
// When you are using pre-compiled headers, this source file is necessary for compilation to succeed.

13
core/ANSLibsLoader/pch.h Normal file
View File

@@ -0,0 +1,13 @@
// pch.h: This is a precompiled header file.
// Files listed below are compiled only once, improving build performance for future builds.
// This also affects IntelliSense performance, including code completion and many code browsing features.
// However, files listed here are ALL re-compiled if any one of them is updated between builds.
// Do not add files here that you will be updating frequently as this negates the performance advantage.
#ifndef PCH_H
#define PCH_H
// add headers that you want to pre-compile here
#include "framework.h"
#endif //PCH_H