Working version on Razor Blade

This commit is contained in:
2026-06-21 19:58:17 +10:00
parent 74ad349c0f
commit f16d0d5623
274 changed files with 304168 additions and 1539 deletions
+376 -20
View File
@@ -12,6 +12,7 @@
#include <iterator>
#include <string>
#include <type_traits>
#include <utility>
#include <vector>
// Convert OrtStatus to Ort::Status and return
@@ -669,9 +670,12 @@ inline void KeyValuePairs::Remove(const char* key) {
GetApi().RemoveKeyValuePair(this->p_, key);
}
inline void* SyncStream::GetHandle() const {
namespace detail {
template <typename T>
inline void* SyncStreamImpl<T>::GetHandle() {
return GetApi().SyncStream_GetHandle(this->p_);
}
} // namespace detail
namespace detail {
template <typename T>
@@ -743,6 +747,61 @@ inline EpDevice::EpDevice(OrtEpFactory& ep_factory, ConstHardwareDevice& hardwar
ThrowOnError(GetEpApi().CreateEpDevice(&ep_factory, hardware_device, ep_metadata, ep_options, &p_));
}
namespace detail {
template <typename T>
inline std::string EpAssignedSubgraphImpl<T>::GetEpName() const {
const char* ep_name = nullptr;
// Returned null-terminated string will not be null if API function returns successfully.
ThrowOnError(GetApi().EpAssignedSubgraph_GetEpName(this->p_, &ep_name));
return std::string(ep_name);
}
template <typename T>
inline std::vector<ConstEpAssignedNode> EpAssignedSubgraphImpl<T>::GetNodes() const {
size_t num_ep_nodes = 0;
const OrtEpAssignedNode* const* ep_node_ptrs = nullptr;
ThrowOnError(GetApi().EpAssignedSubgraph_GetNodes(this->p_, &ep_node_ptrs, &num_ep_nodes));
std::vector<ConstEpAssignedNode> ep_nodes;
if (num_ep_nodes > 0) {
ep_nodes.reserve(num_ep_nodes);
for (size_t i = 0; i < num_ep_nodes; ++i) {
ep_nodes.emplace_back(ep_node_ptrs[i]);
}
}
return ep_nodes;
}
template <typename T>
inline std::string EpAssignedNodeImpl<T>::GetName() const {
const char* node_name = nullptr;
// Returned null-terminated string will not be null if API function returns successfully.
ThrowOnError(GetApi().EpAssignedNode_GetName(this->p_, &node_name));
return std::string(node_name);
}
template <typename T>
inline std::string EpAssignedNodeImpl<T>::GetDomain() const {
const char* domain = nullptr;
// Returned null-terminated string will not be null if API function returns successfully.
ThrowOnError(GetApi().EpAssignedNode_GetDomain(this->p_, &domain));
return std::string(domain);
}
template <typename T>
inline std::string EpAssignedNodeImpl<T>::GetOperatorType() const {
const char* op_type = nullptr;
// Returned null-terminated string will not be null if API function returns successfully.
ThrowOnError(GetApi().EpAssignedNode_GetOperatorType(this->p_, &op_type));
return std::string(op_type);
}
} // namespace detail
inline Env::Env(OrtLoggingLevel logging_level, _In_ const char* logid) {
ThrowOnError(GetApi().CreateEnv(logging_level, logid, &p_));
if (strcmp(logid, "onnxruntime-node") == 0) {
@@ -780,6 +839,15 @@ inline Env::Env(const OrtThreadingOptions* tp_options, OrtLoggingFunction loggin
}
}
inline Env::Env(const OrtEnvCreationOptions* options) {
ThrowOnError(GetApi().CreateEnvWithOptions(options, &p_));
if (strcmp(options->log_id, "onnxruntime-node") == 0) {
ThrowOnError(GetApi().SetLanguageProjection(p_, OrtLanguageProjection::ORT_PROJECTION_NODEJS));
} else {
ThrowOnError(GetApi().SetLanguageProjection(p_, OrtLanguageProjection::ORT_PROJECTION_CPLUSPLUS));
}
}
inline Env& Env::EnableTelemetryEvents() {
ThrowOnError(GetApi().EnableTelemetryEvents(p_));
return *this;
@@ -915,6 +983,20 @@ inline OrtCompiledModelCompatibility GetModelCompatibilityForEpDevices(
return status;
}
inline AllocatedStringPtr GetCompatibilityInfoFromModelAllocated(const ORTCHAR_T* model_path, const char* ep_type,
OrtAllocator* allocator) {
char* compat_info = nullptr;
ThrowOnError(GetApi().GetCompatibilityInfoFromModel(model_path, ep_type, allocator, &compat_info));
return AllocatedStringPtr(compat_info, detail::AllocatedFree(allocator));
}
inline AllocatedStringPtr GetCompatibilityInfoFromModelBytesAllocated(const void* model_data, size_t model_data_length,
const char* ep_type, OrtAllocator* allocator) {
char* compat_info = nullptr;
ThrowOnError(GetApi().GetCompatibilityInfoFromModelBytes(model_data, model_data_length, ep_type, allocator, &compat_info));
return AllocatedStringPtr(compat_info, detail::AllocatedFree(allocator));
}
inline LoraAdapter LoraAdapter::CreateLoraAdapter(const std::basic_string<ORTCHAR_T>& adapter_path,
OrtAllocator* allocator) {
OrtLoraAdapter* p;
@@ -990,6 +1072,11 @@ inline RunOptions& RunOptions::AddActiveLoraAdapter(const LoraAdapter& adapter)
return *this;
}
inline RunOptions& RunOptions::SetSyncStream(OrtSyncStream* stream) {
GetApi().RunOptionsSetSyncStream(p_, stream);
return *this;
}
inline ModelCompilationOptions::ModelCompilationOptions(const Env& env, const SessionOptions& session_options) {
ThrowOnError(GetCompileApi().CreateModelCompilationOptionsFromSessionOptions(env, session_options, &this->p_));
}
@@ -1500,6 +1587,18 @@ inline SessionOptionsImpl<T>& SessionOptionsImpl<T>::RegisterCustomOpsUsingFunct
return *this;
}
template <typename T>
inline SessionOptionsImpl<T>& SessionOptionsImpl<T>::AddFreeDimensionOverride(const char* dim_denotation, int64_t dim_value) {
ThrowOnError(GetApi().AddFreeDimensionOverrideByName(this->p_, dim_denotation, dim_value));
return *this;
}
template <typename T>
inline SessionOptionsImpl<T>& SessionOptionsImpl<T>::AddFreeDimensionOverrideByName(const char* dim_name, int64_t dim_value) {
ThrowOnError(GetApi().AddFreeDimensionOverrideByName(this->p_, dim_name, dim_value));
return *this;
}
/// Session
template <typename T>
inline size_t ConstSessionImpl<T>::GetInputCount() const {
@@ -1582,11 +1681,13 @@ inline std::vector<ConstMemoryInfo> ConstSessionImpl<T>::GetMemoryInfoForInputs(
auto num_inputs = GetInputCount();
std::vector<ConstMemoryInfo> mem_infos;
mem_infos.resize(num_inputs);
if (num_inputs > 0) {
mem_infos.resize(num_inputs);
ThrowOnError(GetApi().SessionGetMemoryInfoForInputs(this->p_,
reinterpret_cast<const OrtMemoryInfo**>(mem_infos.data()),
num_inputs));
ThrowOnError(GetApi().SessionGetMemoryInfoForInputs(this->p_,
reinterpret_cast<const OrtMemoryInfo**>(mem_infos.data()),
num_inputs));
}
return mem_infos;
}
@@ -1598,11 +1699,13 @@ inline std::vector<ConstMemoryInfo> ConstSessionImpl<T>::GetMemoryInfoForOutputs
auto num_outputs = GetOutputCount();
std::vector<ConstMemoryInfo> mem_infos;
mem_infos.resize(num_outputs);
if (num_outputs > 0) {
mem_infos.resize(num_outputs);
ThrowOnError(GetApi().SessionGetMemoryInfoForOutputs(this->p_,
reinterpret_cast<const OrtMemoryInfo**>(mem_infos.data()),
num_outputs));
ThrowOnError(GetApi().SessionGetMemoryInfoForOutputs(this->p_,
reinterpret_cast<const OrtMemoryInfo**>(mem_infos.data()),
num_outputs));
}
return mem_infos;
}
@@ -1631,15 +1734,28 @@ template <typename T>
inline std::vector<ConstEpDevice> ConstSessionImpl<T>::GetEpDeviceForInputs() const {
auto num_inputs = GetInputCount();
std::vector<ConstEpDevice> input_devices;
input_devices.resize(num_inputs);
ThrowOnError(GetApi().SessionGetEpDeviceForInputs(this->p_,
reinterpret_cast<const OrtEpDevice**>(input_devices.data()),
num_inputs));
if (num_inputs > 0) {
input_devices.resize(num_inputs);
ThrowOnError(GetApi().SessionGetEpDeviceForInputs(this->p_,
reinterpret_cast<const OrtEpDevice**>(input_devices.data()),
num_inputs));
}
return input_devices;
}
template <typename T>
inline std::vector<ConstEpDevice> ConstSessionImpl<T>::GetEpDeviceForOutputs() const {
auto num_outputs = GetOutputCount();
std::vector<ConstEpDevice> output_devices;
if (num_outputs > 0) {
output_devices.resize(num_outputs);
ThrowOnError(GetApi().SessionGetEpDeviceForOutputs(this->p_,
reinterpret_cast<const OrtEpDevice**>(output_devices.data()),
num_outputs));
}
return output_devices;
}
template <typename T>
inline uint64_t ConstSessionImpl<T>::GetProfilingStartTimeNs() const {
uint64_t out;
@@ -1714,6 +1830,23 @@ std::vector<ValueInfo> ConstSessionImpl<T>::GetOutputs() const {
return outputs;
}
template <typename T>
inline std::vector<ConstEpAssignedSubgraph> ConstSessionImpl<T>::GetEpGraphAssignmentInfo() const {
size_t num_ep_subgraphs = 0;
const OrtEpAssignedSubgraph* const* ep_subgraph_ptrs = nullptr;
ThrowOnError(GetApi().Session_GetEpGraphAssignmentInfo(this->p_, &ep_subgraph_ptrs, &num_ep_subgraphs));
std::vector<ConstEpAssignedSubgraph> ep_subgraphs;
if (num_ep_subgraphs > 0) {
ep_subgraphs.reserve(num_ep_subgraphs);
for (size_t i = 0; i < num_ep_subgraphs; ++i) {
ep_subgraphs.emplace_back(ep_subgraph_ptrs[i]);
}
}
return ep_subgraphs;
}
template <typename T>
inline std::vector<Value> SessionImpl<T>::Run(const RunOptions& run_options, const char* const* input_names, const Value* input_values, size_t input_count,
const char* const* output_names, size_t output_count) {
@@ -1975,7 +2108,12 @@ template <typename T>
inline size_t TensorTypeAndShapeInfoImpl<T>::GetElementCount() const {
size_t out;
ThrowOnError(GetApi().GetTensorShapeElementCount(this->p_, &out));
return static_cast<size_t>(out);
return out;
}
template <typename T>
inline bool TensorTypeAndShapeInfoImpl<T>::HasShape() const {
return GetApi().TensorTypeAndShape_HasShape(this->p_);
}
template <typename T>
@@ -1997,8 +2135,12 @@ inline void TensorTypeAndShapeInfoImpl<T>::GetSymbolicDimensions(const char** va
template <typename T>
inline std::vector<const char*> TensorTypeAndShapeInfoImpl<T>::GetSymbolicDimensions() const {
std::vector<const char*> out(GetDimensionsCount(), nullptr);
ThrowOnError(GetApi().GetSymbolicDimensions(this->p_, out.data(), out.size()));
std::vector<const char*> out;
size_t dim_count = GetDimensionsCount();
if (dim_count > 0) {
out.resize(dim_count, nullptr);
ThrowOnError(GetApi().GetSymbolicDimensions(this->p_, out.data(), out.size()));
}
return out;
}
@@ -2235,6 +2377,13 @@ inline const R* ConstValueImpl<T>::GetSparseTensorValues() const {
#endif
template <typename T>
void ConstValueImpl<T>::GetTensorElementTypeAndShapeDataReference(ONNXTensorElementDataType& elem_type,
Shape& shape) const {
ThrowOnError(GetApi().GetTensorElementTypeAndShapeDataReference(this->p_, &elem_type, &shape.shape,
&shape.shape_len));
}
template <typename T>
void ValueImpl<T>::FillStringTensor(const char* const* s, size_t s_len) {
ThrowOnError(GetApi().FillStringTensor(this->p_, s, s_len));
@@ -2518,10 +2667,10 @@ inline void* KernelContext::GetGPUComputeStream() const {
return out;
}
inline OrtAllocator* KernelContext::GetAllocator(const OrtMemoryInfo& memory_info) const {
inline Ort::Allocator KernelContext::GetAllocator(const OrtMemoryInfo& memory_info) const {
OrtAllocator* out = nullptr;
Ort::ThrowOnError(GetApi().KernelContext_GetAllocator(ctx_, &memory_info, &out));
return out;
return Ort::Allocator{out};
}
inline Logger KernelContext::GetLogger() const {
@@ -2806,6 +2955,57 @@ inline Logger KernelInfoImpl<T>::GetLogger() const {
return Logger{out};
}
template <typename T>
inline KeyValuePairs KernelInfoImpl<T>::GetConfigEntries() const {
OrtKeyValuePairs* out = nullptr;
Ort::ThrowOnError(GetApi().KernelInfo_GetConfigEntries(this->p_, &out));
return KeyValuePairs{out};
}
template <typename T>
inline std::string KernelInfoImpl<T>::GetOperatorDomain() const {
size_t size = 0;
// Feed nullptr for the data buffer to query the true size of the string value
Ort::ThrowOnError(GetApi().KernelInfo_GetOperatorDomain(this->p_, nullptr, &size));
std::string out;
out.resize(size);
Ort::ThrowOnError(GetApi().KernelInfo_GetOperatorDomain(this->p_, &out[0], &size));
out.resize(size - 1); // remove the terminating character '\0'
return out;
}
template <typename T>
inline std::string KernelInfoImpl<T>::GetOperatorType() const {
size_t size = 0;
// Feed nullptr for the data buffer to query the true size of the string value
Ort::ThrowOnError(GetApi().KernelInfo_GetOperatorType(this->p_, nullptr, &size));
std::string out;
out.resize(size);
Ort::ThrowOnError(GetApi().KernelInfo_GetOperatorType(this->p_, &out[0], &size));
out.resize(size - 1); // remove the terminating character '\0'
return out;
}
template <typename T>
inline int KernelInfoImpl<T>::GetOperatorSinceVersion() const {
int out = 0;
Ort::ThrowOnError(GetApi().KernelInfo_GetOperatorSinceVersion(this->p_, &out));
return out;
}
template <typename T>
inline const OrtEp* KernelInfoImpl<T>::GetEp() const {
const OrtEp* ep = nullptr;
Ort::ThrowOnError(GetEpApi().KernelInfo_GetEp(this->p_, &ep));
return ep;
}
inline void attr_utils::GetAttr(const OrtKernelInfo* p, const char* name, float& out) {
Ort::ThrowOnError(GetApi().KernelInfoGetAttribute_float(p, name, &out));
}
@@ -3537,4 +3737,160 @@ inline Model::Model(const std::vector<DomainOpsetPair>& opsets) {
}
#endif
namespace detail {
template <typename T>
inline const char* ConstKernelDefImpl<T>::GetOperatorType() const {
return GetEpApi().KernelDef_GetOperatorType(this->p_);
}
template <typename T>
inline const char* ConstKernelDefImpl<T>::GetDomain() const {
return GetEpApi().KernelDef_GetDomain(this->p_);
}
template <typename T>
inline std::pair<int, int> ConstKernelDefImpl<T>::GetSinceVersion() const {
int start = 0;
int end = 0;
ThrowOnError(GetEpApi().KernelDef_GetSinceVersion(this->p_, &start, &end));
return std::pair<int, int>(start, end);
}
template <typename T>
inline const char* ConstKernelDefImpl<T>::GetExecutionProvider() const {
return GetEpApi().KernelDef_GetExecutionProvider(this->p_);
}
template <typename T>
inline OrtMemType ConstKernelDefImpl<T>::GetInputMemType(size_t input_index) const {
OrtMemType mem_type{};
ThrowOnError(GetEpApi().KernelDef_GetInputMemType(this->p_, input_index, &mem_type));
return mem_type;
}
template <typename T>
inline OrtMemType ConstKernelDefImpl<T>::GetOutputMemType(size_t output_index) const {
OrtMemType mem_type{};
ThrowOnError(GetEpApi().KernelDef_GetOutputMemType(this->p_, output_index, &mem_type));
return mem_type;
}
} // namespace detail
inline KernelDefBuilder::KernelDefBuilder() {
ThrowOnError(GetEpApi().CreateKernelDefBuilder(&p_));
}
inline KernelDefBuilder::KernelDefBuilder(OrtKernelDefBuilder* p) : detail::Base<OrtKernelDefBuilder>{p} {
}
inline KernelDefBuilder& KernelDefBuilder::SetOperatorType(const char* op_type) {
ThrowOnError(GetEpApi().KernelDefBuilder_SetOperatorType(p_, op_type));
return *this;
}
inline KernelDefBuilder& KernelDefBuilder::SetDomain(const char* domain) {
ThrowOnError(GetEpApi().KernelDefBuilder_SetDomain(p_, domain));
return *this;
}
inline KernelDefBuilder& KernelDefBuilder::SetSinceVersion(int since_version_start, int since_version_end) {
ThrowOnError(GetEpApi().KernelDefBuilder_SetSinceVersion(p_, since_version_start, since_version_end));
return *this;
}
inline KernelDefBuilder& KernelDefBuilder::SetExecutionProvider(const char* ep_name) {
ThrowOnError(GetEpApi().KernelDefBuilder_SetExecutionProvider(p_, ep_name));
return *this;
}
inline KernelDefBuilder& KernelDefBuilder::SetInputMemType(size_t input_index, OrtMemType mem_type) {
ThrowOnError(GetEpApi().KernelDefBuilder_SetInputMemType(p_, input_index, mem_type));
return *this;
}
inline KernelDefBuilder& KernelDefBuilder::SetOutputMemType(size_t output_index, OrtMemType mem_type) {
ThrowOnError(GetEpApi().KernelDefBuilder_SetOutputMemType(p_, output_index, mem_type));
return *this;
}
inline KernelDefBuilder& KernelDefBuilder::AddTypeConstraint(const char* arg_name,
const OrtDataType* data_type) {
ThrowOnError(GetEpApi().KernelDefBuilder_AddTypeConstraint(p_, arg_name, &data_type, 1));
return *this;
}
inline KernelDefBuilder& KernelDefBuilder::AddTypeConstraint(const char* arg_name,
const std::vector<const OrtDataType*>& data_types) {
ThrowOnError(GetEpApi().KernelDefBuilder_AddTypeConstraint(p_, arg_name, data_types.data(), data_types.size()));
return *this;
}
inline KernelDefBuilder& KernelDefBuilder::AddInputOutputAlias(int input_index, int output_index) {
ThrowOnError(GetEpApi().KernelDefBuilder_AddInputOutputAliases(p_, &input_index, &output_index, 1));
return *this;
}
inline KernelDefBuilder& KernelDefBuilder::AddInputOutputAliases(const std::vector<int>& input_indices,
const std::vector<int>& output_indices) {
if (input_indices.size() != output_indices.size()) {
ORT_CXX_API_THROW("Expecting input and output indices to have the same element count", ORT_INVALID_ARGUMENT);
}
ThrowOnError(GetEpApi().KernelDefBuilder_AddInputOutputAliases(p_, input_indices.data(), output_indices.data(),
input_indices.size()));
return *this;
}
inline KernelDefBuilder& KernelDefBuilder::AddInputOutputMutableAlias(int input_index, int output_index) {
ThrowOnError(GetEpApi().KernelDefBuilder_AddInputOutputMutableAliases(p_, &input_index, &output_index, 1));
return *this;
}
inline KernelDefBuilder& KernelDefBuilder::AddInputOutputMutableAliases(const std::vector<int>& input_indices,
const std::vector<int>& output_indices) {
if (input_indices.size() != output_indices.size()) {
ORT_CXX_API_THROW("Expecting input and output indices to have the same element count", ORT_INVALID_ARGUMENT);
}
ThrowOnError(GetEpApi().KernelDefBuilder_AddInputOutputMutableAliases(p_, input_indices.data(), output_indices.data(),
input_indices.size()));
return *this;
}
inline KernelDef KernelDefBuilder::Build() {
OrtKernelDef* kernel_def = nullptr;
ThrowOnError(GetEpApi().KernelDefBuilder_Build(p_, &kernel_def));
return KernelDef(kernel_def);
}
inline KernelRegistry::KernelRegistry() {
ThrowOnError(GetEpApi().CreateKernelRegistry(&p_));
}
inline KernelRegistry::KernelRegistry(OrtKernelRegistry* p) : detail::Base<OrtKernelRegistry>{p} {
}
inline Status KernelRegistry::AddKernel(const OrtKernelDef* kernel_def, OrtKernelCreateFunc kernel_create_func,
void* kernel_create_func_state) {
return Status{GetEpApi().KernelRegistry_AddKernel(p_, kernel_def, kernel_create_func, kernel_create_func_state)};
}
namespace detail {
template <typename T>
inline Status SharedPrePackedWeightCacheImpl<T>::StoreWeightData(void** buffer_data_ptrs, size_t* buffer_sizes,
size_t num_buffers) {
return Status{GetEpApi().SharedPrePackedWeightCache_StoreWeightData(this->p_, buffer_data_ptrs, buffer_sizes,
num_buffers)};
}
} // namespace detail
inline Ort::KeyValuePairs GetEnvConfigEntries() {
OrtKeyValuePairs* entries = nullptr;
Ort::ThrowOnError(GetEpApi().GetEnvConfigEntries(&entries));
return Ort::KeyValuePairs{entries};
}
} // namespace Ort