Working version on Razor Blade

This commit is contained in:
2026-06-21 19:58:17 +10:00
parent 74ad349c0f
commit f16d0d5623
274 changed files with 304168 additions and 1539 deletions
File diff suppressed because it is too large Load Diff
+357 -33
View File
@@ -80,22 +80,19 @@ struct Exception : std::exception {
throw Ort::Exception(string, code)
#endif
// This is used internally by the C++ API. This class holds the global variable that points to the OrtApi,
// it's in a template so that we can define a global variable in a header and make
// it transparent to the users of the API.
template <typename T>
struct Global {
static const OrtApi* api_;
};
// If macro ORT_API_MANUAL_INIT is defined, no static initialization will be performed. Instead, user must call InitApi() before using it.
template <typename T>
#ifdef ORT_API_MANUAL_INIT
const OrtApi* Global<T>::api_{};
inline void InitApi() noexcept { Global<void>::api_ = OrtGetApiBase()->GetApi(ORT_API_VERSION); }
// If the macro ORT_API_MANUAL_INIT is defined, no static initialization
// will be performed. Instead, users must call InitApi() before using the
// ORT C++ APIs..
//
// InitApi() sets the global API object using the default initialization
// logic. Users call this to initialize the ORT C++ APIs at a time that
// makes sense in their program.
inline void InitApi() noexcept;
// Used by custom operator libraries that are not linked to onnxruntime. Sets the global API object, which is
// required by C++ APIs.
// InitApi(const OrtApi*) is used by custom operator libraries that are not
// linked to onnxruntime. It sets the global API object, which is required
// by the ORT C++ APIs.
//
// Example mycustomop.cc:
//
@@ -108,22 +105,88 @@ inline void InitApi() noexcept { Global<void>::api_ = OrtGetApiBase()->GetApi(OR
// // ...
// }
//
inline void InitApi(const OrtApi* api) noexcept { Global<void>::api_ = api; }
#else
#if defined(_MSC_VER) && !defined(__clang__)
#pragma warning(push)
// "Global initializer calls a non-constexpr function." Therefore you can't use ORT APIs in the other global initializers.
// Please define ORT_API_MANUAL_INIT if it conerns you.
#pragma warning(disable : 26426)
#endif
const OrtApi* Global<T>::api_ = OrtGetApiBase()->GetApi(ORT_API_VERSION);
#if defined(_MSC_VER) && !defined(__clang__)
#pragma warning(pop)
#endif
inline void InitApi(const OrtApi* api) noexcept;
#endif
namespace detail {
// This is used internally by the C++ API. This class holds the global
// variable that points to the OrtApi.
struct Global {
static const OrtApi* Api(const OrtApi* newValue = nullptr) noexcept {
// This block-level static will be initialized once when this function is
// first executed, delaying the call to DefaultInit() until it is first needed.
//
// When ORT_API_MANUAL_INIT is not defined, DefaultInit() calls
// OrtGetApiBase()->GetApi(), which may result in a shared library being
// loaded.
//
// Using a block-level static instead of a class-level static helps
// avoid issues with static initialization order and dynamic libraries
// loading other dynamic libraries.
//
// This makes it safe to include the C++ API headers in a shared library
// that is delay loaded or delay loads its dependencies.
//
// This DOES NOT make it safe to _use_ arbitrary ORT C++ APIs when
// initializing static members, however.
static const OrtApi* api = DefaultInit();
if (newValue) {
api = newValue;
}
return api;
}
private:
// Has different definitions based on ORT_API_MANUAL_INIT
static const OrtApi* DefaultInit() noexcept;
#ifdef ORT_API_MANUAL_INIT
// Public APIs to set the OrtApi* to use.
friend void ::Ort::InitApi() noexcept;
friend void ::Ort::InitApi(const OrtApi*) noexcept;
#endif
};
} // namespace detail
#ifdef ORT_API_MANUAL_INIT
// See comments on declaration above for usage.
inline void InitApi(const OrtApi* api) noexcept { detail::Global::Api(api); }
inline void InitApi() noexcept { InitApi(OrtGetApiBase()->GetApi(ORT_API_VERSION)); }
#ifdef _MSC_VER
// If you get a linker error about a mismatch here, you are trying to
// link two compilation units that have different definitions for
// ORT_API_MANUAL_INIT together. All compilation units must agree on the
// definition of ORT_API_MANUAL_INIT.
#pragma detect_mismatch("ORT_API_MANUAL_INIT", "enabled")
#endif
inline const OrtApi* detail::Global::DefaultInit() noexcept {
// When ORT_API_MANUAL_INIT is defined, there's no default init that can
// be done.
return nullptr;
}
#else // ORT_API_MANUAL_INIT
#ifdef _MSC_VER
// If you get a linker error about a mismatch here, you are trying to link
// two compilation units that have different definitions for
// ORT_API_MANUAL_INIT together. All compilation units must agree on the
// definition of ORT_API_MANUAL_INIT.
#pragma detect_mismatch("ORT_API_MANUAL_INIT", "disabled")
#endif
inline const OrtApi* detail::Global::DefaultInit() noexcept {
return OrtGetApiBase()->GetApi(ORT_API_VERSION);
}
#endif // ORT_API_MANUAL_INIT
/// This returns a reference to the ORT C API.
inline const OrtApi& GetApi() noexcept { return *Global<void>::api_; }
inline const OrtApi& GetApi() noexcept { return *detail::Global::Api(); }
/// <summary>
/// This function returns the onnxruntime version string
@@ -173,6 +236,20 @@ inline const OrtCompileApi& GetCompileApi() {
return *api;
}
/// <summary>
/// This returns a reference to the ORT C Interop API. Used for external resource import with EPs.
/// </summary>
/// <returns>ORT C Interop API reference</returns>
inline const OrtInteropApi& GetInteropApi() {
auto* api = GetApi().GetInteropApi();
if (api == nullptr) {
// minimal build
ORT_CXX_API_THROW("Interop API is not available in this build", ORT_FAIL);
}
return *api;
}
/// <summary>
/// This returns a reference to the ORT C EP API. Used if authoring a plugin execution provider.
/// </summary>
@@ -581,6 +658,9 @@ ORT_DEFINE_RELEASE(ValueInfo);
ORT_DEFINE_RELEASE_FROM_API_STRUCT(ModelCompilationOptions, GetCompileApi);
ORT_DEFINE_RELEASE_FROM_API_STRUCT(EpDevice, GetEpApi);
ORT_DEFINE_RELEASE_FROM_API_STRUCT(KernelDef, GetEpApi);
ORT_DEFINE_RELEASE_FROM_API_STRUCT(KernelDefBuilder, GetEpApi);
ORT_DEFINE_RELEASE_FROM_API_STRUCT(KernelRegistry, GetEpApi);
// This is defined explicitly since OrtTensorRTProviderOptionsV2 is not a C API type,
// but the struct has V2 in its name to indicate that it is the second version of the options.
@@ -993,6 +1073,9 @@ struct AllocatorWithDefaultOptions : detail::AllocatorImpl<detail::Unowned<OrtAl
struct Allocator : detail::AllocatorImpl<OrtAllocator> {
explicit Allocator(std::nullptr_t) {} ///< Convenience to create a class member and then replace with an instance
Allocator(const Session& session, const OrtMemoryInfo*);
///< Take ownership of a pointer created by C API
explicit Allocator(OrtAllocator* p) : AllocatorImpl<OrtAllocator>{p} {}
};
using UnownedAllocator = detail::AllocatorImpl<detail::Unowned<OrtAllocator>>;
@@ -1000,11 +1083,25 @@ using UnownedAllocator = detail::AllocatorImpl<detail::Unowned<OrtAllocator>>;
/** \brief Wrapper around ::OrtSyncStream
*
*/
struct SyncStream : detail::Base<OrtSyncStream> {
explicit SyncStream(std::nullptr_t) {} ///< Create an empty SyncStream object, must be assigned a valid one to be used
explicit SyncStream(OrtSyncStream* p) : Base<OrtSyncStream>{p} {} ///< Take ownership of a pointer created by C API
void* GetHandle() const; ///< Wraps SyncStream_GetHandle
namespace detail {
template <typename T>
struct SyncStreamImpl : Base<T> {
using B = Base<T>;
using B::B;
// For some reason this is not a const method on the stream
void* GetHandle(); ///< Wraps SyncStream_GetHandle
};
} // namespace detail
struct SyncStream : detail::SyncStreamImpl<OrtSyncStream> {
///< Create an empty SyncStream object, must be assigned a valid one to be used
explicit SyncStream(std::nullptr_t) {}
///< Take ownership of a pointer created by C API
explicit SyncStream(OrtSyncStream* p) : SyncStreamImpl<OrtSyncStream>{p} {}
};
using UnownedSyncStream = detail::SyncStreamImpl<detail::Unowned<OrtSyncStream>>;
namespace detail {
template <typename T>
@@ -1067,6 +1164,72 @@ OrtCompiledModelCompatibility GetModelCompatibilityForEpDevices(
const std::vector<ConstEpDevice>& ep_devices,
const char* compatibility_info);
/** \brief Extract EP compatibility info from a precompiled model file.
*
* Parses the model file to extract the compatibility info string for a specific execution provider
* from the model's metadata properties. This is only applicable to models that have been precompiled
* for an EP. Standard ONNX models do not contain this information.
*
* \note This operation parses the full ONNX ModelProto from disk.
*
* \param model_path Path to the ONNX model file.
* \param ep_type The execution provider type string. Must be non-empty.
* Use ConstEpDevice::EpName() to get this value.
* \param allocator Allocator to use for the output string.
* \return The compatibility info string, or nullptr if not found for this EP. Caller must free via allocator.
* \throws Ort::Exception on error.
*/
AllocatedStringPtr GetCompatibilityInfoFromModelAllocated(const ORTCHAR_T* model_path, const char* ep_type,
OrtAllocator* allocator);
/** \brief Extract EP compatibility info from precompiled model bytes in memory.
*
* Same as GetCompatibilityInfoFromModelAllocated but reads from a memory buffer.
* Useful when precompiled models are loaded from encrypted storage, network, or other non-file sources.
*
* \param model_data Pointer to the model data in memory.
* \param model_data_length Size of the model data in bytes.
* \param ep_type The execution provider type string. Must be non-empty.
* \param allocator Allocator to use for the output string.
* \return The compatibility info string, or nullptr if not found for this EP. Caller must free via allocator.
* \throws Ort::Exception on error.
*/
AllocatedStringPtr GetCompatibilityInfoFromModelBytesAllocated(const void* model_data, size_t model_data_length,
const char* ep_type, OrtAllocator* allocator);
namespace detail {
template <typename T>
struct EpAssignedNodeImpl : Ort::detail::Base<T> {
using B = Ort::detail::Base<T>;
using B::B;
std::string GetName() const;
std::string GetDomain() const;
std::string GetOperatorType() const;
};
} // namespace detail
/** \brief Constant wrapper around ::OrtEpAssignedNode
* \remarks EpAssignedNode is always read-only for ORT API users.
*/
using ConstEpAssignedNode = detail::EpAssignedNodeImpl<Ort::detail::Unowned<const OrtEpAssignedNode>>;
namespace detail {
template <typename T>
struct EpAssignedSubgraphImpl : Ort::detail::Base<T> {
using B = Ort::detail::Base<T>;
using B::B;
std::string GetEpName() const;
std::vector<ConstEpAssignedNode> GetNodes() const;
};
} // namespace detail
/** \brief Constant wrapper around ::OrtEpAssignedSubgraph
* \remarks EpAssignedSubgraph is always read-only for ORT API users.
*/
using ConstEpAssignedSubgraph = detail::EpAssignedSubgraphImpl<Ort::detail::Unowned<const OrtEpAssignedSubgraph>>;
/** \brief The Env (Environment)
*
* The Env holds the logging state used by all other objects.
@@ -1088,6 +1251,9 @@ struct Env : detail::Base<OrtEnv> {
Env(const OrtThreadingOptions* tp_options, OrtLoggingFunction logging_function, void* logger_param,
OrtLoggingLevel logging_level = ORT_LOGGING_LEVEL_WARNING, _In_ const char* logid = "");
/// \brief Wraps OrtApi::CreateEnvWithOptions
explicit Env(const OrtEnvCreationOptions* options);
/// \brief C Interop Helper
explicit Env(OrtEnv* p) : Base<OrtEnv>{p} {}
@@ -1207,6 +1373,15 @@ struct RunOptions : detail::Base<OrtRunOptions> {
* \param adapter The LoraAdapter to be used as the active adapter
*/
RunOptions& AddActiveLoraAdapter(const LoraAdapter& adapter);
/** \brief Associate a sync stream with the run options.
*
* When set, the EP uses this stream for execution, enabling proper
* synchronization with imported external semaphores. Wraps OrtApi::RunOptionsSetSyncStream.
*
* \param stream The OrtSyncStream to associate with these run options. May be nullptr to clear.
*/
RunOptions& SetSyncStream(OrtSyncStream* stream);
};
namespace detail {
@@ -1364,6 +1539,12 @@ struct SessionOptionsImpl : ConstSessionOptionsImpl<T> {
///< Wraps OrtApi::SessionOptionsAppendExecutionProvider_VitisAI
SessionOptionsImpl& AppendExecutionProvider_VitisAI(const std::unordered_map<std::string, std::string>& provider_options = {});
///< Wraps OrtApi::AddFreeDimensionOverride
SessionOptionsImpl& AddFreeDimensionOverride(const char* dim_denotation, int64_t dim_value);
///< Wraps OrtApi::AddFreeDimensionOverrideByName
SessionOptionsImpl& AddFreeDimensionOverrideByName(const char* dim_name, int64_t dim_value);
};
} // namespace detail
@@ -1521,6 +1702,7 @@ struct ConstSessionImpl : Base<T> {
std::vector<ConstMemoryInfo> GetMemoryInfoForInputs() const; ///< Wrapper for OrtApi::SessionGetMemoryInfoForInputs
std::vector<ConstMemoryInfo> GetMemoryInfoForOutputs() const; ///< Wrapper for OrtApi::SessionGetMemoryInfoForOutputs
std::vector<ConstEpDevice> GetEpDeviceForInputs() const; ///< Wrapper for OrtApi::SessionGetEpDeviceForInputs
std::vector<ConstEpDevice> GetEpDeviceForOutputs() const; ///< Wrapper for OrtApi::SessionGetEpDeviceForOutputs
/** \brief Returns a copy of input name at the specified index.
*
@@ -1558,9 +1740,14 @@ struct ConstSessionImpl : Base<T> {
int GetOpset(const std::string& domain) const; ///< Wraps OrtApi::SessionGetOpsetForDomain
// Will move before checkin if that's the case.
std::vector<ValueInfo> GetInputs() const;
std::vector<ValueInfo> GetOutputs() const;
/** \brief Returns information on the subgraph/nodes assigned to execution providers in the session.
*
* \return A list of ConstEpAssignedSubgraph instances.
*/
std::vector<ConstEpAssignedSubgraph> GetEpGraphAssignmentInfo() const;
};
template <typename T>
@@ -1705,6 +1892,7 @@ struct TensorTypeAndShapeInfoImpl : Base<T> {
void GetSymbolicDimensions(const char** values, size_t values_count) const; ///< Wraps OrtApi::GetSymbolicDimensions
std::vector<const char*> GetSymbolicDimensions() const;
bool HasShape() const; ///< Wraps OrtApi::TensorTypeAndShape_HasShape
std::vector<int64_t> GetShape() const; ///< Uses GetDimensionsCount & GetDimensions to return a std::vector of the shape
};
@@ -2032,6 +2220,19 @@ struct ConstValueImpl : Base<T> {
const R* GetSparseTensorValues() const;
#endif
/// <summary>
/// Returns the tensor's element type and a reference to the tensor's internal shape data. The shape data is owned
/// by the Ort::Value and becomes invalid when the Ort::Value is destroyed or if the underlying shape data is
/// updated or reallocated.
///
/// For a scalar, shape.shape is nullptr and shape.shape_len is 0.
///
/// Wraps OrtApi::GetTensorElementTypeAndShapeDataReference.
/// </summary>
/// <param name="elem_type">Output parameter set to the element's data type.</param>
/// <param name="shape">Output parameter set to the OrtValue instance's shape data and number of elements.</param>
void GetTensorElementTypeAndShapeDataReference(ONNXTensorElementDataType& elem_type, Shape& shape) const;
};
template <typename T>
@@ -2635,7 +2836,7 @@ struct KernelContext {
UnownedValue GetOutput(size_t index, const std::vector<int64_t>& dims) const;
void* GetGPUComputeStream() const;
Logger GetLogger() const;
OrtAllocator* GetAllocator(const OrtMemoryInfo& memory_info) const;
Ort::Allocator GetAllocator(const OrtMemoryInfo& memory_info) const;
OrtKernelContext* GetOrtKernelContext() const { return ctx_; }
void ParallelFor(void (*fn)(void*, size_t), size_t total, size_t num_batch, void* usr_data) const;
@@ -2690,6 +2891,13 @@ struct KernelInfoImpl : Base<T> {
std::string GetNodeName() const;
Logger GetLogger() const;
KeyValuePairs GetConfigEntries() const;
std::string GetOperatorDomain() const; ///< Wraps OrtApi::KernelInfo_GetOperatorDomain
std::string GetOperatorType() const; ///< Wraps OrtApi::KernelInfo_GetOperatorType
int GetOperatorSinceVersion() const; ///< Wraps OrtApi::KernelInfo_GetOperatorSinceVersion
const OrtEp* GetEp() const; ///< Wraps OrtEpApi::KernelInfo_GetEp
};
} // namespace detail
@@ -3206,5 +3414,121 @@ struct Model : detail::ModelImpl<OrtModel> {
explicit Model(const std::vector<DomainOpsetPair>& opsets);
#endif
};
namespace detail {
template <typename T>
struct ConstKernelDefImpl : Base<T> {
using B = Base<T>;
using B::B;
///< Wraps OrtEpApi::KernelDef_GetOperatorType
const char* GetOperatorType() const;
///< Wraps OrtEpApi::KernelDef_GetDomain
const char* GetDomain() const;
///< Wraps OrtEpApi::KernelDef_GetSinceVersion
std::pair<int, int> GetSinceVersion() const;
///< Wraps OrtEpApi::KernelDef_GetExecutionProvider
const char* GetExecutionProvider() const;
///< Wraps OrtEpApi::KernelDef_GetInputMemType
OrtMemType GetInputMemType(size_t input_index) const;
///< Wraps OrtEpApi::KernelDef_GetOutputMemType
OrtMemType GetOutputMemType(size_t output_index) const;
};
} // namespace detail
using ConstKernelDef = detail::ConstKernelDefImpl<detail::Unowned<const OrtKernelDef>>;
struct KernelDef : detail::ConstKernelDefImpl<OrtKernelDef> {
using Base = detail::ConstKernelDefImpl<OrtKernelDef>;
using Base::Base;
explicit KernelDef(std::nullptr_t) {}
explicit KernelDef(OrtKernelDef* p) : detail::ConstKernelDefImpl<OrtKernelDef>{p} {}
ConstKernelDef GetConst() const { return ConstKernelDef{this->p_}; }
};
/** \brief Builder for OrtKernelDef.
*
* Used by plugin EPs to build a kernel definition.
*/
struct KernelDefBuilder : detail::Base<OrtKernelDefBuilder> {
KernelDefBuilder(); ///< Wraps OrtEpApi::CreateKernelDefBuilder
explicit KernelDefBuilder(std::nullptr_t) {} ///< Create an empty object, must be assigned a valid one to be used
explicit KernelDefBuilder(OrtKernelDefBuilder* ort_kernel_def_builder);
KernelDefBuilder& SetOperatorType(const char* op_type);
KernelDefBuilder& SetDomain(const char* domain);
KernelDefBuilder& SetSinceVersion(int since_version_start, int since_version_end);
KernelDefBuilder& SetExecutionProvider(const char* ep_name);
KernelDefBuilder& SetInputMemType(size_t input_index, OrtMemType mem_type);
KernelDefBuilder& SetOutputMemType(size_t output_index, OrtMemType mem_type);
KernelDefBuilder& AddTypeConstraint(const char* arg_name, const OrtDataType* data_type);
KernelDefBuilder& AddTypeConstraint(const char* arg_name, const std::vector<const OrtDataType*>& data_types);
KernelDefBuilder& AddInputOutputAlias(int input_index, int output_index);
KernelDefBuilder& AddInputOutputAliases(const std::vector<int>& input_indices,
const std::vector<int>& output_indices);
KernelDefBuilder& AddInputOutputMutableAlias(int input_index, int output_index);
KernelDefBuilder& AddInputOutputMutableAliases(const std::vector<int>& input_indices,
const std::vector<int>& output_indices);
KernelDef Build();
};
/** \brief Registry for kernels supported by an EP.
*
* Used by plugin EPs to register definitions for supported kernels.
*/
struct KernelRegistry : detail::Base<OrtKernelRegistry> {
///< Wrapper around OrtEpApi::CreateKernelRegistry
KernelRegistry();
///< Create an empty object, must be assigned a valid one to be used
explicit KernelRegistry(std::nullptr_t) {}
///< Take ownership of a pointer created with the C API.
explicit KernelRegistry(OrtKernelRegistry* ort_kernel_registry);
///< Wraps KernelRegistry_AddKernel
Status AddKernel(const OrtKernelDef* kernel_def, OrtKernelCreateFunc kernel_create_func,
void* kernel_create_func_state);
};
namespace detail {
template <typename T>
struct SharedPrePackedWeightCacheImpl : Ort::detail::Base<T> {
using B = Ort::detail::Base<T>;
using B::B;
//< Wraps SharedPrePackedWeightCache_StoreWeightData
Status StoreWeightData(void** buffer_data_ptrs, size_t* buffer_sizes, size_t num_buffers);
};
} // namespace detail
/** \brief Convenience C++ wrapper class around a ::OrtSharedPrePackedWeightCache instance owned by ORT.
*
* An `OrtSharedPrePackedWeightCache*` instance is passed as an argument to OrtKernelImpl::PrePackWeight.
* Example use:
* OrtStatus* MyKernel::PrePackWeightImpl(OrtKernelImpl*, ..., OrtSharedPrePackedWeightCache* c_cache, ...) {
* ...
* if (c_cache != nullptr) {
* Ort::UnownedSharedPrePackedWeightCache cpp_cache(c_cache);
* Ort::Status status = cpp_cache.StoreWeightData(...);
* }
* ...
* }
*
* \remarks OrtSharedPrePackedWeightCache is always unowned, but mutable, for EpApi users.
*/
using UnownedSharedPrePackedWeightCache =
detail::SharedPrePackedWeightCacheImpl<Ort::detail::Unowned<OrtSharedPrePackedWeightCache>>;
///< Wraps OrtEpApi::GetEnvConfigEntries()
Ort::KeyValuePairs GetEnvConfigEntries();
} // namespace Ort
#include "onnxruntime_cxx_inline.h"
+376 -20
View File
@@ -12,6 +12,7 @@
#include <iterator>
#include <string>
#include <type_traits>
#include <utility>
#include <vector>
// Convert OrtStatus to Ort::Status and return
@@ -669,9 +670,12 @@ inline void KeyValuePairs::Remove(const char* key) {
GetApi().RemoveKeyValuePair(this->p_, key);
}
inline void* SyncStream::GetHandle() const {
namespace detail {
template <typename T>
inline void* SyncStreamImpl<T>::GetHandle() {
return GetApi().SyncStream_GetHandle(this->p_);
}
} // namespace detail
namespace detail {
template <typename T>
@@ -743,6 +747,61 @@ inline EpDevice::EpDevice(OrtEpFactory& ep_factory, ConstHardwareDevice& hardwar
ThrowOnError(GetEpApi().CreateEpDevice(&ep_factory, hardware_device, ep_metadata, ep_options, &p_));
}
namespace detail {
template <typename T>
inline std::string EpAssignedSubgraphImpl<T>::GetEpName() const {
const char* ep_name = nullptr;
// Returned null-terminated string will not be null if API function returns successfully.
ThrowOnError(GetApi().EpAssignedSubgraph_GetEpName(this->p_, &ep_name));
return std::string(ep_name);
}
template <typename T>
inline std::vector<ConstEpAssignedNode> EpAssignedSubgraphImpl<T>::GetNodes() const {
size_t num_ep_nodes = 0;
const OrtEpAssignedNode* const* ep_node_ptrs = nullptr;
ThrowOnError(GetApi().EpAssignedSubgraph_GetNodes(this->p_, &ep_node_ptrs, &num_ep_nodes));
std::vector<ConstEpAssignedNode> ep_nodes;
if (num_ep_nodes > 0) {
ep_nodes.reserve(num_ep_nodes);
for (size_t i = 0; i < num_ep_nodes; ++i) {
ep_nodes.emplace_back(ep_node_ptrs[i]);
}
}
return ep_nodes;
}
template <typename T>
inline std::string EpAssignedNodeImpl<T>::GetName() const {
const char* node_name = nullptr;
// Returned null-terminated string will not be null if API function returns successfully.
ThrowOnError(GetApi().EpAssignedNode_GetName(this->p_, &node_name));
return std::string(node_name);
}
template <typename T>
inline std::string EpAssignedNodeImpl<T>::GetDomain() const {
const char* domain = nullptr;
// Returned null-terminated string will not be null if API function returns successfully.
ThrowOnError(GetApi().EpAssignedNode_GetDomain(this->p_, &domain));
return std::string(domain);
}
template <typename T>
inline std::string EpAssignedNodeImpl<T>::GetOperatorType() const {
const char* op_type = nullptr;
// Returned null-terminated string will not be null if API function returns successfully.
ThrowOnError(GetApi().EpAssignedNode_GetOperatorType(this->p_, &op_type));
return std::string(op_type);
}
} // namespace detail
inline Env::Env(OrtLoggingLevel logging_level, _In_ const char* logid) {
ThrowOnError(GetApi().CreateEnv(logging_level, logid, &p_));
if (strcmp(logid, "onnxruntime-node") == 0) {
@@ -780,6 +839,15 @@ inline Env::Env(const OrtThreadingOptions* tp_options, OrtLoggingFunction loggin
}
}
inline Env::Env(const OrtEnvCreationOptions* options) {
ThrowOnError(GetApi().CreateEnvWithOptions(options, &p_));
if (strcmp(options->log_id, "onnxruntime-node") == 0) {
ThrowOnError(GetApi().SetLanguageProjection(p_, OrtLanguageProjection::ORT_PROJECTION_NODEJS));
} else {
ThrowOnError(GetApi().SetLanguageProjection(p_, OrtLanguageProjection::ORT_PROJECTION_CPLUSPLUS));
}
}
inline Env& Env::EnableTelemetryEvents() {
ThrowOnError(GetApi().EnableTelemetryEvents(p_));
return *this;
@@ -915,6 +983,20 @@ inline OrtCompiledModelCompatibility GetModelCompatibilityForEpDevices(
return status;
}
inline AllocatedStringPtr GetCompatibilityInfoFromModelAllocated(const ORTCHAR_T* model_path, const char* ep_type,
OrtAllocator* allocator) {
char* compat_info = nullptr;
ThrowOnError(GetApi().GetCompatibilityInfoFromModel(model_path, ep_type, allocator, &compat_info));
return AllocatedStringPtr(compat_info, detail::AllocatedFree(allocator));
}
inline AllocatedStringPtr GetCompatibilityInfoFromModelBytesAllocated(const void* model_data, size_t model_data_length,
const char* ep_type, OrtAllocator* allocator) {
char* compat_info = nullptr;
ThrowOnError(GetApi().GetCompatibilityInfoFromModelBytes(model_data, model_data_length, ep_type, allocator, &compat_info));
return AllocatedStringPtr(compat_info, detail::AllocatedFree(allocator));
}
inline LoraAdapter LoraAdapter::CreateLoraAdapter(const std::basic_string<ORTCHAR_T>& adapter_path,
OrtAllocator* allocator) {
OrtLoraAdapter* p;
@@ -990,6 +1072,11 @@ inline RunOptions& RunOptions::AddActiveLoraAdapter(const LoraAdapter& adapter)
return *this;
}
inline RunOptions& RunOptions::SetSyncStream(OrtSyncStream* stream) {
GetApi().RunOptionsSetSyncStream(p_, stream);
return *this;
}
inline ModelCompilationOptions::ModelCompilationOptions(const Env& env, const SessionOptions& session_options) {
ThrowOnError(GetCompileApi().CreateModelCompilationOptionsFromSessionOptions(env, session_options, &this->p_));
}
@@ -1500,6 +1587,18 @@ inline SessionOptionsImpl<T>& SessionOptionsImpl<T>::RegisterCustomOpsUsingFunct
return *this;
}
template <typename T>
inline SessionOptionsImpl<T>& SessionOptionsImpl<T>::AddFreeDimensionOverride(const char* dim_denotation, int64_t dim_value) {
ThrowOnError(GetApi().AddFreeDimensionOverrideByName(this->p_, dim_denotation, dim_value));
return *this;
}
template <typename T>
inline SessionOptionsImpl<T>& SessionOptionsImpl<T>::AddFreeDimensionOverrideByName(const char* dim_name, int64_t dim_value) {
ThrowOnError(GetApi().AddFreeDimensionOverrideByName(this->p_, dim_name, dim_value));
return *this;
}
/// Session
template <typename T>
inline size_t ConstSessionImpl<T>::GetInputCount() const {
@@ -1582,11 +1681,13 @@ inline std::vector<ConstMemoryInfo> ConstSessionImpl<T>::GetMemoryInfoForInputs(
auto num_inputs = GetInputCount();
std::vector<ConstMemoryInfo> mem_infos;
mem_infos.resize(num_inputs);
if (num_inputs > 0) {
mem_infos.resize(num_inputs);
ThrowOnError(GetApi().SessionGetMemoryInfoForInputs(this->p_,
reinterpret_cast<const OrtMemoryInfo**>(mem_infos.data()),
num_inputs));
ThrowOnError(GetApi().SessionGetMemoryInfoForInputs(this->p_,
reinterpret_cast<const OrtMemoryInfo**>(mem_infos.data()),
num_inputs));
}
return mem_infos;
}
@@ -1598,11 +1699,13 @@ inline std::vector<ConstMemoryInfo> ConstSessionImpl<T>::GetMemoryInfoForOutputs
auto num_outputs = GetOutputCount();
std::vector<ConstMemoryInfo> mem_infos;
mem_infos.resize(num_outputs);
if (num_outputs > 0) {
mem_infos.resize(num_outputs);
ThrowOnError(GetApi().SessionGetMemoryInfoForOutputs(this->p_,
reinterpret_cast<const OrtMemoryInfo**>(mem_infos.data()),
num_outputs));
ThrowOnError(GetApi().SessionGetMemoryInfoForOutputs(this->p_,
reinterpret_cast<const OrtMemoryInfo**>(mem_infos.data()),
num_outputs));
}
return mem_infos;
}
@@ -1631,15 +1734,28 @@ template <typename T>
inline std::vector<ConstEpDevice> ConstSessionImpl<T>::GetEpDeviceForInputs() const {
auto num_inputs = GetInputCount();
std::vector<ConstEpDevice> input_devices;
input_devices.resize(num_inputs);
ThrowOnError(GetApi().SessionGetEpDeviceForInputs(this->p_,
reinterpret_cast<const OrtEpDevice**>(input_devices.data()),
num_inputs));
if (num_inputs > 0) {
input_devices.resize(num_inputs);
ThrowOnError(GetApi().SessionGetEpDeviceForInputs(this->p_,
reinterpret_cast<const OrtEpDevice**>(input_devices.data()),
num_inputs));
}
return input_devices;
}
template <typename T>
inline std::vector<ConstEpDevice> ConstSessionImpl<T>::GetEpDeviceForOutputs() const {
auto num_outputs = GetOutputCount();
std::vector<ConstEpDevice> output_devices;
if (num_outputs > 0) {
output_devices.resize(num_outputs);
ThrowOnError(GetApi().SessionGetEpDeviceForOutputs(this->p_,
reinterpret_cast<const OrtEpDevice**>(output_devices.data()),
num_outputs));
}
return output_devices;
}
template <typename T>
inline uint64_t ConstSessionImpl<T>::GetProfilingStartTimeNs() const {
uint64_t out;
@@ -1714,6 +1830,23 @@ std::vector<ValueInfo> ConstSessionImpl<T>::GetOutputs() const {
return outputs;
}
template <typename T>
inline std::vector<ConstEpAssignedSubgraph> ConstSessionImpl<T>::GetEpGraphAssignmentInfo() const {
size_t num_ep_subgraphs = 0;
const OrtEpAssignedSubgraph* const* ep_subgraph_ptrs = nullptr;
ThrowOnError(GetApi().Session_GetEpGraphAssignmentInfo(this->p_, &ep_subgraph_ptrs, &num_ep_subgraphs));
std::vector<ConstEpAssignedSubgraph> ep_subgraphs;
if (num_ep_subgraphs > 0) {
ep_subgraphs.reserve(num_ep_subgraphs);
for (size_t i = 0; i < num_ep_subgraphs; ++i) {
ep_subgraphs.emplace_back(ep_subgraph_ptrs[i]);
}
}
return ep_subgraphs;
}
template <typename T>
inline std::vector<Value> SessionImpl<T>::Run(const RunOptions& run_options, const char* const* input_names, const Value* input_values, size_t input_count,
const char* const* output_names, size_t output_count) {
@@ -1975,7 +2108,12 @@ template <typename T>
inline size_t TensorTypeAndShapeInfoImpl<T>::GetElementCount() const {
size_t out;
ThrowOnError(GetApi().GetTensorShapeElementCount(this->p_, &out));
return static_cast<size_t>(out);
return out;
}
template <typename T>
inline bool TensorTypeAndShapeInfoImpl<T>::HasShape() const {
return GetApi().TensorTypeAndShape_HasShape(this->p_);
}
template <typename T>
@@ -1997,8 +2135,12 @@ inline void TensorTypeAndShapeInfoImpl<T>::GetSymbolicDimensions(const char** va
template <typename T>
inline std::vector<const char*> TensorTypeAndShapeInfoImpl<T>::GetSymbolicDimensions() const {
std::vector<const char*> out(GetDimensionsCount(), nullptr);
ThrowOnError(GetApi().GetSymbolicDimensions(this->p_, out.data(), out.size()));
std::vector<const char*> out;
size_t dim_count = GetDimensionsCount();
if (dim_count > 0) {
out.resize(dim_count, nullptr);
ThrowOnError(GetApi().GetSymbolicDimensions(this->p_, out.data(), out.size()));
}
return out;
}
@@ -2235,6 +2377,13 @@ inline const R* ConstValueImpl<T>::GetSparseTensorValues() const {
#endif
template <typename T>
void ConstValueImpl<T>::GetTensorElementTypeAndShapeDataReference(ONNXTensorElementDataType& elem_type,
Shape& shape) const {
ThrowOnError(GetApi().GetTensorElementTypeAndShapeDataReference(this->p_, &elem_type, &shape.shape,
&shape.shape_len));
}
template <typename T>
void ValueImpl<T>::FillStringTensor(const char* const* s, size_t s_len) {
ThrowOnError(GetApi().FillStringTensor(this->p_, s, s_len));
@@ -2518,10 +2667,10 @@ inline void* KernelContext::GetGPUComputeStream() const {
return out;
}
inline OrtAllocator* KernelContext::GetAllocator(const OrtMemoryInfo& memory_info) const {
inline Ort::Allocator KernelContext::GetAllocator(const OrtMemoryInfo& memory_info) const {
OrtAllocator* out = nullptr;
Ort::ThrowOnError(GetApi().KernelContext_GetAllocator(ctx_, &memory_info, &out));
return out;
return Ort::Allocator{out};
}
inline Logger KernelContext::GetLogger() const {
@@ -2806,6 +2955,57 @@ inline Logger KernelInfoImpl<T>::GetLogger() const {
return Logger{out};
}
template <typename T>
inline KeyValuePairs KernelInfoImpl<T>::GetConfigEntries() const {
OrtKeyValuePairs* out = nullptr;
Ort::ThrowOnError(GetApi().KernelInfo_GetConfigEntries(this->p_, &out));
return KeyValuePairs{out};
}
template <typename T>
inline std::string KernelInfoImpl<T>::GetOperatorDomain() const {
size_t size = 0;
// Feed nullptr for the data buffer to query the true size of the string value
Ort::ThrowOnError(GetApi().KernelInfo_GetOperatorDomain(this->p_, nullptr, &size));
std::string out;
out.resize(size);
Ort::ThrowOnError(GetApi().KernelInfo_GetOperatorDomain(this->p_, &out[0], &size));
out.resize(size - 1); // remove the terminating character '\0'
return out;
}
template <typename T>
inline std::string KernelInfoImpl<T>::GetOperatorType() const {
size_t size = 0;
// Feed nullptr for the data buffer to query the true size of the string value
Ort::ThrowOnError(GetApi().KernelInfo_GetOperatorType(this->p_, nullptr, &size));
std::string out;
out.resize(size);
Ort::ThrowOnError(GetApi().KernelInfo_GetOperatorType(this->p_, &out[0], &size));
out.resize(size - 1); // remove the terminating character '\0'
return out;
}
template <typename T>
inline int KernelInfoImpl<T>::GetOperatorSinceVersion() const {
int out = 0;
Ort::ThrowOnError(GetApi().KernelInfo_GetOperatorSinceVersion(this->p_, &out));
return out;
}
template <typename T>
inline const OrtEp* KernelInfoImpl<T>::GetEp() const {
const OrtEp* ep = nullptr;
Ort::ThrowOnError(GetEpApi().KernelInfo_GetEp(this->p_, &ep));
return ep;
}
inline void attr_utils::GetAttr(const OrtKernelInfo* p, const char* name, float& out) {
Ort::ThrowOnError(GetApi().KernelInfoGetAttribute_float(p, name, &out));
}
@@ -3537,4 +3737,160 @@ inline Model::Model(const std::vector<DomainOpsetPair>& opsets) {
}
#endif
namespace detail {
template <typename T>
inline const char* ConstKernelDefImpl<T>::GetOperatorType() const {
return GetEpApi().KernelDef_GetOperatorType(this->p_);
}
template <typename T>
inline const char* ConstKernelDefImpl<T>::GetDomain() const {
return GetEpApi().KernelDef_GetDomain(this->p_);
}
template <typename T>
inline std::pair<int, int> ConstKernelDefImpl<T>::GetSinceVersion() const {
int start = 0;
int end = 0;
ThrowOnError(GetEpApi().KernelDef_GetSinceVersion(this->p_, &start, &end));
return std::pair<int, int>(start, end);
}
template <typename T>
inline const char* ConstKernelDefImpl<T>::GetExecutionProvider() const {
return GetEpApi().KernelDef_GetExecutionProvider(this->p_);
}
template <typename T>
inline OrtMemType ConstKernelDefImpl<T>::GetInputMemType(size_t input_index) const {
OrtMemType mem_type{};
ThrowOnError(GetEpApi().KernelDef_GetInputMemType(this->p_, input_index, &mem_type));
return mem_type;
}
template <typename T>
inline OrtMemType ConstKernelDefImpl<T>::GetOutputMemType(size_t output_index) const {
OrtMemType mem_type{};
ThrowOnError(GetEpApi().KernelDef_GetOutputMemType(this->p_, output_index, &mem_type));
return mem_type;
}
} // namespace detail
inline KernelDefBuilder::KernelDefBuilder() {
ThrowOnError(GetEpApi().CreateKernelDefBuilder(&p_));
}
inline KernelDefBuilder::KernelDefBuilder(OrtKernelDefBuilder* p) : detail::Base<OrtKernelDefBuilder>{p} {
}
inline KernelDefBuilder& KernelDefBuilder::SetOperatorType(const char* op_type) {
ThrowOnError(GetEpApi().KernelDefBuilder_SetOperatorType(p_, op_type));
return *this;
}
inline KernelDefBuilder& KernelDefBuilder::SetDomain(const char* domain) {
ThrowOnError(GetEpApi().KernelDefBuilder_SetDomain(p_, domain));
return *this;
}
inline KernelDefBuilder& KernelDefBuilder::SetSinceVersion(int since_version_start, int since_version_end) {
ThrowOnError(GetEpApi().KernelDefBuilder_SetSinceVersion(p_, since_version_start, since_version_end));
return *this;
}
inline KernelDefBuilder& KernelDefBuilder::SetExecutionProvider(const char* ep_name) {
ThrowOnError(GetEpApi().KernelDefBuilder_SetExecutionProvider(p_, ep_name));
return *this;
}
inline KernelDefBuilder& KernelDefBuilder::SetInputMemType(size_t input_index, OrtMemType mem_type) {
ThrowOnError(GetEpApi().KernelDefBuilder_SetInputMemType(p_, input_index, mem_type));
return *this;
}
inline KernelDefBuilder& KernelDefBuilder::SetOutputMemType(size_t output_index, OrtMemType mem_type) {
ThrowOnError(GetEpApi().KernelDefBuilder_SetOutputMemType(p_, output_index, mem_type));
return *this;
}
inline KernelDefBuilder& KernelDefBuilder::AddTypeConstraint(const char* arg_name,
const OrtDataType* data_type) {
ThrowOnError(GetEpApi().KernelDefBuilder_AddTypeConstraint(p_, arg_name, &data_type, 1));
return *this;
}
inline KernelDefBuilder& KernelDefBuilder::AddTypeConstraint(const char* arg_name,
const std::vector<const OrtDataType*>& data_types) {
ThrowOnError(GetEpApi().KernelDefBuilder_AddTypeConstraint(p_, arg_name, data_types.data(), data_types.size()));
return *this;
}
inline KernelDefBuilder& KernelDefBuilder::AddInputOutputAlias(int input_index, int output_index) {
ThrowOnError(GetEpApi().KernelDefBuilder_AddInputOutputAliases(p_, &input_index, &output_index, 1));
return *this;
}
inline KernelDefBuilder& KernelDefBuilder::AddInputOutputAliases(const std::vector<int>& input_indices,
const std::vector<int>& output_indices) {
if (input_indices.size() != output_indices.size()) {
ORT_CXX_API_THROW("Expecting input and output indices to have the same element count", ORT_INVALID_ARGUMENT);
}
ThrowOnError(GetEpApi().KernelDefBuilder_AddInputOutputAliases(p_, input_indices.data(), output_indices.data(),
input_indices.size()));
return *this;
}
inline KernelDefBuilder& KernelDefBuilder::AddInputOutputMutableAlias(int input_index, int output_index) {
ThrowOnError(GetEpApi().KernelDefBuilder_AddInputOutputMutableAliases(p_, &input_index, &output_index, 1));
return *this;
}
inline KernelDefBuilder& KernelDefBuilder::AddInputOutputMutableAliases(const std::vector<int>& input_indices,
const std::vector<int>& output_indices) {
if (input_indices.size() != output_indices.size()) {
ORT_CXX_API_THROW("Expecting input and output indices to have the same element count", ORT_INVALID_ARGUMENT);
}
ThrowOnError(GetEpApi().KernelDefBuilder_AddInputOutputMutableAliases(p_, input_indices.data(), output_indices.data(),
input_indices.size()));
return *this;
}
inline KernelDef KernelDefBuilder::Build() {
OrtKernelDef* kernel_def = nullptr;
ThrowOnError(GetEpApi().KernelDefBuilder_Build(p_, &kernel_def));
return KernelDef(kernel_def);
}
inline KernelRegistry::KernelRegistry() {
ThrowOnError(GetEpApi().CreateKernelRegistry(&p_));
}
inline KernelRegistry::KernelRegistry(OrtKernelRegistry* p) : detail::Base<OrtKernelRegistry>{p} {
}
inline Status KernelRegistry::AddKernel(const OrtKernelDef* kernel_def, OrtKernelCreateFunc kernel_create_func,
void* kernel_create_func_state) {
return Status{GetEpApi().KernelRegistry_AddKernel(p_, kernel_def, kernel_create_func, kernel_create_func_state)};
}
namespace detail {
template <typename T>
inline Status SharedPrePackedWeightCacheImpl<T>::StoreWeightData(void** buffer_data_ptrs, size_t* buffer_sizes,
size_t num_buffers) {
return Status{GetEpApi().SharedPrePackedWeightCache_StoreWeightData(this->p_, buffer_data_ptrs, buffer_sizes,
num_buffers)};
}
} // namespace detail
inline Ort::KeyValuePairs GetEnvConfigEntries() {
OrtKeyValuePairs* entries = nullptr;
Ort::ThrowOnError(GetEpApi().GetEnvConfigEntries(&entries));
return Ort::KeyValuePairs{entries};
}
} // namespace Ort
File diff suppressed because it is too large Load Diff
@@ -3,12 +3,15 @@
#pragma once
// This file contains well-known keys for OrtEpDevice EP metadata entries.
// This file contains well-known keys for OrtEpDevice and OrtHardwareDevice metadata entries.
// It does NOT specify all available metadata keys.
// Key for the execution provider version string. This should be available for all plugin EPs.
static const char* const kOrtEpDevice_EpMetadataKey_Version = "version";
// Key for the execution provider OS driver version.
static const char* const kOrtEpDevice_EpMetadataKey_OSDriverVersion = "os_driver_version";
// Prefix for execution provider compatibility information stored in model metadata.
// Used when generating EP context models to store compatibility strings for each EP.
// Full key format: "ep_compatibility_info.<EP_TYPE>"
@@ -16,3 +19,10 @@ static const char* const kOrtModelMetadata_EpCompatibilityInfoPrefix = "ep_compa
// Key for the execution provider library path (for dynamically loaded EPs)
static const char* const kOrtEpDevice_EpMetadataKey_LibraryPath = "library_path";
// Optional metadata key to determine if a OrtHardwareDevice represents a virtual (non-hardware) device.
// Possible values:
// - "0": OrtHardwareDevice is not virtual (i.e., actual hardware device). This is the assumed default value
// if this metadata key is not present.
// - "1": OrtHardwareDevice is virtual.
static const char* const kOrtHardwareDevice_MetadataKey_IsVirtual = "is_virtual";
@@ -76,6 +76,7 @@ struct Float16Impl {
static constexpr uint16_t kPositiveQNaNBits = 0x7E00U;
static constexpr uint16_t kNegativeQNaNBits = 0xFE00U;
static constexpr uint16_t kMaxValueBits = 0x7BFFU; // Largest normal number
static constexpr uint16_t kMinValueBits = 0xFBFFU; // Lowest normal number
static constexpr uint16_t kOneBits = 0x3C00U;
static constexpr uint16_t kMinusOneBits = 0xBC00U;
@@ -363,6 +364,7 @@ struct BFloat16Impl {
static constexpr uint16_t kPositiveQNaNBits = 0x7FC1U;
static constexpr uint16_t kNegativeQNaNBits = 0xFFC1U;
static constexpr uint16_t kMaxValueBits = 0x7F7FU;
static constexpr uint16_t kMinValueBits = 0xFF7FU;
static constexpr uint16_t kRoundToNearest = 0x7FFFU;
static constexpr uint16_t kOneBits = 0x3F80U;
static constexpr uint16_t kMinusOneBits = 0xBF80U;
@@ -447,18 +447,6 @@ struct OrtLiteCustomOp : public OrtCustomOp {
}
#endif
#ifdef ORT_ROCM_CTX
template <size_t ith_input, size_t ith_output, typename T, typename... Ts>
static typename std::enable_if<std::is_same<T, const RocmContext&>::value, std::tuple<T, Ts...>>::type
CreateTuple(OrtKernelContext* context, ArgPtrs& args, size_t num_input, size_t num_output, const std::string& ep) {
thread_local RocmContext rocm_context;
rocm_context.Init(*context);
std::tuple<T> current = std::tuple<const RocmContext&>{rocm_context};
auto next = CreateTuple<ith_input, ith_output, Ts...>(context, args, num_input, num_output, ep);
return std::tuple_cat(current, next);
}
#endif
template <size_t ith_input, size_t ith_output, typename T, typename... Ts>
static typename std::enable_if<std::is_same<T, const TensorArray*>::value, std::tuple<T, Ts...>>::type
CreateTuple(OrtKernelContext* context, ArgPtrs& args, size_t num_input, size_t num_output, const std::string& ep) {
@@ -674,14 +662,6 @@ struct OrtLiteCustomOp : public OrtCustomOp {
}
#endif
#ifdef ORT_ROCM_CTX
template <typename T, typename... Ts>
static typename std::enable_if<0 <= sizeof...(Ts) && std::is_same<T, const RocmContext&>::value>::type
ParseArgs(std::vector<ONNXTensorElementDataType>& input_types, std::vector<ONNXTensorElementDataType>& output_types) {
ParseArgs<Ts...>(input_types, output_types);
}
#endif
template <typename T, typename... Ts>
static typename std::enable_if<0 <= sizeof...(Ts) && std::is_same<T, const TensorArray&>::value>::type
ParseArgs(std::vector<ONNXTensorElementDataType>& input_types, std::vector<ONNXTensorElementDataType>& output_types) {
@@ -374,6 +374,12 @@ static const char* const kOrtSessionOptionsEpContextModelExternalInitializersFil
// - "1": Gemm FastMath mode is enabled.
static const char* const kOrtSessionOptionsMlasGemmFastMathArm64Bfloat16 = "mlas.enable_gemm_fastmath_arm64_bfloat16";
// Use LUT (Lookup Table) based GEMM for quantized models when available.
// Option values:
// - "0": Do not use LUT based GEMM. [DEFAULT]
// - "1": Use LUT based GEMM when available.
static const char* const kOrtSessionOptionsMlasLutGemm = "mlas.use_lut_gemm";
// When converting DQ + MatMul -> MatMulNBits, the accuracy level of the MatMulNBits is controlled by this option.
// Refer to MatMulNBits op schema for more details.
// If not provided, default is 4.
@@ -408,3 +414,18 @@ static const char* const kOrtSessionOptionsDisableModelCompile = "session.disabl
// Note: UNSUPPORTED models always fail regardless of this setting.
static const char* const kOrtSessionOptionsFailOnSuboptimalCompiledModel =
"session.fail_on_suboptimal_compiled_model";
// THIS OPTION IS NOT A REGULAR SESSION OPTION SINCE IT CAN BE MODIFIED AT ANY TIME
// Meant to be used with SetEpDynamicOptions
// options for HTP performance mode: "burst", "balanced", "default", "high_performance",
// "high_power_saver", "low_balanced", "extreme_power_saver", "low_power_saver", "power_saver",
// "sustained_high_performance". Default to "default".
static const char* const kOrtEpDynamicOptionsQnnHtpPerformanceMode = "ep.dynamic.qnn_htp_performance_mode";
// Enables the session to record information about the subgraphs/nodes assigned to execution providers.
// When enabled, an application may call Session_GetEpGraphAssignmentInfo() to retrieve the information.
//
// Option values:
// - "0": Recording of EP graph assignment information is disabled. [DEFAULT]
// - "1": Recording of EP graph assignment information is enabled.
static const char* const kOrtSessionOptionsRecordEpGraphAssignmentInfo = "session.record_ep_graph_assignment_info";
@@ -1,731 +0,0 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
// This file contains the training c apis.
#pragma once
#include <stdbool.h>
#include "onnxruntime_c_api.h"
/** \page training_c_cpp_api Training C & C++ APIs
*
* Training C and C++ APIs are an extension of the \ref c_cpp_api "onnxruntime core C and C++ APIs" and should be used in conjunction with them.
*
* In order to train a model with onnxruntime, the following training artifacts must be generated:
* - The training onnx model
* - The checkpoint file
* - The optimizer onnx model
* - The eval onnx model model (optional)
*
* These training artifacts can be generated as part of an offline step using the python [utilities](https://github.com/microsoft/onnxruntime/blob/main/orttraining/orttraining/python/training/onnxblock/README.md) made available in the `onnxruntime-training` python package.
*
* After these artifacts have been generated, the C and C++ utilities listed in this documentation can be leveraged to perform training.
*
* If any problem is encountered, please create an [issue](https://github.com/microsoft/onnxruntime/issues/new) with your scenario and requirements, and we will be sure to respond and follow up on the request.
*
* <h1>Training C API</h1>
*
* ::OrtTrainingApi - Training C API functions.
*
* This C structure contains functions that enable users to perform training with onnxruntime.
*
* _Sample Code_:
*
* ```c
* #include <onnxruntime_training_api.h>
*
* OrtApi* g_ort_api = OrtGetApiBase()->GetApi(ORT_API_VERSION);
* OrtTrainingApi* g_ort_training_api = g_ort_api->GetTrainingApi(ORT_API_VERSION);
*
* OrtEnv* env = NULL;
* g_ort_api->CreateEnv(logging_level, logid, &env);
* OrtSessionOptions* session_options = NULL;
* g_ort_api->CreateSessionOptions(&session_options);
*
* OrtCheckpointState* state = NULL;
* g_ort_training_api->LoadCheckpoint(path_to_checkpoint, &state);
*
* OrtTrainingSession* training_session = NULL;
* g_ort_training_api->CreateTrainingSession(env, session_options, training_model_path,
* state, eval_model_path, optimizer_model_path,
* &training_session);
* // Training loop
* {
* g_ort_training_api->TrainStep(...);
* g_ort_training_api->OptimizerStep(...);
* g_ort_training_api->LazyResetGrad(...);
* }
*
* g_ort_training_api->ExportModelForInferencing(training_session, inference_model_path, ...);
* g_ort_training_api->SaveCheckpoint(state, path_to_checkpoint, false);
*
* g_ort_training_api->ReleaseTrainingSession(training_session);
* g_ort_training_api->ReleaseCheckpointState(state);
* ```
*
* > **Note**
* > The ::OrtCheckpointState contains the entire training state that the ::OrtTrainingSession uses. As a result, the training session must always have access to the state. That is to say, the ::OrtCheckpointState instance must outlive the lifetime of the ::OrtTrainingSession instance.
*
* <h1>Training C++ API</h1>
*
* @ref TrainingCpp - Training C++ API classes and functions.
*
* These C++ classes and functions enable users to perform training with onnxruntime.
*
* _Sample Code_:
*
* ```cc
* #include <onnxruntime_training_cxx_api.h>
*
* Ort::Env env;
* Ort::SessionOptions session_options;
*
* auto state = Ort::CheckpointState::LoadCheckpoint(path_to_checkpoint);
* auto training_session = Ort::TrainingSession(env, session_options, state, training_model_path,
* eval_model_path, optimizer_model_path);
*
* // Training Loop
* {
* training_session.TrainStep(...);
* training_session.OptimizerStep(...);
* training_session.LazyResetGrad(...);
* }
*
* training_session->ExportModelForInferencing(inference_model_path, ...);
* Ort::CheckpointState::SaveCheckpoint(state, path_to_checkpoint, false);
* ```
* > **Note**
* > The ::Ort::CheckpointState contains the entire training state that the ::Ort::TrainingSession uses. As a result, the training session must always have access to the state. That is to say, the ::Ort::CheckpointState instance must outlive the lifetime of the ::Ort::TrainingSession instance.
*/
/** @defgroup TrainingC Ort Training C API
* @{
*/
ORT_RUNTIME_CLASS(TrainingSession); // Type that enables performing training for the given user models.
ORT_RUNTIME_CLASS(CheckpointState); // Type that holds the training states for the training session.
/** \brief Type of property to be added to or returned from the ::OrtCheckpointState.
*/
typedef enum OrtPropertyType {
OrtIntProperty = 0,
OrtFloatProperty = 1,
OrtStringProperty = 2,
} OrtPropertyType;
/** \brief The Training C API that holds onnxruntime training function pointers
*
* All the Training C API functions are defined inside this structure as pointers to functions.
* Call OrtApi::GetTrainingApi to get a pointer to this struct.
*
* \nosubgrouping
*/
struct OrtTrainingApi {
/// \name Accessing The Training Session State
/// @{
/** \brief Load a checkpoint state from a file on disk into checkpoint_state.
*
* This function will parse a checkpoint file, pull relevant data and load the training
* state into the checkpoint_state. This checkpoint state can then be used to create the
* training session by invoking OrtTrainingApi::CreateTrainingSession. By doing so, the training
* session will resume training from the given checkpoint state.
* \note Note that the training session created with a checkpoint state uses this state to store the entire
* training state (including model parameters, its gradients, the optimizer states and the properties).
* As a result, it is required that the checkpoint state outlive the lifetime of the training session.
* \note Note that the checkpoint file can be either the complete checkpoint or the nominal checkpoint.
*
* \param[in] checkpoint_path Path to the checkpoint file
* \param[out] checkpoint_state Checkpoint state that contains the states of the training session.
*
* \snippet{doc} snippets.dox OrtStatus Return Value
*
*/
ORT_API2_STATUS(LoadCheckpoint, _In_ const ORTCHAR_T* checkpoint_path,
_Outptr_ OrtCheckpointState** checkpoint_state);
/** \brief Save the given state to a checkpoint file on disk.
*
* This function serializes the provided checkpoint state to a file on disk.
* This checkpoint can later be loaded by invoking OrtTrainingApi::LoadCheckpoint to resume
* training from this snapshot of the state.
*
* \param[in] checkpoint_state The checkpoint state to save.
* \param[in] checkpoint_path Path to the checkpoint file.
* \param[in] include_optimizer_state Flag to indicate whether to save the optimizer state or not.
*
* \snippet{doc} snippets.dox OrtStatus Return Value
*
*/
ORT_API2_STATUS(SaveCheckpoint, _In_ OrtCheckpointState* checkpoint_state, _In_ const ORTCHAR_T* checkpoint_path,
const bool include_optimizer_state);
/// @}
/// \name Implementing The Training Loop
/// @{
/** \brief Create a training session that can be used to begin or resume training.
*
* This function creates a training session based on the env and session options provided that can
* begin or resume training from a given checkpoint state for the given onnx models.
* The checkpoint state represents the parameters of the training session which will be moved
* to the device specified by the user through the session options (if necessary).
* The training session requires four training artifacts
* - The training onnx model
* - The evaluation onnx model (optional)
* - The optimizer onnx model
* - The checkpoint file
*
* These artifacts can be generated using the `onnxruntime-training` python [utility](https://github.com/microsoft/onnxruntime/blob/main/orttraining/orttraining/python/training/onnxblock/README.md).
*
* \param[in] env Environment to be used for the training session.
* \param[in] options Session options that the user can customize for this training session.
* \param[in] checkpoint_state Training states that the training session uses as a starting point for training.
* \param[in] train_model_path Model to be used to perform training.
* \param[in] eval_model_path Model to be used to perform evaluation.
* \param[in] optimizer_model_path Model to be used to perform gradient descent.
* \param[out] out Created training session.
*
* \snippet{doc} snippets.dox OrtStatus Return Value
*
*/
ORT_API2_STATUS(CreateTrainingSession, _In_ const OrtEnv* env, _In_ const OrtSessionOptions* options,
_Inout_ OrtCheckpointState* checkpoint_state, _In_ const ORTCHAR_T* train_model_path,
_In_ const ORTCHAR_T* eval_model_path, _In_ const ORTCHAR_T* optimizer_model_path,
_Outptr_result_maybenull_ OrtTrainingSession** out);
/** \brief Create a training session that can be used to begin or resume training.
* This api provides a way to load all the training artifacts from buffers instead of files.
*
* \param[in] env Environment to be used for the training session.
* \param[in] options Session options that the user can customize for this training session.
* \param[in] checkpoint_state Training states that the training session uses as a starting point for training.
* \param[in] train_model_data Buffer containing the model data to be used to perform training
* \param[in] train_data_length Length of the buffer containing train_model_data
* \param[in] eval_model_data Buffer containing the model data to be used to perform evaluation
* \param[in] eval_data_length Length of the buffer containing eval_model_data
* \param[in] optim_model_data Buffer containing the model data to be used to perform weight update
* \param[in] optim_data_length Length of the buffer containing optim_model_data
* \param[out] out Created training session.
*
*/
ORT_API2_STATUS(CreateTrainingSessionFromBuffer, _In_ const OrtEnv* env,
_In_ const OrtSessionOptions* options, _Inout_ OrtCheckpointState* checkpoint_state,
_In_ const void* train_model_data, size_t train_data_length,
_In_ const void* eval_model_data, size_t eval_data_length,
_In_ const void* optim_model_data, size_t optim_data_length,
_Outptr_result_maybenull_ OrtTrainingSession** out);
/// @}
/// \name Model IO Information
/// @{
/** \brief Retrieves the number of user outputs in the training model.
*
* This function returns the number of outputs of the training model so that the user can
* allocate space for the number of outputs when OrtTrainingApi::TrainStep is invoked.
*
* \param[in] sess The `this` pointer to the training session.
* \param[out] out Number of user outputs in the training model.
*
* \snippet{doc} snippets.dox OrtStatus Return Value
*
*/
ORT_API2_STATUS(TrainingSessionGetTrainingModelOutputCount, _In_ const OrtTrainingSession* sess, _Out_ size_t* out);
/** \brief Retrieves the number of user outputs in the eval model.
*
* This function returns the number of outputs of the eval model so that the user can
* allocate space for the number of outputs when OrtTrainingApi::EvalStep is invoked.
*
* \param[in] sess The `this` pointer to the training session.
* \param[out] out Number of user outputs in the eval model.
*
* \snippet{doc} snippets.dox OrtStatus Return Value
*
*/
ORT_API2_STATUS(TrainingSessionGetEvalModelOutputCount, _In_ const OrtTrainingSession* sess, _Out_ size_t* out);
/** \brief Retrieves the names of user outputs in the training model.
*
* This function returns the names of outputs of the training model that can be associated with the OrtValue(s)
* returned by the OrtTrainingApi::TrainStep function.
*
* \param[in] sess The `this` pointer to the training session.
* \param[in] index Index of the output name requested.
* \param[in] allocator Allocator to use to allocate the memory for the name.
* \param[out] output Name of the training model output at the given index.
*
* \snippet{doc} snippets.dox OrtStatus Return Value
*
*/
ORT_API2_STATUS(TrainingSessionGetTrainingModelOutputName, _In_ const OrtTrainingSession* sess, size_t index, _Inout_ OrtAllocator* allocator, _Outptr_ char** output);
/** \brief Retrieves the names of user outputs in the eval model.
*
* This function returns the names of outputs of the eval model that can be associated with the OrtValue(s) returned
* by the OrtTrainingApi::EvalStep function.
*
* \param[in] sess The `this` pointer to the training session.
* \param[in] index Index of the output name requested.
* \param[in] allocator Allocator to use to allocate the memory for the name.
* \param[out] output Name of the eval model output at the given index.
*
* \snippet{doc} snippets.dox OrtStatus Return Value
*
*/
ORT_API2_STATUS(TrainingSessionGetEvalModelOutputName, _In_ const OrtTrainingSession* sess, size_t index, _Inout_ OrtAllocator* allocator, _Outptr_ char** output);
/// @}
/// \name Implementing The Training Loop
/// @{
/** \brief Reset the gradients of all trainable parameters to zero lazily.
*
* This function sets the internal state of the training session such that the gradients of the trainable
* parameters in the OrtCheckpointState will be scheduled to be reset just before the new gradients are
* computed on the next invocation of the next OrtTrainingApi::TrainStep.
*
* \param[in] session The `this` pointer to the training session.
*
* \snippet{doc} snippets.dox OrtStatus Return Value
*
*/
ORT_API2_STATUS(LazyResetGrad, _Inout_ OrtTrainingSession* session);
/** \brief Computes the outputs of the training model and the gradients of the trainable parameters for the given inputs
*
* This function performs a training step that computes the outputs of the training model and the gradients
* of the trainable parameters for the given inputs. The train step is performed based on the training model
* that was provided to the training session.
* The OrtTrainingApi::TrainStep is equivalent of running forward propagation and backward propagation in a single
* step.
* The gradients computed are stored inside the training session state so they can be later consumed
* by the OrtTrainingApi::OptimizerStep function.
* The gradients can be lazily reset by invoking the OrtTrainingApi::LazyResetGrad function.
*
* \param[in] sess The `this` pointer to the training session.
* \param[in] run_options Run options for this training step.
* \param[in] inputs_len Number of user inputs to the training model.
* \param[in] inputs The user inputs to the training model.
* \param[in] outputs_len Number of user outputs expected from this training step.
* \param[out] outputs User outputs computed by train step.
*
* \snippet{doc} snippets.dox OrtStatus Return Value
*
*/
ORT_API2_STATUS(TrainStep, _Inout_ OrtTrainingSession* sess, _In_opt_ const OrtRunOptions* run_options,
_In_ size_t inputs_len, _In_reads_(inputs_len) const OrtValue* const* inputs,
_In_ size_t outputs_len, _Inout_updates_all_(outputs_len) OrtValue** outputs);
/** \brief Computes the outputs for the eval model for the given inputs
*
* This function performs an eval step that computes the outputs of the eval model for the given inputs.
* The eval step is performed based on the eval model that was provided to the training session.
*
* \param[in] sess The `this` pointer to the training session.
* \param[in] run_options Run options for this eval step.
* \param[in] inputs_len Number of user inputs to the eval model.
* \param[in] inputs The user inputs to the eval model.
* \param[in] outputs_len Number of user outputs expected from this eval step.
* \param[out] outputs User outputs computed by eval step.
*
* \snippet{doc} snippets.dox OrtStatus Return Value
*
*/
ORT_API2_STATUS(EvalStep, _In_ const OrtTrainingSession* sess, _In_opt_ const OrtRunOptions* run_options,
_In_ size_t inputs_len, _In_reads_(inputs_len) const OrtValue* const* inputs,
_In_ size_t outputs_len, _Inout_updates_all_(outputs_len) OrtValue** outputs);
/** \brief Sets the learning rate for this training session.
*
* This function allows users to set the learning rate for the training session. The current
* learning rate is maintained by the training session and can be overwritten by invoking
* this function with the desired learning rate. This function should not be used when a valid
* learning rate scheduler is registered. It should be used either to set the learning rate
* derived from a custom learning rate scheduler or to set a constant learning rate to be used
* throughout the training session.
* \note Please note that this function does not set the initial learning rate that may be needed
* by the predefined learning rate schedulers. To set the initial learning rate for learning
* rate schedulers, please look at the function OrtTrainingApi::RegisterLinearLRScheduler.
*
* \param[in] sess The `this` pointer to the training session.
* \param[in] learning_rate Desired learning rate to be set.
*
* \snippet{doc} snippets.dox OrtStatus Return Value
*
*/
ORT_API2_STATUS(SetLearningRate, _Inout_ OrtTrainingSession* sess, _In_ float learning_rate);
/** \brief Gets the current learning rate for this training session.
*
* This function allows users to get the learning rate for the training session. The current
* learning rate is maintained by the training session, and users can query it for the purpose
* of implementing their own learning rate schedulers.
*
* \param[in] sess The `this` pointer to the training session.
* \param[out] learning_rate Learning rate currently in use by the training session.
*
* \snippet{doc} snippets.dox OrtStatus Return Value
*
*/
ORT_API2_STATUS(GetLearningRate, _Inout_ OrtTrainingSession* sess, _Out_ float* learning_rate);
/** \brief Performs the weight updates for the trainable parameters using the optimizer model.
*
* This function performs the weight update step that updates the trainable parameters such that they
* take a step in the direction of their gradients (gradient descent). The optimizer step is performed
* based on the optimizer model that was provided to the training session.
* The updated parameters are stored inside the training state so that they can be used by the next
* OrtTrainingApi::TrainStep function call.
*
* \param[in] sess The `this` pointer to the training session.
* \param[in] run_options Run options for this optimizer step.
*
* \snippet{doc} snippets.dox OrtStatus Return Value
*
*/
ORT_API2_STATUS(OptimizerStep, _Inout_ OrtTrainingSession* sess,
_In_opt_ const OrtRunOptions* run_options);
/** \brief Registers a linear learning rate scheduler for the training session.
*
* Register a linear learning rate scheduler that decays the learning rate by linearly updated
* multiplicative factor from the initial learning rate set on the training session to 0. The decay
* is performed after the initial warm up phase where the learning rate is linearly incremented
* from 0 to the initial learning rate provided.
*
* \param[in] sess The `this` pointer to the training session.
* \param[in] warmup_step_count Warmup steps for LR warmup.
* \param[in] total_step_count Total step count.
* \param[in] initial_lr The initial learning rate to be used by the training session.
*
* \snippet{doc} snippets.dox OrtStatus Return Value
*
*/
ORT_API2_STATUS(RegisterLinearLRScheduler, _Inout_ OrtTrainingSession* sess, _In_ const int64_t warmup_step_count,
_In_ const int64_t total_step_count, _In_ const float initial_lr);
/** \brief Update the learning rate based on the registered learing rate scheduler.
*
* Takes a scheduler step that updates the learning rate that is being used by the training session.
* This function should typically be called before invoking the optimizer step for each round,
* or as determined necessary to update the learning rate being used by the training session.
* \note Please note that a valid predefined learning rate scheduler must be first registered to invoke this
* function.
*
* \param[in] sess The `this` pointer to the training session.
*
* \snippet{doc} snippets.dox OrtStatus Return Value
*
*/
ORT_API2_STATUS(SchedulerStep, _Inout_ OrtTrainingSession* sess);
/// @}
/// \name Accessing The Training Session State
/// @{
/** \brief Retrieves the size of all the parameters.
*
* Calculates the total number of primitive (datatype of the parameters) elements of all the parameters in the
* training state.
* When trainable_only argument is true, the size is calculated for trainable params only.
*
* \param[in] sess The `this` pointer to the training session.
* \param[out] out Size of all parameter elements.
* \param[in] trainable_only Whether to skip non-trainable parameters
*
* \snippet{doc} snippets.dox OrtStatus Return Value
*
*/
ORT_API2_STATUS(GetParametersSize, _Inout_ OrtTrainingSession* sess, _Out_ size_t* out, bool trainable_only);
/** \brief Copy all parameters to a contiguous buffer held by the argument parameters_buffer
*
* The parameters_buffer has to be of the size given by GetParametersSize api call,
* with matching setting for the argument trainable_only. All the target parameters must be of the same
* datatype. The OrtValue must be pre-allocated onto
* the desired device. This is a complementary function to OrtTrainingApi::CopyBufferToParameters.
* Parameter ordering is preserved.
* User is responsible for allocating and freeing the resources used by the parameters_buffer.
*
* \param[in] sess The `this` pointer to the training session.
* \param[in] trainable_only Whether to skip non-trainable parameters
* \param[out] parameters_buffer The pre-allocated OrtValue buffer to copy onto.
*
* \snippet{doc} snippets.dox OrtStatus Return Value
*
*/
ORT_API2_STATUS(CopyParametersToBuffer, _Inout_ OrtTrainingSession* sess,
_Inout_ OrtValue* parameters_buffer, bool trainable_only);
/** \brief Copy parameter values from the given contiguous buffer held by parameters_buffer to the training state
*
* The parameters_buffer argument has to be of the size given by OrtTrainingApi::GetParametersSize api call,
* with matching setting for trainable_only argument. All the target parameters must be of the same
* datatype. This is a complementary function to OrtTrainingApi::CopyParametersToBuffer
* and can be used to load updated buffer values onto the training state.
* Parameter ordering is preserved.
* User is responsible for allocating and freeing the resources used by the parameters_buffer.
* In case the training session was created with a nominal checkpoint, invoking this function is required
* to load the updated parameters onto the checkpoint to complete it.
*
* \param[in] sess The `this` pointer to the training session.
* \param[in] trainable_only Whether to skip non-trainable parameters
* \param[out] parameters_buffer The pre-allocated OrtValue buffer to copy from.
*
* \snippet{doc} snippets.dox OrtStatus Return Value
*
*/
ORT_API2_STATUS(CopyBufferToParameters, _Inout_ OrtTrainingSession* sess,
_Inout_ OrtValue* parameters_buffer, bool trainable_only);
/// @}
/// \name Release Training Resources
/// @{
/** \brief Frees up the memory used up by the training session.
*
* This function frees up any memory that was allocated in the training session. The training
* session can no longer be used after this call.
*
*/
ORT_CLASS_RELEASE(TrainingSession);
/** \brief Frees up the memory used up by the checkpoint state.
*
* This function frees up any memory that was allocated in the checkpoint state. The checkpoint
* state can no longer be used after this call.
* \note Note that the checkpoint state must be released only after the training session has been released.
*
*/
ORT_CLASS_RELEASE(CheckpointState);
/// @}
/// \name Prepare For Inferencing
/// @{
/** \brief Export a model that can be used for inferencing.
*
* If the training session was provided with an eval model, the training session can generate
* an inference model if it knows the inference graph outputs. The input inference graph outputs
* are used to prune the eval model so that the inference model's outputs align with the provided outputs.
* The exported model is saved at the path provided and can be used for inferencing with InferenceSession.
* \note Note that the function re-loads the eval model from the path provided to OrtTrainingApi::CreateTrainingSession
* and expects that this path still be valid.
*
* \param[in] sess The `this` pointer to the training session.
* \param[in] inference_model_path Path where the inference model should be serialized to.
* \param[in] graph_outputs_len Size of the graph output names array.
* \param[in] graph_output_names Names of the outputs that are needed in the inference model.
*
* \snippet{doc} snippets.dox OrtStatus Return Value
*
*/
ORT_API2_STATUS(ExportModelForInferencing, _Inout_ OrtTrainingSession* sess,
_In_ const ORTCHAR_T* inference_model_path, size_t graph_outputs_len,
_In_reads_(graph_outputs_len) const char* const* graph_output_names);
/// @}
/// \name Training Utilities
/// @{
/** \brief Sets the seed used for random number generation in Onnxruntime.
*
* Use this function to generate reproducible results. It should be noted that completely reproducible
* results are not guaranteed.
*
* \param[in] seed The seed to be set.
*
* \snippet{doc} snippets.dox OrtStatus Return Value
*
*/
ORT_API2_STATUS(SetSeed, _In_ const int64_t seed);
/// @}
/// \name Model IO Information
/// @{
/** \brief Retrieves the number of user inputs in the training model.
*
* This function returns the number of inputs of the training model so that the user can accordingly
* allocate the OrtValue(s) provided to the OrtTrainingApi::TrainStep function.
*
* \param[in] sess The `this` pointer to the training session.
* \param[out] out Number of user inputs in the training model.
*
* \snippet{doc} snippets.dox OrtStatus Return Value
*
*/
ORT_API2_STATUS(TrainingSessionGetTrainingModelInputCount, _In_ const OrtTrainingSession* sess, _Out_ size_t* out);
/** \brief Retrieves the number of user inputs in the eval model.
*
* This function returns the number of inputs of the eval model so that the user can accordingly
* allocate the OrtValue(s) provided to the OrtTrainingApi::EvalStep function.
*
* \param[in] sess The `this` pointer to the training session.
* \param[out] out Number of user inputs in the eval model.
*
* \snippet{doc} snippets.dox OrtStatus Return Value
*
*/
ORT_API2_STATUS(TrainingSessionGetEvalModelInputCount, _In_ const OrtTrainingSession* sess, _Out_ size_t* out);
/** \brief Retrieves the name of the user input at given index in the training model.
*
* This function returns the names of inputs of the training model that can be associated with the
* OrtValue(s) provided to the OrtTrainingApi::TrainStep function.
*
* \param[in] sess The `this` pointer to the training session.
* \param[in] index The index of the training model input name requested.
* \param[in] allocator The allocator to use to allocate the memory for the requested name.
* \param[out] output Name of the user input for the training model at the given index.
*
* \snippet{doc} snippets.dox OrtStatus Return Value
*
*/
ORT_API2_STATUS(TrainingSessionGetTrainingModelInputName, _In_ const OrtTrainingSession* sess, size_t index,
_In_ OrtAllocator* allocator, _Outptr_ char** output);
/** \brief Retrieves the name of the user input at given index in the eval model.
*
* This function returns the names of inputs of the eval model that can be associated with the OrtValue(s) provided
* to the OrtTrainingApi::EvalStep function.
*
* \param[in] sess The `this` pointer to the training session.
* \param[in] index The index of the eval model input name requested.
* \param[in] allocator The allocator to use to allocate the memory for the requested name.
* \param[out] output Name of the user input for the eval model at the given index.
*
* \snippet{doc} snippets.dox OrtStatus Return Value
*
*/
ORT_API2_STATUS(TrainingSessionGetEvalModelInputName, _In_ const OrtTrainingSession* sess, size_t index,
_In_ OrtAllocator* allocator, _Outptr_ char** output);
/// @}
/// \name Accessing The Training Session State
/// @{
/** \brief Adds or updates the given property to/in the checkpoint state.
*
* Runtime properties such as epoch, training step, best score, and others can be added to the checkpoint
* state by the user by calling this function with the corresponding property name and value.
* The given property name must be unique to be able to successfully add the property.
*
* \param[in] checkpoint_state The checkpoint state which should hold the property.
* \param[in] property_name Name of the property being added or updated.
* \param[in] property_type Type of the property associated with the given name.
* \param[in] property_value Property value associated with the given name.
*
* \snippet{doc} snippets.dox OrtStatus Return Value
*
*/
ORT_API2_STATUS(AddProperty, _Inout_ OrtCheckpointState* checkpoint_state,
_In_ const char* property_name, _In_ enum OrtPropertyType property_type,
_In_ void* property_value);
/** \brief Gets the property value associated with the given name from the checkpoint state.
*
* Gets the property value from an existing entry in the checkpoint state. The property must
* exist in the checkpoint state to be able to retrieve it successfully.
*
* \param[in] checkpoint_state The checkpoint state that is currently holding the property.
* \param[in] property_name Name of the property being retrieved.
* \param[in] allocator Allocator used to allocate the memory for the property_value.
* \param[out] property_type Type of the property associated with the given name.
* \param[out] property_value Property value associated with the given name.
*
* \snippet{doc} snippets.dox OrtStatus Return Value
*
*/
ORT_API2_STATUS(GetProperty, _In_ const OrtCheckpointState* checkpoint_state,
_In_ const char* property_name, _Inout_ OrtAllocator* allocator,
_Out_ enum OrtPropertyType* property_type, _Outptr_ void** property_value);
/// @}
/// \name Accessing The Training Session State
/// @{
/** \brief Load a checkpoint state from a buffer into checkpoint_state.
*
* This function will parse a checkpoint bytes buffer, pull relevant data and load the training
* state into the checkpoint_state. This checkpoint state can then be used to create the
* training session by invoking OrtTrainingApi::CreateTrainingSession. By doing so, the training
* session will resume training from the given checkpoint state.
* \note Note that the training session created with a checkpoint state uses this state to store the entire
* training state (including model parameters, its gradients, the optimizer states and the properties).
* As a result, it is required that the checkpoint state outlive the lifetime of the training session.
*
* \param[in] checkpoint_buffer Path to the checkpoint bytes buffer.
* \param[in] num_bytes Number of bytes in the checkpoint buffer.
* \param[out] checkpoint_state Checkpoint state that contains the states of the training session.
*
* \snippet{doc} snippets.dox OrtStatus Return Value
*
*/
ORT_API2_STATUS(LoadCheckpointFromBuffer, _In_ const void* checkpoint_buffer,
_In_ const size_t num_bytes, _Outptr_ OrtCheckpointState** checkpoint_state);
/** \brief Retrieves the type and shape information of the parameter associated with the given parameter name.
*
* This function retrieves the type and shape of the parameter associated with the given parameter name.
* The parameter must exist in the checkpoint state to be able to retrieve its type and shape information successfully.
*
* \param[in] checkpoint_state The checkpoint state.
* \param[in] parameter_name Name of the parameter being retrieved.
* \param[out] parameter_type_and_shape The type and shape of the parameter being retrieved.
*
* \snippet{doc} snippets.dox OrtStatus Return Value
*
*/
ORT_API2_STATUS(GetParameterTypeAndShape, _In_ const OrtCheckpointState* checkpoint_state,
_In_ const char* parameter_name, _Outptr_ OrtTensorTypeAndShapeInfo** parameter_type_and_shape);
/** \brief Updates the data associated with the model parameter in the checkpoint state for the given parameter name.
*
* This function updates a model parameter in the checkpoint state with the given parameter data.
* The training session must be already created with the checkpoint state that contains the parameter
* being updated. The given parameter is copied over to the registered device for the training session.
* The parameter must exist in the checkpoint state to be able to update it successfully.
*
* \param[in] checkpoint_state The checkpoint state.
* \param[in] parameter_name Name of the parameter being updated.
* \param[in] parameter The parameter data that should replace the existing parameter data.
*
* \snippet{doc} snippets.dox OrtStatus Return Value
*
*/
ORT_API2_STATUS(UpdateParameter, _Inout_ OrtCheckpointState* checkpoint_state,
_In_ const char* parameter_name, _In_ OrtValue* parameter);
/** \brief Gets the data associated with the model parameter from the checkpoint state for the given parameter name.
*
* This function retrieves the model parameter data from the checkpoint state for the given parameter name.
* The parameter is copied over and returned as an OrtValue. The training session must be already created
* with the checkpoint state that contains the parameter being retrieved.
* The parameter must exist in the checkpoint state to be able to retrieve it successfully.
*
* \param[in] checkpoint_state The checkpoint state.
* \param[in] parameter_name Name of the parameter being retrieved.
* \param[in] allocator Allocator used to allocate the memory for the parameter.
* \param[out] parameter The parameter data that is retrieved from the checkpoint state.
*
* \snippet{doc} snippets.dox OrtStatus Return Value
*
*/
ORT_API2_STATUS(GetParameter, _In_ const OrtCheckpointState* checkpoint_state,
_In_ const char* parameter_name, _Inout_ OrtAllocator* allocator,
_Outptr_ OrtValue** parameter);
/// @}
};
typedef struct OrtTrainingApi OrtTrainingApi;
/// @}
@@ -1,418 +0,0 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
#pragma once
#include "onnxruntime_training_c_api.h"
#include <optional>
#include <variant>
namespace Ort::detail {
#define ORT_DECLARE_TRAINING_RELEASE(NAME) \
void OrtRelease(Ort##NAME* ptr);
// These release methods must be forward declared before including onnxruntime_cxx_api.h
// otherwise class Base won't be aware of them
ORT_DECLARE_TRAINING_RELEASE(CheckpointState);
ORT_DECLARE_TRAINING_RELEASE(TrainingSession);
} // namespace Ort::detail
#include "onnxruntime_cxx_api.h"
namespace Ort {
/// <summary>
/// This function returns the C training api struct with the pointers to the ort training C functions.
/// If using C++, please use the class instances instead of invoking the C functions directly.
/// </summary>
/// <returns>OrtTrainingApi struct with ort training C function pointers.</returns>
inline const OrtTrainingApi& GetTrainingApi() { return *GetApi().GetTrainingApi(ORT_API_VERSION); }
namespace detail {
#define ORT_DEFINE_TRAINING_RELEASE(NAME) \
inline void OrtRelease(Ort##NAME* ptr) { GetTrainingApi().Release##NAME(ptr); }
ORT_DEFINE_TRAINING_RELEASE(CheckpointState);
ORT_DEFINE_TRAINING_RELEASE(TrainingSession);
#undef ORT_DECLARE_TRAINING_RELEASE
#undef ORT_DEFINE_TRAINING_RELEASE
} // namespace detail
using Property = std::variant<int64_t, float, std::string>;
/**
* \defgroup TrainingCpp Ort Training C++ API
* @{
*/
/** \brief Holds the state of the training session.
*
* This class holds the entire training session state that includes model parameters, their gradients,
* optimizer parameters, and user properties. The Ort::TrainingSession leverages the Ort::CheckpointState
* by accessing and updating the contained training state.
* \note Note that the training session created with a checkpoint state uses this state to store the entire
* training state (including model parameters, its gradients, the optimizer states and the properties).
* The Ort::TrainingSession does not hold a copy of the Ort::CheckpointState and as a result, it is required
* that the checkpoint state outlive the lifetime of the training session.
* \note Note that the checkpoint state can be either the complete checkpoint state or the nominal checkpoint
* state depending on the version provided while loading the checkpoint.
*
*/
class CheckpointState : public detail::Base<OrtCheckpointState> {
private:
CheckpointState(OrtCheckpointState* checkpoint_state) { p_ = checkpoint_state; }
public:
// Construct the checkpoint state by loading the checkpoint by calling LoadCheckpoint
CheckpointState() = delete;
/// \name Accessing The Training Session State
/// @{
/** \brief Load a checkpoint state from a file on disk into checkpoint_state.
*
* This function will parse a checkpoint file, pull relevant data and load the training
* state and return an instance of Ort::CheckpointState. This checkpoint state can then be used to create the
* training session by instantiating Ort::TrainingSession. By doing so, the training session will resume
* training from the given checkpoint state.
*
* \param[in] path_to_checkpoint Path to the checkpoint file
* \return Ort::CheckpointState object which holds the state of the training session parameters.
*
*/
static CheckpointState LoadCheckpoint(const std::basic_string<ORTCHAR_T>& path_to_checkpoint);
/** \brief Load a checkpoint state from a buffer.
*
* This function will parse a checkpoint buffer, pull relevant data and load the training
* state and return an instance of Ort::CheckpointState. This checkpoint state can then be used to create the
* training session by instantiating Ort::TrainingSession. By doing so, the training session will resume
* training from the given checkpoint state.
*
* \param[in] buffer Buffer containing the checkpoint data.
* \return Ort::CheckpointState object which holds the state of the training session parameters.
*
*/
static CheckpointState LoadCheckpointFromBuffer(const std::vector<uint8_t>& buffer);
/** \brief Save the given state to a checkpoint file on disk.
*
* This function serializes the provided checkpoint state to a file on disk.
* This checkpoint can later be loaded by invoking Ort::CheckpointState::LoadCheckpoint to resume
* training from this snapshot of the state.
*
* \param[in] checkpoint_state The checkpoint state to save.
* \param[in] path_to_checkpoint Path to the checkpoint file.
* \param[in] include_optimizer_state Flag to indicate whether to save the optimizer state or not.
*
*/
static void SaveCheckpoint(const CheckpointState& checkpoint_state,
const std::basic_string<ORTCHAR_T>& path_to_checkpoint,
const bool include_optimizer_state = false);
/** \brief Adds or updates the given property to/in the checkpoint state.
*
* Runtime properties such as epoch, training step, best score, and others can be added to the checkpoint
* state by the user by calling this function with the corresponding property name and value.
* The given property name must be unique to be able to successfully add the property.
*
* \param[in] property_name Name of the property being added or updated.
* \param[in] property_value Property value associated with the given name.
*
*/
void AddProperty(const std::string& property_name, const Property& property_value);
/** \brief Gets the property value associated with the given name from the checkpoint state.
*
* Gets the property value from an existing entry in the checkpoint state. The property must
* exist in the checkpoint state to be able to retrieve it successfully.
*
* \param[in] property_name Name of the property being retrieved.
* \return Property value associated with the given property name.
*
*/
Property GetProperty(const std::string& property_name);
/** \brief Updates the data associated with the model parameter in the checkpoint state for the given parameter name.
*
* This function updates a model parameter in the checkpoint state with the given parameter data.
* The training session must be already created with the checkpoint state that contains the parameter
* being updated. The given parameter is copied over to the registered device for the training session.
* The parameter must exist in the checkpoint state to be able to update it successfully.
*
* \param[in] parameter_name Name of the parameter being updated.
* \param[in] parameter The parameter data that should replace the existing parameter data.
*
*/
void UpdateParameter(const std::string& parameter_name, const Value& parameter);
/** \brief Gets the data associated with the model parameter from the checkpoint state for the given parameter name.
*
* This function retrieves the model parameter data from the checkpoint state for the given parameter name.
* The parameter is copied over to the provided OrtValue. The training session must be already created
* with the checkpoint state that contains the parameter being retrieved.
* The parameter must exist in the checkpoint state to be able to retrieve it successfully.
*
* \param[in] parameter_name Name of the parameter being retrieved.
* \return The parameter data that is retrieved from the checkpoint state.
*
*/
Value GetParameter(const std::string& parameter_name);
/// @}
};
/** \brief Trainer class that provides training, evaluation and optimizer methods for training an ONNX models.
*
* The training session requires four training artifacts
* - The training onnx model
* - The evaluation onnx model (optional)
* - The optimizer onnx model
* - The checkpoint file
*
* These artifacts can be generated using the `onnxruntime-training` python [utility](https://github.com/microsoft/onnxruntime/blob/main/orttraining/orttraining/python/training/onnxblock/README.md).
*
*/
class TrainingSession : public detail::Base<OrtTrainingSession> {
private:
size_t training_model_output_count_, eval_model_output_count_;
public:
/// \name Constructing the Training Session
/// @{
/** \brief Create a training session that can be used to begin or resume training.
*
* This constructor instantiates the training session based on the env and session options provided that can
* begin or resume training from a given checkpoint state for the given onnx models.
* The checkpoint state represents the parameters of the training session which will be moved
* to the device specified by the user through the session options (if necessary).
*
* \param[in] env Env to be used for the training session.
* \param[in] session_options SessionOptions that the user can customize for this training session.
* \param[in] checkpoint_state Training states that the training session uses as a starting point for training.
* \param[in] train_model_path Model to be used to perform training.
* \param[in] eval_model_path Model to be used to perform evaluation.
* \param[in] optimizer_model_path Model to be used to perform gradient descent.
*
*/
TrainingSession(const Env& env, const SessionOptions& session_options, CheckpointState& checkpoint_state,
const std::basic_string<ORTCHAR_T>& train_model_path,
const std::optional<std::basic_string<ORTCHAR_T>>& eval_model_path = std::nullopt,
const std::optional<std::basic_string<ORTCHAR_T>>& optimizer_model_path = std::nullopt);
/** \brief Create a training session that can be used to begin or resume training.
* This constructor allows the users to load the models from buffers instead of files.
*
* \param[in] env Env to be used for the training session.
* \param[in] session_options SessionOptions that the user can customize for this training session.
* \param[in] checkpoint_state Training states that the training session uses as a starting point for training.
* \param[in] train_model_data Buffer containing training model data.
* \param[in] eval_model_data Buffer containing evaluation model data.
* \param[in] optim_model_data Buffer containing optimizer model (used for performing weight/parameter update).
*
*/
TrainingSession(const Env& env, const SessionOptions& session_options, CheckpointState& checkpoint_state,
const std::vector<uint8_t>& train_model_data, const std::vector<uint8_t>& eval_model_data = {},
const std::vector<uint8_t>& optim_model_data = {});
/// @}
/// \name Implementing The Training Loop
/// @{
/** \brief Computes the outputs of the training model and the gradients of the trainable parameters for the given inputs
*
* This function performs a training step that computes the outputs of the training model and the gradients
* of the trainable parameters for the given inputs. The train step is performed based on the training model
* that was provided to the training session.
* The Ort::TrainingSession::TrainStep is equivalent of running forward propagation and backward propagation in a single
* step.
* The gradients computed are stored inside the training session state so they can be later consumed
* by the Ort::TrainingSession::OptimizerStep function.
* The gradients can be lazily reset by invoking the Ort::TrainingSession::LazyResetGrad function.
*
* \param[in] input_values The user inputs to the training model.
* \return A std::vector of Ort::Value objects that represents the output of the forward pass of the training model.
*
*
*/
std::vector<Value> TrainStep(const std::vector<Value>& input_values);
/** \brief Reset the gradients of all trainable parameters to zero lazily.
*
* This function sets the internal state of the training session such that the gradients of the trainable
* parameters in the OrtCheckpointState will be scheduled to be reset just before the new gradients are
* computed on the next invocation of the next Ort::TrainingSession::TrainStep.
*
*/
void LazyResetGrad();
/** \brief Computes the outputs for the eval model for the given inputs
*
* This function performs an eval step that computes the outputs of the eval model for the given inputs.
* The eval step is performed based on the eval model that was provided to the training session.
*
* \param[in] input_values The user inputs to the eval model.
* \return A std::vector of Ort::Value objects that represents the output of the eval pass.
*
*/
std::vector<Value> EvalStep(const std::vector<Value>& input_values);
/** \brief Sets the learning rate for this training session.
*
* This function allows users to set the learning rate for the training session. The current
* learning rate is maintained by the training session and can be overwritten by invoking
* this function with the desired learning rate. This function should not be used when a valid
* learning rate scheduler is registered. It should be used either to set the learning rate
* derived from a custom learning rate scheduler or to set a constant learning rate to be used
* throughout the training session.
* \note Please note that this function does not set the initial learning rate that may be needed
* by the predefined learning rate schedulers. To set the initial learning rate for learning
* rate schedulers, please look at the function Ort::TrainingSession::RegisterLinearLRScheduler.
*
* \param[in] learning_rate Desired learning rate to be set.
*
*/
void SetLearningRate(float learning_rate);
/** \brief Gets the current learning rate for this training session.
*
* This function allows users to get the learning rate for the training session. The current
* learning rate is maintained by the training session, and users can query it for the purpose
* of implementing their own learning rate schedulers.
*
* \return float representing the current learning rate.
*
*/
float GetLearningRate() const;
/** \brief Registers a linear learning rate scheduler for the training session.
*
* Register a linear learning rate scheduler that decays the learning rate by linearly updated
* multiplicative factor from the initial learning rate set on the training session to 0. The decay
* is performed after the initial warm up phase where the learning rate is linearly incremented
* from 0 to the initial learning rate provided.
*
* \param[in] warmup_step_count Warmup steps for LR warmup.
* \param[in] total_step_count Total step count.
* \param[in] initial_lr The initial learning rate to be used by the training session.
*
*/
void RegisterLinearLRScheduler(int64_t warmup_step_count, int64_t total_step_count,
float initial_lr);
/** \brief Update the learning rate based on the registered learing rate scheduler.
*
* Takes a scheduler step that updates the learning rate that is being used by the training session.
* This function should typically be called before invoking the optimizer step for each round,
* or as determined necessary to update the learning rate being used by the training session.
* \note Please note that a valid predefined learning rate scheduler must be first registered to invoke this
* function.
*
*/
void SchedulerStep();
/** \brief Performs the weight updates for the trainable parameters using the optimizer model.
*
* This function performs the weight update step that updates the trainable parameters such that they
* take a step in the direction of their gradients (gradient descent). The optimizer step is performed
* based on the optimizer model that was provided to the training session.
* The updated parameters are stored inside the training state so that they can be used by the next
* Ort::TrainingSession::TrainStep function call.
*
*/
void OptimizerStep();
/// @}
/// \name Prepare For Inferencing
/// @{
/** \brief Export a model that can be used for inferencing.
*
* If the training session was provided with an eval model, the training session can generate
* an inference model if it knows the inference graph outputs. The input inference graph outputs
* are used to prune the eval model so that the inference model's outputs align with the provided outputs.
* The exported model is saved at the path provided and can be used for inferencing with Ort::Session.
* \note Note that the function re-loads the eval model from the path provided to Ort::TrainingSession
* and expects that this path still be valid.
*
* \param[in] inference_model_path Path where the inference model should be serialized to.
* \param[in] graph_output_names Names of the outputs that are needed in the inference model.
*
*/
void ExportModelForInferencing(const std::basic_string<ORTCHAR_T>& inference_model_path,
const std::vector<std::string>& graph_output_names);
/// @}
/// \name Model IO Information
/// @{
/** \brief Retrieves the names of the user inputs for the training and eval models.
*
* This function returns the names of inputs of the training or eval model that can be associated
* with the Ort::Value(s) provided to the Ort::TrainingSession::TrainStep or Ort::TrainingSession::EvalStep
* function.
*
* \param[in] training Whether the training model input names are requested or eval model input names.
* \return Graph input names for either the training model or the eval model.
*
*/
std::vector<std::string> InputNames(const bool training);
/** \brief Retrieves the names of the user outputs for the training and eval models.
*
* This function returns the names of outputs of the training or eval model that can be associated
* with the Ort::Value(s) returned by the Ort::TrainingSession::TrainStep or Ort::TrainingSession::EvalStep
* function.
*
* \param[in] training Whether the training model output names are requested or eval model output names.
* \return Graph output names for either the training model or the eval model.
*
*/
std::vector<std::string> OutputNames(const bool training);
/// @}
/// \name Accessing The Training Session State
/// @{
/** \brief Returns a contiguous buffer that holds a copy of all training state parameters
*
* \param[in] only_trainable Whether to only copy trainable parameters or to copy all parameters.
* \return Contiguous buffer to the model parameters.
*
*/
Value ToBuffer(const bool only_trainable);
/** \brief Loads the training session model parameters from a contiguous buffer
*
* In case the training session was created with a nominal checkpoint, invoking this function is required
* to load the updated parameters onto the checkpoint to complete it.
*
* \param[in] buffer Contiguous buffer to load the parameters from.
*/
void FromBuffer(Value& buffer);
/// @}
};
/// \name Training Utilities
/// @{
/** \brief This function sets the seed for generating random numbers.
*
* Use this function to generate reproducible results. It should be noted that completely
* reproducible results are not guaranteed.
*
* \param[in] seed Manual seed to use for random number generation.
*/
void SetSeed(const int64_t seed);
/// @}
/// @}
} // namespace Ort
#include "onnxruntime_training_cxx_inline.h"
@@ -1,295 +0,0 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
#pragma once
#include "onnxruntime_training_c_api.h"
#include "onnxruntime_cxx_api.h"
namespace Ort {
inline TrainingSession::TrainingSession(const Env& env, const SessionOptions& session_options,
CheckpointState& checkpoint_state,
const std::basic_string<ORTCHAR_T>& train_model_path,
const std::optional<std::basic_string<ORTCHAR_T>>& eval_model_path,
const std::optional<std::basic_string<ORTCHAR_T>>& optimizer_model_path) {
ThrowOnError(GetTrainingApi().CreateTrainingSession(
env, session_options, checkpoint_state,
train_model_path.c_str(),
eval_model_path.has_value() ? eval_model_path.value().c_str() : nullptr,
optimizer_model_path.has_value() ? optimizer_model_path.value().c_str() : nullptr,
&p_));
ThrowOnError(GetTrainingApi().TrainingSessionGetTrainingModelOutputCount(p_, &training_model_output_count_));
ThrowOnError(GetTrainingApi().TrainingSessionGetEvalModelOutputCount(p_, &eval_model_output_count_));
}
inline TrainingSession::TrainingSession(const Env& env, const SessionOptions& session_options,
CheckpointState& checkpoint_state,
const std::vector<uint8_t>& train_model_data,
const std::vector<uint8_t>& eval_model_data,
const std::vector<uint8_t>& optim_model_data) {
ThrowOnError(GetTrainingApi().CreateTrainingSessionFromBuffer(
env, session_options, checkpoint_state,
train_model_data.data(), train_model_data.size(),
eval_model_data.data(), eval_model_data.size(),
optim_model_data.data(), optim_model_data.size(),
&p_));
ThrowOnError(GetTrainingApi().TrainingSessionGetTrainingModelOutputCount(p_, &training_model_output_count_));
ThrowOnError(GetTrainingApi().TrainingSessionGetEvalModelOutputCount(p_, &eval_model_output_count_));
}
inline std::vector<Value> TrainingSession::TrainStep(const std::vector<Value>& input_values) {
std::vector<Value> output_values;
output_values.reserve(training_model_output_count_);
for (size_t i = 0; i < training_model_output_count_; i++) output_values.emplace_back(nullptr);
auto ort_input_values = reinterpret_cast<const OrtValue* const*>(input_values.data());
auto ort_output_values = reinterpret_cast<OrtValue**>(output_values.data());
RunOptions run_options;
ThrowOnError(GetTrainingApi().TrainStep(
p_, run_options, input_values.size(), ort_input_values,
training_model_output_count_, ort_output_values));
return output_values;
}
inline void TrainingSession::LazyResetGrad() {
ThrowOnError(GetTrainingApi().LazyResetGrad(p_));
}
inline std::vector<Value> TrainingSession::EvalStep(const std::vector<Value>& input_values) {
std::vector<Value> output_values;
output_values.reserve(eval_model_output_count_);
for (size_t i = 0; i < eval_model_output_count_; i++) output_values.emplace_back(nullptr);
auto ort_input_values = reinterpret_cast<const OrtValue* const*>(input_values.data());
auto ort_output_values = reinterpret_cast<OrtValue**>(output_values.data());
RunOptions run_options;
ThrowOnError(GetTrainingApi().EvalStep(
p_, run_options, input_values.size(), ort_input_values,
eval_model_output_count_, ort_output_values));
return output_values;
}
inline void TrainingSession::SetLearningRate(float learning_rate) {
ThrowOnError(GetTrainingApi().SetLearningRate(p_, learning_rate));
}
inline float TrainingSession::GetLearningRate() const {
float learning_rate = 0;
ThrowOnError(GetTrainingApi().GetLearningRate(p_, &learning_rate));
return learning_rate;
}
inline void TrainingSession::RegisterLinearLRScheduler(int64_t warmup_step_count, int64_t total_step_count,
float initial_lr) {
ThrowOnError(GetTrainingApi().RegisterLinearLRScheduler(p_, warmup_step_count, total_step_count,
initial_lr));
}
inline void TrainingSession::SchedulerStep() {
ThrowOnError(GetTrainingApi().SchedulerStep(p_));
}
inline void TrainingSession::OptimizerStep() {
RunOptions run_options;
ThrowOnError(GetTrainingApi().OptimizerStep(p_, run_options));
}
inline std::vector<std::string> TrainingSession::InputNames(const bool training) {
auto& input_count_function = training ? GetTrainingApi().TrainingSessionGetTrainingModelInputCount
: GetTrainingApi().TrainingSessionGetEvalModelInputCount;
auto& input_name_function = training ? GetTrainingApi().TrainingSessionGetTrainingModelInputName
: GetTrainingApi().TrainingSessionGetEvalModelInputName;
size_t input_count = 0;
ThrowOnError(input_count_function(p_, &input_count));
std::vector<std::string> input_names(input_count);
AllocatorWithDefaultOptions allocator;
for (size_t index = 0; index < input_count; ++index) {
char* input_name;
ThrowOnError(input_name_function(p_, index, allocator, &input_name));
input_names[index] = std::string(input_name);
allocator.Free(input_name);
}
return input_names;
}
inline std::vector<std::string> TrainingSession::OutputNames(const bool training) {
auto& output_count_function = training ? GetTrainingApi().TrainingSessionGetTrainingModelOutputCount
: GetTrainingApi().TrainingSessionGetEvalModelOutputCount;
auto& output_name_function = training ? GetTrainingApi().TrainingSessionGetTrainingModelOutputName
: GetTrainingApi().TrainingSessionGetEvalModelOutputName;
size_t output_count = 0;
ThrowOnError(output_count_function(p_, &output_count));
std::vector<std::string> output_names(output_count);
AllocatorWithDefaultOptions allocator;
for (size_t index = 0; index < output_count; ++index) {
char* output_name;
ThrowOnError(output_name_function(p_, index, allocator, &output_name));
output_names[index] = std::string(output_name);
allocator.Free(output_name);
}
return output_names;
}
inline Value TrainingSession::ToBuffer(const bool only_trainable) {
size_t buffer_size = 0U;
ThrowOnError(GetTrainingApi().GetParametersSize(p_, &buffer_size, only_trainable));
std::array<int64_t, 1> buffer_shape{static_cast<int64_t>(buffer_size)};
AllocatorWithDefaultOptions allocator;
Value buffer = Value::CreateTensor(allocator, buffer_shape.data(), 1U,
ONNXTensorElementDataType::ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT);
ThrowOnError(GetTrainingApi().CopyParametersToBuffer(p_, buffer, only_trainable));
return buffer;
}
inline void TrainingSession::FromBuffer(Value& buffer) {
if (!buffer.IsTensor()) {
ThrowStatus(Status("Incorrect buffer received. Expected a tensor buffer.", OrtErrorCode::ORT_INVALID_ARGUMENT));
}
auto tensor_info = buffer.GetTensorTypeAndShapeInfo();
auto buffer_shape = tensor_info.GetShape();
if (buffer_shape.size() != 1U) {
ThrowStatus(Status("Incorrect buffer received. Expected a contiguous tensor buffer.",
OrtErrorCode::ORT_INVALID_ARGUMENT));
}
auto buffer_size = buffer_shape.front();
size_t session_buffer_size = 0U;
ThrowOnError(GetTrainingApi().GetParametersSize(p_, &session_buffer_size, false));
if (buffer_size == static_cast<int64_t>(session_buffer_size)) {
ThrowOnError(GetTrainingApi().CopyBufferToParameters(p_, buffer, false));
return;
}
size_t session_buffer_size_trainable_only = 0U;
ThrowOnError(GetTrainingApi().GetParametersSize(p_, &session_buffer_size_trainable_only, true));
if (buffer_size == static_cast<int64_t>(session_buffer_size_trainable_only)) {
ThrowOnError(GetTrainingApi().CopyBufferToParameters(p_, buffer, true));
return;
} else {
ThrowStatus(Status("Incorrect buffer size received.", OrtErrorCode::ORT_INVALID_ARGUMENT));
}
}
inline CheckpointState CheckpointState::LoadCheckpoint(const std::basic_string<ORTCHAR_T>& path_to_checkpoint) {
OrtCheckpointState* checkpoint_state;
ThrowOnError(GetTrainingApi().LoadCheckpoint(path_to_checkpoint.c_str(), &checkpoint_state));
return CheckpointState(checkpoint_state);
}
inline CheckpointState CheckpointState::LoadCheckpointFromBuffer(const std::vector<uint8_t>& buffer) {
OrtCheckpointState* checkpoint_state;
ThrowOnError(GetTrainingApi().LoadCheckpointFromBuffer(buffer.data(), buffer.size(), &checkpoint_state));
return CheckpointState(checkpoint_state);
}
inline void CheckpointState::SaveCheckpoint(const CheckpointState& checkpoint_states,
const std::basic_string<ORTCHAR_T>& path_to_checkpoint,
const bool include_optimizer_state) {
ThrowOnError(GetTrainingApi().SaveCheckpoint(checkpoint_states, path_to_checkpoint.c_str(),
include_optimizer_state));
}
inline void TrainingSession::ExportModelForInferencing(const std::basic_string<ORTCHAR_T>& inference_model_path,
const std::vector<std::string>& graph_output_names) {
std::vector<const char*> output_names;
output_names.reserve(graph_output_names.size());
for (const auto& output_name : graph_output_names) {
output_names.push_back(output_name.c_str());
}
ThrowOnError(GetTrainingApi().ExportModelForInferencing(
p_, inference_model_path.c_str(), graph_output_names.size(), output_names.data()));
}
inline void SetSeed(const int64_t seed) {
ThrowOnError(GetTrainingApi().SetSeed(seed));
}
inline void CheckpointState::AddProperty(const std::string& property_name, const Property& property_value) {
if (std::holds_alternative<int64_t>(property_value)) {
int64_t value = std::get<int64_t>(property_value);
void* value_p = &value;
ThrowOnError(GetTrainingApi().AddProperty(p_, property_name.c_str(), OrtPropertyType::OrtIntProperty, value_p));
} else if (std::holds_alternative<float>(property_value)) {
float value = std::get<float>(property_value);
void* value_p = &value;
ThrowOnError(GetTrainingApi().AddProperty(p_, property_name.c_str(), OrtPropertyType::OrtFloatProperty, value_p));
} else if (std::holds_alternative<std::string>(property_value)) {
std::string value = std::get<std::string>(property_value);
auto buffer = std::make_unique<char[]>(value.length() + 1);
memcpy(buffer.get(), value.c_str(), value.length());
// AddProperty takes a char* and calls PropertyBag::AddProperty which takes a std::string. The data will be
// copied at that point so buffer can free the local allocation once the call is made.
ThrowOnError(GetTrainingApi().AddProperty(p_, property_name.c_str(), OrtPropertyType::OrtStringProperty,
buffer.get()));
} else {
ThrowStatus(Status("Unknown property type received.", OrtErrorCode::ORT_INVALID_ARGUMENT));
}
}
inline Property CheckpointState::GetProperty(const std::string& property_name) {
void* property_value = nullptr;
OrtPropertyType property_type;
AllocatorWithDefaultOptions allocator;
ThrowOnError(GetTrainingApi().GetProperty(p_, property_name.c_str(), allocator, &property_type, &property_value));
Property property;
switch (property_type) {
case OrtPropertyType::OrtIntProperty: {
auto value_p = reinterpret_cast<int64_t*>(property_value);
property = *value_p;
allocator.Free(property_value);
break;
}
case OrtPropertyType::OrtFloatProperty: {
auto value_p = reinterpret_cast<float*>(property_value);
property = *value_p;
allocator.Free(property_value);
break;
}
case OrtPropertyType::OrtStringProperty: {
auto value_p = reinterpret_cast<char*>(property_value);
property = std::string(value_p);
allocator.Free(property_value);
break;
}
default: {
ThrowStatus(Status("Unknown property type received.", OrtErrorCode::ORT_INVALID_ARGUMENT));
break;
}
}
return property;
}
inline void CheckpointState::UpdateParameter(const std::string& parameter_name, const Value& parameter) {
ThrowOnError(GetTrainingApi().UpdateParameter(p_, parameter_name.c_str(), parameter));
}
inline Value CheckpointState::GetParameter(const std::string& parameter_name) {
AllocatorWithDefaultOptions allocator;
OrtValue* parameter;
ThrowOnError(GetTrainingApi().GetParameter(p_, parameter_name.c_str(), allocator, &parameter));
return Value{parameter};
}
} // namespace Ort