Working version on Razor Blade
This commit is contained in:
File diff suppressed because it is too large
Load Diff
@@ -80,22 +80,19 @@ struct Exception : std::exception {
|
||||
throw Ort::Exception(string, code)
|
||||
#endif
|
||||
|
||||
// This is used internally by the C++ API. This class holds the global variable that points to the OrtApi,
|
||||
// it's in a template so that we can define a global variable in a header and make
|
||||
// it transparent to the users of the API.
|
||||
template <typename T>
|
||||
struct Global {
|
||||
static const OrtApi* api_;
|
||||
};
|
||||
|
||||
// If macro ORT_API_MANUAL_INIT is defined, no static initialization will be performed. Instead, user must call InitApi() before using it.
|
||||
template <typename T>
|
||||
#ifdef ORT_API_MANUAL_INIT
|
||||
const OrtApi* Global<T>::api_{};
|
||||
inline void InitApi() noexcept { Global<void>::api_ = OrtGetApiBase()->GetApi(ORT_API_VERSION); }
|
||||
// If the macro ORT_API_MANUAL_INIT is defined, no static initialization
|
||||
// will be performed. Instead, users must call InitApi() before using the
|
||||
// ORT C++ APIs..
|
||||
//
|
||||
// InitApi() sets the global API object using the default initialization
|
||||
// logic. Users call this to initialize the ORT C++ APIs at a time that
|
||||
// makes sense in their program.
|
||||
inline void InitApi() noexcept;
|
||||
|
||||
// Used by custom operator libraries that are not linked to onnxruntime. Sets the global API object, which is
|
||||
// required by C++ APIs.
|
||||
// InitApi(const OrtApi*) is used by custom operator libraries that are not
|
||||
// linked to onnxruntime. It sets the global API object, which is required
|
||||
// by the ORT C++ APIs.
|
||||
//
|
||||
// Example mycustomop.cc:
|
||||
//
|
||||
@@ -108,22 +105,88 @@ inline void InitApi() noexcept { Global<void>::api_ = OrtGetApiBase()->GetApi(OR
|
||||
// // ...
|
||||
// }
|
||||
//
|
||||
inline void InitApi(const OrtApi* api) noexcept { Global<void>::api_ = api; }
|
||||
#else
|
||||
#if defined(_MSC_VER) && !defined(__clang__)
|
||||
#pragma warning(push)
|
||||
// "Global initializer calls a non-constexpr function." Therefore you can't use ORT APIs in the other global initializers.
|
||||
// Please define ORT_API_MANUAL_INIT if it conerns you.
|
||||
#pragma warning(disable : 26426)
|
||||
#endif
|
||||
const OrtApi* Global<T>::api_ = OrtGetApiBase()->GetApi(ORT_API_VERSION);
|
||||
#if defined(_MSC_VER) && !defined(__clang__)
|
||||
#pragma warning(pop)
|
||||
#endif
|
||||
inline void InitApi(const OrtApi* api) noexcept;
|
||||
#endif
|
||||
|
||||
namespace detail {
|
||||
// This is used internally by the C++ API. This class holds the global
|
||||
// variable that points to the OrtApi.
|
||||
struct Global {
|
||||
static const OrtApi* Api(const OrtApi* newValue = nullptr) noexcept {
|
||||
// This block-level static will be initialized once when this function is
|
||||
// first executed, delaying the call to DefaultInit() until it is first needed.
|
||||
//
|
||||
// When ORT_API_MANUAL_INIT is not defined, DefaultInit() calls
|
||||
// OrtGetApiBase()->GetApi(), which may result in a shared library being
|
||||
// loaded.
|
||||
//
|
||||
// Using a block-level static instead of a class-level static helps
|
||||
// avoid issues with static initialization order and dynamic libraries
|
||||
// loading other dynamic libraries.
|
||||
//
|
||||
// This makes it safe to include the C++ API headers in a shared library
|
||||
// that is delay loaded or delay loads its dependencies.
|
||||
//
|
||||
// This DOES NOT make it safe to _use_ arbitrary ORT C++ APIs when
|
||||
// initializing static members, however.
|
||||
static const OrtApi* api = DefaultInit();
|
||||
|
||||
if (newValue) {
|
||||
api = newValue;
|
||||
}
|
||||
|
||||
return api;
|
||||
}
|
||||
|
||||
private:
|
||||
// Has different definitions based on ORT_API_MANUAL_INIT
|
||||
static const OrtApi* DefaultInit() noexcept;
|
||||
|
||||
#ifdef ORT_API_MANUAL_INIT
|
||||
// Public APIs to set the OrtApi* to use.
|
||||
friend void ::Ort::InitApi() noexcept;
|
||||
friend void ::Ort::InitApi(const OrtApi*) noexcept;
|
||||
#endif
|
||||
};
|
||||
} // namespace detail
|
||||
|
||||
#ifdef ORT_API_MANUAL_INIT
|
||||
|
||||
// See comments on declaration above for usage.
|
||||
inline void InitApi(const OrtApi* api) noexcept { detail::Global::Api(api); }
|
||||
inline void InitApi() noexcept { InitApi(OrtGetApiBase()->GetApi(ORT_API_VERSION)); }
|
||||
|
||||
#ifdef _MSC_VER
|
||||
// If you get a linker error about a mismatch here, you are trying to
|
||||
// link two compilation units that have different definitions for
|
||||
// ORT_API_MANUAL_INIT together. All compilation units must agree on the
|
||||
// definition of ORT_API_MANUAL_INIT.
|
||||
#pragma detect_mismatch("ORT_API_MANUAL_INIT", "enabled")
|
||||
#endif
|
||||
|
||||
inline const OrtApi* detail::Global::DefaultInit() noexcept {
|
||||
// When ORT_API_MANUAL_INIT is defined, there's no default init that can
|
||||
// be done.
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
#else // ORT_API_MANUAL_INIT
|
||||
|
||||
#ifdef _MSC_VER
|
||||
// If you get a linker error about a mismatch here, you are trying to link
|
||||
// two compilation units that have different definitions for
|
||||
// ORT_API_MANUAL_INIT together. All compilation units must agree on the
|
||||
// definition of ORT_API_MANUAL_INIT.
|
||||
#pragma detect_mismatch("ORT_API_MANUAL_INIT", "disabled")
|
||||
#endif
|
||||
|
||||
inline const OrtApi* detail::Global::DefaultInit() noexcept {
|
||||
return OrtGetApiBase()->GetApi(ORT_API_VERSION);
|
||||
}
|
||||
#endif // ORT_API_MANUAL_INIT
|
||||
|
||||
/// This returns a reference to the ORT C API.
|
||||
inline const OrtApi& GetApi() noexcept { return *Global<void>::api_; }
|
||||
inline const OrtApi& GetApi() noexcept { return *detail::Global::Api(); }
|
||||
|
||||
/// <summary>
|
||||
/// This function returns the onnxruntime version string
|
||||
@@ -173,6 +236,20 @@ inline const OrtCompileApi& GetCompileApi() {
|
||||
return *api;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// This returns a reference to the ORT C Interop API. Used for external resource import with EPs.
|
||||
/// </summary>
|
||||
/// <returns>ORT C Interop API reference</returns>
|
||||
inline const OrtInteropApi& GetInteropApi() {
|
||||
auto* api = GetApi().GetInteropApi();
|
||||
if (api == nullptr) {
|
||||
// minimal build
|
||||
ORT_CXX_API_THROW("Interop API is not available in this build", ORT_FAIL);
|
||||
}
|
||||
|
||||
return *api;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// This returns a reference to the ORT C EP API. Used if authoring a plugin execution provider.
|
||||
/// </summary>
|
||||
@@ -581,6 +658,9 @@ ORT_DEFINE_RELEASE(ValueInfo);
|
||||
|
||||
ORT_DEFINE_RELEASE_FROM_API_STRUCT(ModelCompilationOptions, GetCompileApi);
|
||||
ORT_DEFINE_RELEASE_FROM_API_STRUCT(EpDevice, GetEpApi);
|
||||
ORT_DEFINE_RELEASE_FROM_API_STRUCT(KernelDef, GetEpApi);
|
||||
ORT_DEFINE_RELEASE_FROM_API_STRUCT(KernelDefBuilder, GetEpApi);
|
||||
ORT_DEFINE_RELEASE_FROM_API_STRUCT(KernelRegistry, GetEpApi);
|
||||
|
||||
// This is defined explicitly since OrtTensorRTProviderOptionsV2 is not a C API type,
|
||||
// but the struct has V2 in its name to indicate that it is the second version of the options.
|
||||
@@ -993,6 +1073,9 @@ struct AllocatorWithDefaultOptions : detail::AllocatorImpl<detail::Unowned<OrtAl
|
||||
struct Allocator : detail::AllocatorImpl<OrtAllocator> {
|
||||
explicit Allocator(std::nullptr_t) {} ///< Convenience to create a class member and then replace with an instance
|
||||
Allocator(const Session& session, const OrtMemoryInfo*);
|
||||
|
||||
///< Take ownership of a pointer created by C API
|
||||
explicit Allocator(OrtAllocator* p) : AllocatorImpl<OrtAllocator>{p} {}
|
||||
};
|
||||
|
||||
using UnownedAllocator = detail::AllocatorImpl<detail::Unowned<OrtAllocator>>;
|
||||
@@ -1000,11 +1083,25 @@ using UnownedAllocator = detail::AllocatorImpl<detail::Unowned<OrtAllocator>>;
|
||||
/** \brief Wrapper around ::OrtSyncStream
|
||||
*
|
||||
*/
|
||||
struct SyncStream : detail::Base<OrtSyncStream> {
|
||||
explicit SyncStream(std::nullptr_t) {} ///< Create an empty SyncStream object, must be assigned a valid one to be used
|
||||
explicit SyncStream(OrtSyncStream* p) : Base<OrtSyncStream>{p} {} ///< Take ownership of a pointer created by C API
|
||||
void* GetHandle() const; ///< Wraps SyncStream_GetHandle
|
||||
|
||||
namespace detail {
|
||||
template <typename T>
|
||||
struct SyncStreamImpl : Base<T> {
|
||||
using B = Base<T>;
|
||||
using B::B;
|
||||
// For some reason this is not a const method on the stream
|
||||
void* GetHandle(); ///< Wraps SyncStream_GetHandle
|
||||
};
|
||||
} // namespace detail
|
||||
|
||||
struct SyncStream : detail::SyncStreamImpl<OrtSyncStream> {
|
||||
///< Create an empty SyncStream object, must be assigned a valid one to be used
|
||||
explicit SyncStream(std::nullptr_t) {}
|
||||
///< Take ownership of a pointer created by C API
|
||||
explicit SyncStream(OrtSyncStream* p) : SyncStreamImpl<OrtSyncStream>{p} {}
|
||||
};
|
||||
|
||||
using UnownedSyncStream = detail::SyncStreamImpl<detail::Unowned<OrtSyncStream>>;
|
||||
|
||||
namespace detail {
|
||||
template <typename T>
|
||||
@@ -1067,6 +1164,72 @@ OrtCompiledModelCompatibility GetModelCompatibilityForEpDevices(
|
||||
const std::vector<ConstEpDevice>& ep_devices,
|
||||
const char* compatibility_info);
|
||||
|
||||
/** \brief Extract EP compatibility info from a precompiled model file.
|
||||
*
|
||||
* Parses the model file to extract the compatibility info string for a specific execution provider
|
||||
* from the model's metadata properties. This is only applicable to models that have been precompiled
|
||||
* for an EP. Standard ONNX models do not contain this information.
|
||||
*
|
||||
* \note This operation parses the full ONNX ModelProto from disk.
|
||||
*
|
||||
* \param model_path Path to the ONNX model file.
|
||||
* \param ep_type The execution provider type string. Must be non-empty.
|
||||
* Use ConstEpDevice::EpName() to get this value.
|
||||
* \param allocator Allocator to use for the output string.
|
||||
* \return The compatibility info string, or nullptr if not found for this EP. Caller must free via allocator.
|
||||
* \throws Ort::Exception on error.
|
||||
*/
|
||||
AllocatedStringPtr GetCompatibilityInfoFromModelAllocated(const ORTCHAR_T* model_path, const char* ep_type,
|
||||
OrtAllocator* allocator);
|
||||
|
||||
/** \brief Extract EP compatibility info from precompiled model bytes in memory.
|
||||
*
|
||||
* Same as GetCompatibilityInfoFromModelAllocated but reads from a memory buffer.
|
||||
* Useful when precompiled models are loaded from encrypted storage, network, or other non-file sources.
|
||||
*
|
||||
* \param model_data Pointer to the model data in memory.
|
||||
* \param model_data_length Size of the model data in bytes.
|
||||
* \param ep_type The execution provider type string. Must be non-empty.
|
||||
* \param allocator Allocator to use for the output string.
|
||||
* \return The compatibility info string, or nullptr if not found for this EP. Caller must free via allocator.
|
||||
* \throws Ort::Exception on error.
|
||||
*/
|
||||
AllocatedStringPtr GetCompatibilityInfoFromModelBytesAllocated(const void* model_data, size_t model_data_length,
|
||||
const char* ep_type, OrtAllocator* allocator);
|
||||
|
||||
namespace detail {
|
||||
template <typename T>
|
||||
struct EpAssignedNodeImpl : Ort::detail::Base<T> {
|
||||
using B = Ort::detail::Base<T>;
|
||||
using B::B;
|
||||
|
||||
std::string GetName() const;
|
||||
std::string GetDomain() const;
|
||||
std::string GetOperatorType() const;
|
||||
};
|
||||
} // namespace detail
|
||||
|
||||
/** \brief Constant wrapper around ::OrtEpAssignedNode
|
||||
* \remarks EpAssignedNode is always read-only for ORT API users.
|
||||
*/
|
||||
using ConstEpAssignedNode = detail::EpAssignedNodeImpl<Ort::detail::Unowned<const OrtEpAssignedNode>>;
|
||||
|
||||
namespace detail {
|
||||
template <typename T>
|
||||
struct EpAssignedSubgraphImpl : Ort::detail::Base<T> {
|
||||
using B = Ort::detail::Base<T>;
|
||||
using B::B;
|
||||
|
||||
std::string GetEpName() const;
|
||||
std::vector<ConstEpAssignedNode> GetNodes() const;
|
||||
};
|
||||
} // namespace detail
|
||||
|
||||
/** \brief Constant wrapper around ::OrtEpAssignedSubgraph
|
||||
* \remarks EpAssignedSubgraph is always read-only for ORT API users.
|
||||
*/
|
||||
using ConstEpAssignedSubgraph = detail::EpAssignedSubgraphImpl<Ort::detail::Unowned<const OrtEpAssignedSubgraph>>;
|
||||
|
||||
/** \brief The Env (Environment)
|
||||
*
|
||||
* The Env holds the logging state used by all other objects.
|
||||
@@ -1088,6 +1251,9 @@ struct Env : detail::Base<OrtEnv> {
|
||||
Env(const OrtThreadingOptions* tp_options, OrtLoggingFunction logging_function, void* logger_param,
|
||||
OrtLoggingLevel logging_level = ORT_LOGGING_LEVEL_WARNING, _In_ const char* logid = "");
|
||||
|
||||
/// \brief Wraps OrtApi::CreateEnvWithOptions
|
||||
explicit Env(const OrtEnvCreationOptions* options);
|
||||
|
||||
/// \brief C Interop Helper
|
||||
explicit Env(OrtEnv* p) : Base<OrtEnv>{p} {}
|
||||
|
||||
@@ -1207,6 +1373,15 @@ struct RunOptions : detail::Base<OrtRunOptions> {
|
||||
* \param adapter The LoraAdapter to be used as the active adapter
|
||||
*/
|
||||
RunOptions& AddActiveLoraAdapter(const LoraAdapter& adapter);
|
||||
|
||||
/** \brief Associate a sync stream with the run options.
|
||||
*
|
||||
* When set, the EP uses this stream for execution, enabling proper
|
||||
* synchronization with imported external semaphores. Wraps OrtApi::RunOptionsSetSyncStream.
|
||||
*
|
||||
* \param stream The OrtSyncStream to associate with these run options. May be nullptr to clear.
|
||||
*/
|
||||
RunOptions& SetSyncStream(OrtSyncStream* stream);
|
||||
};
|
||||
|
||||
namespace detail {
|
||||
@@ -1364,6 +1539,12 @@ struct SessionOptionsImpl : ConstSessionOptionsImpl<T> {
|
||||
|
||||
///< Wraps OrtApi::SessionOptionsAppendExecutionProvider_VitisAI
|
||||
SessionOptionsImpl& AppendExecutionProvider_VitisAI(const std::unordered_map<std::string, std::string>& provider_options = {});
|
||||
|
||||
///< Wraps OrtApi::AddFreeDimensionOverride
|
||||
SessionOptionsImpl& AddFreeDimensionOverride(const char* dim_denotation, int64_t dim_value);
|
||||
|
||||
///< Wraps OrtApi::AddFreeDimensionOverrideByName
|
||||
SessionOptionsImpl& AddFreeDimensionOverrideByName(const char* dim_name, int64_t dim_value);
|
||||
};
|
||||
} // namespace detail
|
||||
|
||||
@@ -1521,6 +1702,7 @@ struct ConstSessionImpl : Base<T> {
|
||||
std::vector<ConstMemoryInfo> GetMemoryInfoForInputs() const; ///< Wrapper for OrtApi::SessionGetMemoryInfoForInputs
|
||||
std::vector<ConstMemoryInfo> GetMemoryInfoForOutputs() const; ///< Wrapper for OrtApi::SessionGetMemoryInfoForOutputs
|
||||
std::vector<ConstEpDevice> GetEpDeviceForInputs() const; ///< Wrapper for OrtApi::SessionGetEpDeviceForInputs
|
||||
std::vector<ConstEpDevice> GetEpDeviceForOutputs() const; ///< Wrapper for OrtApi::SessionGetEpDeviceForOutputs
|
||||
|
||||
/** \brief Returns a copy of input name at the specified index.
|
||||
*
|
||||
@@ -1558,9 +1740,14 @@ struct ConstSessionImpl : Base<T> {
|
||||
|
||||
int GetOpset(const std::string& domain) const; ///< Wraps OrtApi::SessionGetOpsetForDomain
|
||||
|
||||
// Will move before checkin if that's the case.
|
||||
std::vector<ValueInfo> GetInputs() const;
|
||||
std::vector<ValueInfo> GetOutputs() const;
|
||||
|
||||
/** \brief Returns information on the subgraph/nodes assigned to execution providers in the session.
|
||||
*
|
||||
* \return A list of ConstEpAssignedSubgraph instances.
|
||||
*/
|
||||
std::vector<ConstEpAssignedSubgraph> GetEpGraphAssignmentInfo() const;
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
@@ -1705,6 +1892,7 @@ struct TensorTypeAndShapeInfoImpl : Base<T> {
|
||||
void GetSymbolicDimensions(const char** values, size_t values_count) const; ///< Wraps OrtApi::GetSymbolicDimensions
|
||||
std::vector<const char*> GetSymbolicDimensions() const;
|
||||
|
||||
bool HasShape() const; ///< Wraps OrtApi::TensorTypeAndShape_HasShape
|
||||
std::vector<int64_t> GetShape() const; ///< Uses GetDimensionsCount & GetDimensions to return a std::vector of the shape
|
||||
};
|
||||
|
||||
@@ -2032,6 +2220,19 @@ struct ConstValueImpl : Base<T> {
|
||||
const R* GetSparseTensorValues() const;
|
||||
|
||||
#endif
|
||||
|
||||
/// <summary>
|
||||
/// Returns the tensor's element type and a reference to the tensor's internal shape data. The shape data is owned
|
||||
/// by the Ort::Value and becomes invalid when the Ort::Value is destroyed or if the underlying shape data is
|
||||
/// updated or reallocated.
|
||||
///
|
||||
/// For a scalar, shape.shape is nullptr and shape.shape_len is 0.
|
||||
///
|
||||
/// Wraps OrtApi::GetTensorElementTypeAndShapeDataReference.
|
||||
/// </summary>
|
||||
/// <param name="elem_type">Output parameter set to the element's data type.</param>
|
||||
/// <param name="shape">Output parameter set to the OrtValue instance's shape data and number of elements.</param>
|
||||
void GetTensorElementTypeAndShapeDataReference(ONNXTensorElementDataType& elem_type, Shape& shape) const;
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
@@ -2635,7 +2836,7 @@ struct KernelContext {
|
||||
UnownedValue GetOutput(size_t index, const std::vector<int64_t>& dims) const;
|
||||
void* GetGPUComputeStream() const;
|
||||
Logger GetLogger() const;
|
||||
OrtAllocator* GetAllocator(const OrtMemoryInfo& memory_info) const;
|
||||
Ort::Allocator GetAllocator(const OrtMemoryInfo& memory_info) const;
|
||||
OrtKernelContext* GetOrtKernelContext() const { return ctx_; }
|
||||
void ParallelFor(void (*fn)(void*, size_t), size_t total, size_t num_batch, void* usr_data) const;
|
||||
|
||||
@@ -2690,6 +2891,13 @@ struct KernelInfoImpl : Base<T> {
|
||||
|
||||
std::string GetNodeName() const;
|
||||
Logger GetLogger() const;
|
||||
|
||||
KeyValuePairs GetConfigEntries() const;
|
||||
|
||||
std::string GetOperatorDomain() const; ///< Wraps OrtApi::KernelInfo_GetOperatorDomain
|
||||
std::string GetOperatorType() const; ///< Wraps OrtApi::KernelInfo_GetOperatorType
|
||||
int GetOperatorSinceVersion() const; ///< Wraps OrtApi::KernelInfo_GetOperatorSinceVersion
|
||||
const OrtEp* GetEp() const; ///< Wraps OrtEpApi::KernelInfo_GetEp
|
||||
};
|
||||
|
||||
} // namespace detail
|
||||
@@ -3206,5 +3414,121 @@ struct Model : detail::ModelImpl<OrtModel> {
|
||||
explicit Model(const std::vector<DomainOpsetPair>& opsets);
|
||||
#endif
|
||||
};
|
||||
|
||||
namespace detail {
|
||||
template <typename T>
|
||||
struct ConstKernelDefImpl : Base<T> {
|
||||
using B = Base<T>;
|
||||
using B::B;
|
||||
|
||||
///< Wraps OrtEpApi::KernelDef_GetOperatorType
|
||||
const char* GetOperatorType() const;
|
||||
|
||||
///< Wraps OrtEpApi::KernelDef_GetDomain
|
||||
const char* GetDomain() const;
|
||||
|
||||
///< Wraps OrtEpApi::KernelDef_GetSinceVersion
|
||||
std::pair<int, int> GetSinceVersion() const;
|
||||
|
||||
///< Wraps OrtEpApi::KernelDef_GetExecutionProvider
|
||||
const char* GetExecutionProvider() const;
|
||||
|
||||
///< Wraps OrtEpApi::KernelDef_GetInputMemType
|
||||
OrtMemType GetInputMemType(size_t input_index) const;
|
||||
|
||||
///< Wraps OrtEpApi::KernelDef_GetOutputMemType
|
||||
OrtMemType GetOutputMemType(size_t output_index) const;
|
||||
};
|
||||
} // namespace detail
|
||||
|
||||
using ConstKernelDef = detail::ConstKernelDefImpl<detail::Unowned<const OrtKernelDef>>;
|
||||
|
||||
struct KernelDef : detail::ConstKernelDefImpl<OrtKernelDef> {
|
||||
using Base = detail::ConstKernelDefImpl<OrtKernelDef>;
|
||||
using Base::Base;
|
||||
|
||||
explicit KernelDef(std::nullptr_t) {}
|
||||
explicit KernelDef(OrtKernelDef* p) : detail::ConstKernelDefImpl<OrtKernelDef>{p} {}
|
||||
|
||||
ConstKernelDef GetConst() const { return ConstKernelDef{this->p_}; }
|
||||
};
|
||||
|
||||
/** \brief Builder for OrtKernelDef.
|
||||
*
|
||||
* Used by plugin EPs to build a kernel definition.
|
||||
*/
|
||||
struct KernelDefBuilder : detail::Base<OrtKernelDefBuilder> {
|
||||
KernelDefBuilder(); ///< Wraps OrtEpApi::CreateKernelDefBuilder
|
||||
explicit KernelDefBuilder(std::nullptr_t) {} ///< Create an empty object, must be assigned a valid one to be used
|
||||
explicit KernelDefBuilder(OrtKernelDefBuilder* ort_kernel_def_builder);
|
||||
|
||||
KernelDefBuilder& SetOperatorType(const char* op_type);
|
||||
KernelDefBuilder& SetDomain(const char* domain);
|
||||
KernelDefBuilder& SetSinceVersion(int since_version_start, int since_version_end);
|
||||
KernelDefBuilder& SetExecutionProvider(const char* ep_name);
|
||||
KernelDefBuilder& SetInputMemType(size_t input_index, OrtMemType mem_type);
|
||||
KernelDefBuilder& SetOutputMemType(size_t output_index, OrtMemType mem_type);
|
||||
KernelDefBuilder& AddTypeConstraint(const char* arg_name, const OrtDataType* data_type);
|
||||
KernelDefBuilder& AddTypeConstraint(const char* arg_name, const std::vector<const OrtDataType*>& data_types);
|
||||
KernelDefBuilder& AddInputOutputAlias(int input_index, int output_index);
|
||||
KernelDefBuilder& AddInputOutputAliases(const std::vector<int>& input_indices,
|
||||
const std::vector<int>& output_indices);
|
||||
KernelDefBuilder& AddInputOutputMutableAlias(int input_index, int output_index);
|
||||
KernelDefBuilder& AddInputOutputMutableAliases(const std::vector<int>& input_indices,
|
||||
const std::vector<int>& output_indices);
|
||||
|
||||
KernelDef Build();
|
||||
};
|
||||
|
||||
/** \brief Registry for kernels supported by an EP.
|
||||
*
|
||||
* Used by plugin EPs to register definitions for supported kernels.
|
||||
*/
|
||||
struct KernelRegistry : detail::Base<OrtKernelRegistry> {
|
||||
///< Wrapper around OrtEpApi::CreateKernelRegistry
|
||||
KernelRegistry();
|
||||
|
||||
///< Create an empty object, must be assigned a valid one to be used
|
||||
explicit KernelRegistry(std::nullptr_t) {}
|
||||
|
||||
///< Take ownership of a pointer created with the C API.
|
||||
explicit KernelRegistry(OrtKernelRegistry* ort_kernel_registry);
|
||||
|
||||
///< Wraps KernelRegistry_AddKernel
|
||||
Status AddKernel(const OrtKernelDef* kernel_def, OrtKernelCreateFunc kernel_create_func,
|
||||
void* kernel_create_func_state);
|
||||
};
|
||||
|
||||
namespace detail {
|
||||
template <typename T>
|
||||
struct SharedPrePackedWeightCacheImpl : Ort::detail::Base<T> {
|
||||
using B = Ort::detail::Base<T>;
|
||||
using B::B;
|
||||
|
||||
//< Wraps SharedPrePackedWeightCache_StoreWeightData
|
||||
Status StoreWeightData(void** buffer_data_ptrs, size_t* buffer_sizes, size_t num_buffers);
|
||||
};
|
||||
} // namespace detail
|
||||
|
||||
/** \brief Convenience C++ wrapper class around a ::OrtSharedPrePackedWeightCache instance owned by ORT.
|
||||
*
|
||||
* An `OrtSharedPrePackedWeightCache*` instance is passed as an argument to OrtKernelImpl::PrePackWeight.
|
||||
* Example use:
|
||||
* OrtStatus* MyKernel::PrePackWeightImpl(OrtKernelImpl*, ..., OrtSharedPrePackedWeightCache* c_cache, ...) {
|
||||
* ...
|
||||
* if (c_cache != nullptr) {
|
||||
* Ort::UnownedSharedPrePackedWeightCache cpp_cache(c_cache);
|
||||
* Ort::Status status = cpp_cache.StoreWeightData(...);
|
||||
* }
|
||||
* ...
|
||||
* }
|
||||
*
|
||||
* \remarks OrtSharedPrePackedWeightCache is always unowned, but mutable, for EpApi users.
|
||||
*/
|
||||
using UnownedSharedPrePackedWeightCache =
|
||||
detail::SharedPrePackedWeightCacheImpl<Ort::detail::Unowned<OrtSharedPrePackedWeightCache>>;
|
||||
|
||||
///< Wraps OrtEpApi::GetEnvConfigEntries()
|
||||
Ort::KeyValuePairs GetEnvConfigEntries();
|
||||
} // namespace Ort
|
||||
#include "onnxruntime_cxx_inline.h"
|
||||
|
||||
@@ -12,6 +12,7 @@
|
||||
#include <iterator>
|
||||
#include <string>
|
||||
#include <type_traits>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
// Convert OrtStatus to Ort::Status and return
|
||||
@@ -669,9 +670,12 @@ inline void KeyValuePairs::Remove(const char* key) {
|
||||
GetApi().RemoveKeyValuePair(this->p_, key);
|
||||
}
|
||||
|
||||
inline void* SyncStream::GetHandle() const {
|
||||
namespace detail {
|
||||
template <typename T>
|
||||
inline void* SyncStreamImpl<T>::GetHandle() {
|
||||
return GetApi().SyncStream_GetHandle(this->p_);
|
||||
}
|
||||
} // namespace detail
|
||||
|
||||
namespace detail {
|
||||
template <typename T>
|
||||
@@ -743,6 +747,61 @@ inline EpDevice::EpDevice(OrtEpFactory& ep_factory, ConstHardwareDevice& hardwar
|
||||
ThrowOnError(GetEpApi().CreateEpDevice(&ep_factory, hardware_device, ep_metadata, ep_options, &p_));
|
||||
}
|
||||
|
||||
namespace detail {
|
||||
template <typename T>
|
||||
inline std::string EpAssignedSubgraphImpl<T>::GetEpName() const {
|
||||
const char* ep_name = nullptr;
|
||||
|
||||
// Returned null-terminated string will not be null if API function returns successfully.
|
||||
ThrowOnError(GetApi().EpAssignedSubgraph_GetEpName(this->p_, &ep_name));
|
||||
return std::string(ep_name);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
inline std::vector<ConstEpAssignedNode> EpAssignedSubgraphImpl<T>::GetNodes() const {
|
||||
size_t num_ep_nodes = 0;
|
||||
const OrtEpAssignedNode* const* ep_node_ptrs = nullptr;
|
||||
ThrowOnError(GetApi().EpAssignedSubgraph_GetNodes(this->p_, &ep_node_ptrs, &num_ep_nodes));
|
||||
|
||||
std::vector<ConstEpAssignedNode> ep_nodes;
|
||||
if (num_ep_nodes > 0) {
|
||||
ep_nodes.reserve(num_ep_nodes);
|
||||
for (size_t i = 0; i < num_ep_nodes; ++i) {
|
||||
ep_nodes.emplace_back(ep_node_ptrs[i]);
|
||||
}
|
||||
}
|
||||
|
||||
return ep_nodes;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
inline std::string EpAssignedNodeImpl<T>::GetName() const {
|
||||
const char* node_name = nullptr;
|
||||
|
||||
// Returned null-terminated string will not be null if API function returns successfully.
|
||||
ThrowOnError(GetApi().EpAssignedNode_GetName(this->p_, &node_name));
|
||||
return std::string(node_name);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
inline std::string EpAssignedNodeImpl<T>::GetDomain() const {
|
||||
const char* domain = nullptr;
|
||||
|
||||
// Returned null-terminated string will not be null if API function returns successfully.
|
||||
ThrowOnError(GetApi().EpAssignedNode_GetDomain(this->p_, &domain));
|
||||
return std::string(domain);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
inline std::string EpAssignedNodeImpl<T>::GetOperatorType() const {
|
||||
const char* op_type = nullptr;
|
||||
|
||||
// Returned null-terminated string will not be null if API function returns successfully.
|
||||
ThrowOnError(GetApi().EpAssignedNode_GetOperatorType(this->p_, &op_type));
|
||||
return std::string(op_type);
|
||||
}
|
||||
} // namespace detail
|
||||
|
||||
inline Env::Env(OrtLoggingLevel logging_level, _In_ const char* logid) {
|
||||
ThrowOnError(GetApi().CreateEnv(logging_level, logid, &p_));
|
||||
if (strcmp(logid, "onnxruntime-node") == 0) {
|
||||
@@ -780,6 +839,15 @@ inline Env::Env(const OrtThreadingOptions* tp_options, OrtLoggingFunction loggin
|
||||
}
|
||||
}
|
||||
|
||||
inline Env::Env(const OrtEnvCreationOptions* options) {
|
||||
ThrowOnError(GetApi().CreateEnvWithOptions(options, &p_));
|
||||
if (strcmp(options->log_id, "onnxruntime-node") == 0) {
|
||||
ThrowOnError(GetApi().SetLanguageProjection(p_, OrtLanguageProjection::ORT_PROJECTION_NODEJS));
|
||||
} else {
|
||||
ThrowOnError(GetApi().SetLanguageProjection(p_, OrtLanguageProjection::ORT_PROJECTION_CPLUSPLUS));
|
||||
}
|
||||
}
|
||||
|
||||
inline Env& Env::EnableTelemetryEvents() {
|
||||
ThrowOnError(GetApi().EnableTelemetryEvents(p_));
|
||||
return *this;
|
||||
@@ -915,6 +983,20 @@ inline OrtCompiledModelCompatibility GetModelCompatibilityForEpDevices(
|
||||
return status;
|
||||
}
|
||||
|
||||
inline AllocatedStringPtr GetCompatibilityInfoFromModelAllocated(const ORTCHAR_T* model_path, const char* ep_type,
|
||||
OrtAllocator* allocator) {
|
||||
char* compat_info = nullptr;
|
||||
ThrowOnError(GetApi().GetCompatibilityInfoFromModel(model_path, ep_type, allocator, &compat_info));
|
||||
return AllocatedStringPtr(compat_info, detail::AllocatedFree(allocator));
|
||||
}
|
||||
|
||||
inline AllocatedStringPtr GetCompatibilityInfoFromModelBytesAllocated(const void* model_data, size_t model_data_length,
|
||||
const char* ep_type, OrtAllocator* allocator) {
|
||||
char* compat_info = nullptr;
|
||||
ThrowOnError(GetApi().GetCompatibilityInfoFromModelBytes(model_data, model_data_length, ep_type, allocator, &compat_info));
|
||||
return AllocatedStringPtr(compat_info, detail::AllocatedFree(allocator));
|
||||
}
|
||||
|
||||
inline LoraAdapter LoraAdapter::CreateLoraAdapter(const std::basic_string<ORTCHAR_T>& adapter_path,
|
||||
OrtAllocator* allocator) {
|
||||
OrtLoraAdapter* p;
|
||||
@@ -990,6 +1072,11 @@ inline RunOptions& RunOptions::AddActiveLoraAdapter(const LoraAdapter& adapter)
|
||||
return *this;
|
||||
}
|
||||
|
||||
inline RunOptions& RunOptions::SetSyncStream(OrtSyncStream* stream) {
|
||||
GetApi().RunOptionsSetSyncStream(p_, stream);
|
||||
return *this;
|
||||
}
|
||||
|
||||
inline ModelCompilationOptions::ModelCompilationOptions(const Env& env, const SessionOptions& session_options) {
|
||||
ThrowOnError(GetCompileApi().CreateModelCompilationOptionsFromSessionOptions(env, session_options, &this->p_));
|
||||
}
|
||||
@@ -1500,6 +1587,18 @@ inline SessionOptionsImpl<T>& SessionOptionsImpl<T>::RegisterCustomOpsUsingFunct
|
||||
return *this;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
inline SessionOptionsImpl<T>& SessionOptionsImpl<T>::AddFreeDimensionOverride(const char* dim_denotation, int64_t dim_value) {
|
||||
ThrowOnError(GetApi().AddFreeDimensionOverrideByName(this->p_, dim_denotation, dim_value));
|
||||
return *this;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
inline SessionOptionsImpl<T>& SessionOptionsImpl<T>::AddFreeDimensionOverrideByName(const char* dim_name, int64_t dim_value) {
|
||||
ThrowOnError(GetApi().AddFreeDimensionOverrideByName(this->p_, dim_name, dim_value));
|
||||
return *this;
|
||||
}
|
||||
|
||||
/// Session
|
||||
template <typename T>
|
||||
inline size_t ConstSessionImpl<T>::GetInputCount() const {
|
||||
@@ -1582,11 +1681,13 @@ inline std::vector<ConstMemoryInfo> ConstSessionImpl<T>::GetMemoryInfoForInputs(
|
||||
|
||||
auto num_inputs = GetInputCount();
|
||||
std::vector<ConstMemoryInfo> mem_infos;
|
||||
mem_infos.resize(num_inputs);
|
||||
if (num_inputs > 0) {
|
||||
mem_infos.resize(num_inputs);
|
||||
|
||||
ThrowOnError(GetApi().SessionGetMemoryInfoForInputs(this->p_,
|
||||
reinterpret_cast<const OrtMemoryInfo**>(mem_infos.data()),
|
||||
num_inputs));
|
||||
ThrowOnError(GetApi().SessionGetMemoryInfoForInputs(this->p_,
|
||||
reinterpret_cast<const OrtMemoryInfo**>(mem_infos.data()),
|
||||
num_inputs));
|
||||
}
|
||||
|
||||
return mem_infos;
|
||||
}
|
||||
@@ -1598,11 +1699,13 @@ inline std::vector<ConstMemoryInfo> ConstSessionImpl<T>::GetMemoryInfoForOutputs
|
||||
|
||||
auto num_outputs = GetOutputCount();
|
||||
std::vector<ConstMemoryInfo> mem_infos;
|
||||
mem_infos.resize(num_outputs);
|
||||
if (num_outputs > 0) {
|
||||
mem_infos.resize(num_outputs);
|
||||
|
||||
ThrowOnError(GetApi().SessionGetMemoryInfoForOutputs(this->p_,
|
||||
reinterpret_cast<const OrtMemoryInfo**>(mem_infos.data()),
|
||||
num_outputs));
|
||||
ThrowOnError(GetApi().SessionGetMemoryInfoForOutputs(this->p_,
|
||||
reinterpret_cast<const OrtMemoryInfo**>(mem_infos.data()),
|
||||
num_outputs));
|
||||
}
|
||||
return mem_infos;
|
||||
}
|
||||
|
||||
@@ -1631,15 +1734,28 @@ template <typename T>
|
||||
inline std::vector<ConstEpDevice> ConstSessionImpl<T>::GetEpDeviceForInputs() const {
|
||||
auto num_inputs = GetInputCount();
|
||||
std::vector<ConstEpDevice> input_devices;
|
||||
input_devices.resize(num_inputs);
|
||||
|
||||
ThrowOnError(GetApi().SessionGetEpDeviceForInputs(this->p_,
|
||||
reinterpret_cast<const OrtEpDevice**>(input_devices.data()),
|
||||
num_inputs));
|
||||
|
||||
if (num_inputs > 0) {
|
||||
input_devices.resize(num_inputs);
|
||||
ThrowOnError(GetApi().SessionGetEpDeviceForInputs(this->p_,
|
||||
reinterpret_cast<const OrtEpDevice**>(input_devices.data()),
|
||||
num_inputs));
|
||||
}
|
||||
return input_devices;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
inline std::vector<ConstEpDevice> ConstSessionImpl<T>::GetEpDeviceForOutputs() const {
|
||||
auto num_outputs = GetOutputCount();
|
||||
std::vector<ConstEpDevice> output_devices;
|
||||
if (num_outputs > 0) {
|
||||
output_devices.resize(num_outputs);
|
||||
ThrowOnError(GetApi().SessionGetEpDeviceForOutputs(this->p_,
|
||||
reinterpret_cast<const OrtEpDevice**>(output_devices.data()),
|
||||
num_outputs));
|
||||
}
|
||||
return output_devices;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
inline uint64_t ConstSessionImpl<T>::GetProfilingStartTimeNs() const {
|
||||
uint64_t out;
|
||||
@@ -1714,6 +1830,23 @@ std::vector<ValueInfo> ConstSessionImpl<T>::GetOutputs() const {
|
||||
return outputs;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
inline std::vector<ConstEpAssignedSubgraph> ConstSessionImpl<T>::GetEpGraphAssignmentInfo() const {
|
||||
size_t num_ep_subgraphs = 0;
|
||||
const OrtEpAssignedSubgraph* const* ep_subgraph_ptrs = nullptr;
|
||||
ThrowOnError(GetApi().Session_GetEpGraphAssignmentInfo(this->p_, &ep_subgraph_ptrs, &num_ep_subgraphs));
|
||||
|
||||
std::vector<ConstEpAssignedSubgraph> ep_subgraphs;
|
||||
if (num_ep_subgraphs > 0) {
|
||||
ep_subgraphs.reserve(num_ep_subgraphs);
|
||||
for (size_t i = 0; i < num_ep_subgraphs; ++i) {
|
||||
ep_subgraphs.emplace_back(ep_subgraph_ptrs[i]);
|
||||
}
|
||||
}
|
||||
|
||||
return ep_subgraphs;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
inline std::vector<Value> SessionImpl<T>::Run(const RunOptions& run_options, const char* const* input_names, const Value* input_values, size_t input_count,
|
||||
const char* const* output_names, size_t output_count) {
|
||||
@@ -1975,7 +2108,12 @@ template <typename T>
|
||||
inline size_t TensorTypeAndShapeInfoImpl<T>::GetElementCount() const {
|
||||
size_t out;
|
||||
ThrowOnError(GetApi().GetTensorShapeElementCount(this->p_, &out));
|
||||
return static_cast<size_t>(out);
|
||||
return out;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
inline bool TensorTypeAndShapeInfoImpl<T>::HasShape() const {
|
||||
return GetApi().TensorTypeAndShape_HasShape(this->p_);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
@@ -1997,8 +2135,12 @@ inline void TensorTypeAndShapeInfoImpl<T>::GetSymbolicDimensions(const char** va
|
||||
|
||||
template <typename T>
|
||||
inline std::vector<const char*> TensorTypeAndShapeInfoImpl<T>::GetSymbolicDimensions() const {
|
||||
std::vector<const char*> out(GetDimensionsCount(), nullptr);
|
||||
ThrowOnError(GetApi().GetSymbolicDimensions(this->p_, out.data(), out.size()));
|
||||
std::vector<const char*> out;
|
||||
size_t dim_count = GetDimensionsCount();
|
||||
if (dim_count > 0) {
|
||||
out.resize(dim_count, nullptr);
|
||||
ThrowOnError(GetApi().GetSymbolicDimensions(this->p_, out.data(), out.size()));
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
@@ -2235,6 +2377,13 @@ inline const R* ConstValueImpl<T>::GetSparseTensorValues() const {
|
||||
|
||||
#endif
|
||||
|
||||
template <typename T>
|
||||
void ConstValueImpl<T>::GetTensorElementTypeAndShapeDataReference(ONNXTensorElementDataType& elem_type,
|
||||
Shape& shape) const {
|
||||
ThrowOnError(GetApi().GetTensorElementTypeAndShapeDataReference(this->p_, &elem_type, &shape.shape,
|
||||
&shape.shape_len));
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
void ValueImpl<T>::FillStringTensor(const char* const* s, size_t s_len) {
|
||||
ThrowOnError(GetApi().FillStringTensor(this->p_, s, s_len));
|
||||
@@ -2518,10 +2667,10 @@ inline void* KernelContext::GetGPUComputeStream() const {
|
||||
return out;
|
||||
}
|
||||
|
||||
inline OrtAllocator* KernelContext::GetAllocator(const OrtMemoryInfo& memory_info) const {
|
||||
inline Ort::Allocator KernelContext::GetAllocator(const OrtMemoryInfo& memory_info) const {
|
||||
OrtAllocator* out = nullptr;
|
||||
Ort::ThrowOnError(GetApi().KernelContext_GetAllocator(ctx_, &memory_info, &out));
|
||||
return out;
|
||||
return Ort::Allocator{out};
|
||||
}
|
||||
|
||||
inline Logger KernelContext::GetLogger() const {
|
||||
@@ -2806,6 +2955,57 @@ inline Logger KernelInfoImpl<T>::GetLogger() const {
|
||||
return Logger{out};
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
inline KeyValuePairs KernelInfoImpl<T>::GetConfigEntries() const {
|
||||
OrtKeyValuePairs* out = nullptr;
|
||||
Ort::ThrowOnError(GetApi().KernelInfo_GetConfigEntries(this->p_, &out));
|
||||
return KeyValuePairs{out};
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
inline std::string KernelInfoImpl<T>::GetOperatorDomain() const {
|
||||
size_t size = 0;
|
||||
|
||||
// Feed nullptr for the data buffer to query the true size of the string value
|
||||
Ort::ThrowOnError(GetApi().KernelInfo_GetOperatorDomain(this->p_, nullptr, &size));
|
||||
|
||||
std::string out;
|
||||
out.resize(size);
|
||||
Ort::ThrowOnError(GetApi().KernelInfo_GetOperatorDomain(this->p_, &out[0], &size));
|
||||
out.resize(size - 1); // remove the terminating character '\0'
|
||||
|
||||
return out;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
inline std::string KernelInfoImpl<T>::GetOperatorType() const {
|
||||
size_t size = 0;
|
||||
|
||||
// Feed nullptr for the data buffer to query the true size of the string value
|
||||
Ort::ThrowOnError(GetApi().KernelInfo_GetOperatorType(this->p_, nullptr, &size));
|
||||
|
||||
std::string out;
|
||||
out.resize(size);
|
||||
Ort::ThrowOnError(GetApi().KernelInfo_GetOperatorType(this->p_, &out[0], &size));
|
||||
out.resize(size - 1); // remove the terminating character '\0'
|
||||
|
||||
return out;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
inline int KernelInfoImpl<T>::GetOperatorSinceVersion() const {
|
||||
int out = 0;
|
||||
Ort::ThrowOnError(GetApi().KernelInfo_GetOperatorSinceVersion(this->p_, &out));
|
||||
return out;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
inline const OrtEp* KernelInfoImpl<T>::GetEp() const {
|
||||
const OrtEp* ep = nullptr;
|
||||
Ort::ThrowOnError(GetEpApi().KernelInfo_GetEp(this->p_, &ep));
|
||||
return ep;
|
||||
}
|
||||
|
||||
inline void attr_utils::GetAttr(const OrtKernelInfo* p, const char* name, float& out) {
|
||||
Ort::ThrowOnError(GetApi().KernelInfoGetAttribute_float(p, name, &out));
|
||||
}
|
||||
@@ -3537,4 +3737,160 @@ inline Model::Model(const std::vector<DomainOpsetPair>& opsets) {
|
||||
}
|
||||
#endif
|
||||
|
||||
namespace detail {
|
||||
template <typename T>
|
||||
inline const char* ConstKernelDefImpl<T>::GetOperatorType() const {
|
||||
return GetEpApi().KernelDef_GetOperatorType(this->p_);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
inline const char* ConstKernelDefImpl<T>::GetDomain() const {
|
||||
return GetEpApi().KernelDef_GetDomain(this->p_);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
inline std::pair<int, int> ConstKernelDefImpl<T>::GetSinceVersion() const {
|
||||
int start = 0;
|
||||
int end = 0;
|
||||
|
||||
ThrowOnError(GetEpApi().KernelDef_GetSinceVersion(this->p_, &start, &end));
|
||||
return std::pair<int, int>(start, end);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
inline const char* ConstKernelDefImpl<T>::GetExecutionProvider() const {
|
||||
return GetEpApi().KernelDef_GetExecutionProvider(this->p_);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
inline OrtMemType ConstKernelDefImpl<T>::GetInputMemType(size_t input_index) const {
|
||||
OrtMemType mem_type{};
|
||||
ThrowOnError(GetEpApi().KernelDef_GetInputMemType(this->p_, input_index, &mem_type));
|
||||
|
||||
return mem_type;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
inline OrtMemType ConstKernelDefImpl<T>::GetOutputMemType(size_t output_index) const {
|
||||
OrtMemType mem_type{};
|
||||
ThrowOnError(GetEpApi().KernelDef_GetOutputMemType(this->p_, output_index, &mem_type));
|
||||
|
||||
return mem_type;
|
||||
}
|
||||
} // namespace detail
|
||||
|
||||
inline KernelDefBuilder::KernelDefBuilder() {
|
||||
ThrowOnError(GetEpApi().CreateKernelDefBuilder(&p_));
|
||||
}
|
||||
|
||||
inline KernelDefBuilder::KernelDefBuilder(OrtKernelDefBuilder* p) : detail::Base<OrtKernelDefBuilder>{p} {
|
||||
}
|
||||
|
||||
inline KernelDefBuilder& KernelDefBuilder::SetOperatorType(const char* op_type) {
|
||||
ThrowOnError(GetEpApi().KernelDefBuilder_SetOperatorType(p_, op_type));
|
||||
return *this;
|
||||
}
|
||||
|
||||
inline KernelDefBuilder& KernelDefBuilder::SetDomain(const char* domain) {
|
||||
ThrowOnError(GetEpApi().KernelDefBuilder_SetDomain(p_, domain));
|
||||
return *this;
|
||||
}
|
||||
|
||||
inline KernelDefBuilder& KernelDefBuilder::SetSinceVersion(int since_version_start, int since_version_end) {
|
||||
ThrowOnError(GetEpApi().KernelDefBuilder_SetSinceVersion(p_, since_version_start, since_version_end));
|
||||
return *this;
|
||||
}
|
||||
|
||||
inline KernelDefBuilder& KernelDefBuilder::SetExecutionProvider(const char* ep_name) {
|
||||
ThrowOnError(GetEpApi().KernelDefBuilder_SetExecutionProvider(p_, ep_name));
|
||||
return *this;
|
||||
}
|
||||
|
||||
inline KernelDefBuilder& KernelDefBuilder::SetInputMemType(size_t input_index, OrtMemType mem_type) {
|
||||
ThrowOnError(GetEpApi().KernelDefBuilder_SetInputMemType(p_, input_index, mem_type));
|
||||
return *this;
|
||||
}
|
||||
|
||||
inline KernelDefBuilder& KernelDefBuilder::SetOutputMemType(size_t output_index, OrtMemType mem_type) {
|
||||
ThrowOnError(GetEpApi().KernelDefBuilder_SetOutputMemType(p_, output_index, mem_type));
|
||||
return *this;
|
||||
}
|
||||
|
||||
inline KernelDefBuilder& KernelDefBuilder::AddTypeConstraint(const char* arg_name,
|
||||
const OrtDataType* data_type) {
|
||||
ThrowOnError(GetEpApi().KernelDefBuilder_AddTypeConstraint(p_, arg_name, &data_type, 1));
|
||||
return *this;
|
||||
}
|
||||
|
||||
inline KernelDefBuilder& KernelDefBuilder::AddTypeConstraint(const char* arg_name,
|
||||
const std::vector<const OrtDataType*>& data_types) {
|
||||
ThrowOnError(GetEpApi().KernelDefBuilder_AddTypeConstraint(p_, arg_name, data_types.data(), data_types.size()));
|
||||
return *this;
|
||||
}
|
||||
|
||||
inline KernelDefBuilder& KernelDefBuilder::AddInputOutputAlias(int input_index, int output_index) {
|
||||
ThrowOnError(GetEpApi().KernelDefBuilder_AddInputOutputAliases(p_, &input_index, &output_index, 1));
|
||||
return *this;
|
||||
}
|
||||
|
||||
inline KernelDefBuilder& KernelDefBuilder::AddInputOutputAliases(const std::vector<int>& input_indices,
|
||||
const std::vector<int>& output_indices) {
|
||||
if (input_indices.size() != output_indices.size()) {
|
||||
ORT_CXX_API_THROW("Expecting input and output indices to have the same element count", ORT_INVALID_ARGUMENT);
|
||||
}
|
||||
|
||||
ThrowOnError(GetEpApi().KernelDefBuilder_AddInputOutputAliases(p_, input_indices.data(), output_indices.data(),
|
||||
input_indices.size()));
|
||||
return *this;
|
||||
}
|
||||
|
||||
inline KernelDefBuilder& KernelDefBuilder::AddInputOutputMutableAlias(int input_index, int output_index) {
|
||||
ThrowOnError(GetEpApi().KernelDefBuilder_AddInputOutputMutableAliases(p_, &input_index, &output_index, 1));
|
||||
return *this;
|
||||
}
|
||||
|
||||
inline KernelDefBuilder& KernelDefBuilder::AddInputOutputMutableAliases(const std::vector<int>& input_indices,
|
||||
const std::vector<int>& output_indices) {
|
||||
if (input_indices.size() != output_indices.size()) {
|
||||
ORT_CXX_API_THROW("Expecting input and output indices to have the same element count", ORT_INVALID_ARGUMENT);
|
||||
}
|
||||
|
||||
ThrowOnError(GetEpApi().KernelDefBuilder_AddInputOutputMutableAliases(p_, input_indices.data(), output_indices.data(),
|
||||
input_indices.size()));
|
||||
return *this;
|
||||
}
|
||||
|
||||
inline KernelDef KernelDefBuilder::Build() {
|
||||
OrtKernelDef* kernel_def = nullptr;
|
||||
ThrowOnError(GetEpApi().KernelDefBuilder_Build(p_, &kernel_def));
|
||||
return KernelDef(kernel_def);
|
||||
}
|
||||
|
||||
inline KernelRegistry::KernelRegistry() {
|
||||
ThrowOnError(GetEpApi().CreateKernelRegistry(&p_));
|
||||
}
|
||||
|
||||
inline KernelRegistry::KernelRegistry(OrtKernelRegistry* p) : detail::Base<OrtKernelRegistry>{p} {
|
||||
}
|
||||
|
||||
inline Status KernelRegistry::AddKernel(const OrtKernelDef* kernel_def, OrtKernelCreateFunc kernel_create_func,
|
||||
void* kernel_create_func_state) {
|
||||
return Status{GetEpApi().KernelRegistry_AddKernel(p_, kernel_def, kernel_create_func, kernel_create_func_state)};
|
||||
}
|
||||
|
||||
namespace detail {
|
||||
template <typename T>
|
||||
inline Status SharedPrePackedWeightCacheImpl<T>::StoreWeightData(void** buffer_data_ptrs, size_t* buffer_sizes,
|
||||
size_t num_buffers) {
|
||||
return Status{GetEpApi().SharedPrePackedWeightCache_StoreWeightData(this->p_, buffer_data_ptrs, buffer_sizes,
|
||||
num_buffers)};
|
||||
}
|
||||
} // namespace detail
|
||||
|
||||
inline Ort::KeyValuePairs GetEnvConfigEntries() {
|
||||
OrtKeyValuePairs* entries = nullptr;
|
||||
Ort::ThrowOnError(GetEpApi().GetEnvConfigEntries(&entries));
|
||||
|
||||
return Ort::KeyValuePairs{entries};
|
||||
}
|
||||
} // namespace Ort
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -3,12 +3,15 @@
|
||||
|
||||
#pragma once
|
||||
|
||||
// This file contains well-known keys for OrtEpDevice EP metadata entries.
|
||||
// This file contains well-known keys for OrtEpDevice and OrtHardwareDevice metadata entries.
|
||||
// It does NOT specify all available metadata keys.
|
||||
|
||||
// Key for the execution provider version string. This should be available for all plugin EPs.
|
||||
static const char* const kOrtEpDevice_EpMetadataKey_Version = "version";
|
||||
|
||||
// Key for the execution provider OS driver version.
|
||||
static const char* const kOrtEpDevice_EpMetadataKey_OSDriverVersion = "os_driver_version";
|
||||
|
||||
// Prefix for execution provider compatibility information stored in model metadata.
|
||||
// Used when generating EP context models to store compatibility strings for each EP.
|
||||
// Full key format: "ep_compatibility_info.<EP_TYPE>"
|
||||
@@ -16,3 +19,10 @@ static const char* const kOrtModelMetadata_EpCompatibilityInfoPrefix = "ep_compa
|
||||
|
||||
// Key for the execution provider library path (for dynamically loaded EPs)
|
||||
static const char* const kOrtEpDevice_EpMetadataKey_LibraryPath = "library_path";
|
||||
|
||||
// Optional metadata key to determine if a OrtHardwareDevice represents a virtual (non-hardware) device.
|
||||
// Possible values:
|
||||
// - "0": OrtHardwareDevice is not virtual (i.e., actual hardware device). This is the assumed default value
|
||||
// if this metadata key is not present.
|
||||
// - "1": OrtHardwareDevice is virtual.
|
||||
static const char* const kOrtHardwareDevice_MetadataKey_IsVirtual = "is_virtual";
|
||||
|
||||
@@ -76,6 +76,7 @@ struct Float16Impl {
|
||||
static constexpr uint16_t kPositiveQNaNBits = 0x7E00U;
|
||||
static constexpr uint16_t kNegativeQNaNBits = 0xFE00U;
|
||||
static constexpr uint16_t kMaxValueBits = 0x7BFFU; // Largest normal number
|
||||
static constexpr uint16_t kMinValueBits = 0xFBFFU; // Lowest normal number
|
||||
static constexpr uint16_t kOneBits = 0x3C00U;
|
||||
static constexpr uint16_t kMinusOneBits = 0xBC00U;
|
||||
|
||||
@@ -363,6 +364,7 @@ struct BFloat16Impl {
|
||||
static constexpr uint16_t kPositiveQNaNBits = 0x7FC1U;
|
||||
static constexpr uint16_t kNegativeQNaNBits = 0xFFC1U;
|
||||
static constexpr uint16_t kMaxValueBits = 0x7F7FU;
|
||||
static constexpr uint16_t kMinValueBits = 0xFF7FU;
|
||||
static constexpr uint16_t kRoundToNearest = 0x7FFFU;
|
||||
static constexpr uint16_t kOneBits = 0x3F80U;
|
||||
static constexpr uint16_t kMinusOneBits = 0xBF80U;
|
||||
|
||||
@@ -447,18 +447,6 @@ struct OrtLiteCustomOp : public OrtCustomOp {
|
||||
}
|
||||
#endif
|
||||
|
||||
#ifdef ORT_ROCM_CTX
|
||||
template <size_t ith_input, size_t ith_output, typename T, typename... Ts>
|
||||
static typename std::enable_if<std::is_same<T, const RocmContext&>::value, std::tuple<T, Ts...>>::type
|
||||
CreateTuple(OrtKernelContext* context, ArgPtrs& args, size_t num_input, size_t num_output, const std::string& ep) {
|
||||
thread_local RocmContext rocm_context;
|
||||
rocm_context.Init(*context);
|
||||
std::tuple<T> current = std::tuple<const RocmContext&>{rocm_context};
|
||||
auto next = CreateTuple<ith_input, ith_output, Ts...>(context, args, num_input, num_output, ep);
|
||||
return std::tuple_cat(current, next);
|
||||
}
|
||||
#endif
|
||||
|
||||
template <size_t ith_input, size_t ith_output, typename T, typename... Ts>
|
||||
static typename std::enable_if<std::is_same<T, const TensorArray*>::value, std::tuple<T, Ts...>>::type
|
||||
CreateTuple(OrtKernelContext* context, ArgPtrs& args, size_t num_input, size_t num_output, const std::string& ep) {
|
||||
@@ -674,14 +662,6 @@ struct OrtLiteCustomOp : public OrtCustomOp {
|
||||
}
|
||||
#endif
|
||||
|
||||
#ifdef ORT_ROCM_CTX
|
||||
template <typename T, typename... Ts>
|
||||
static typename std::enable_if<0 <= sizeof...(Ts) && std::is_same<T, const RocmContext&>::value>::type
|
||||
ParseArgs(std::vector<ONNXTensorElementDataType>& input_types, std::vector<ONNXTensorElementDataType>& output_types) {
|
||||
ParseArgs<Ts...>(input_types, output_types);
|
||||
}
|
||||
#endif
|
||||
|
||||
template <typename T, typename... Ts>
|
||||
static typename std::enable_if<0 <= sizeof...(Ts) && std::is_same<T, const TensorArray&>::value>::type
|
||||
ParseArgs(std::vector<ONNXTensorElementDataType>& input_types, std::vector<ONNXTensorElementDataType>& output_types) {
|
||||
|
||||
@@ -374,6 +374,12 @@ static const char* const kOrtSessionOptionsEpContextModelExternalInitializersFil
|
||||
// - "1": Gemm FastMath mode is enabled.
|
||||
static const char* const kOrtSessionOptionsMlasGemmFastMathArm64Bfloat16 = "mlas.enable_gemm_fastmath_arm64_bfloat16";
|
||||
|
||||
// Use LUT (Lookup Table) based GEMM for quantized models when available.
|
||||
// Option values:
|
||||
// - "0": Do not use LUT based GEMM. [DEFAULT]
|
||||
// - "1": Use LUT based GEMM when available.
|
||||
static const char* const kOrtSessionOptionsMlasLutGemm = "mlas.use_lut_gemm";
|
||||
|
||||
// When converting DQ + MatMul -> MatMulNBits, the accuracy level of the MatMulNBits is controlled by this option.
|
||||
// Refer to MatMulNBits op schema for more details.
|
||||
// If not provided, default is 4.
|
||||
@@ -408,3 +414,18 @@ static const char* const kOrtSessionOptionsDisableModelCompile = "session.disabl
|
||||
// Note: UNSUPPORTED models always fail regardless of this setting.
|
||||
static const char* const kOrtSessionOptionsFailOnSuboptimalCompiledModel =
|
||||
"session.fail_on_suboptimal_compiled_model";
|
||||
|
||||
// THIS OPTION IS NOT A REGULAR SESSION OPTION SINCE IT CAN BE MODIFIED AT ANY TIME
|
||||
// Meant to be used with SetEpDynamicOptions
|
||||
// options for HTP performance mode: "burst", "balanced", "default", "high_performance",
|
||||
// "high_power_saver", "low_balanced", "extreme_power_saver", "low_power_saver", "power_saver",
|
||||
// "sustained_high_performance". Default to "default".
|
||||
static const char* const kOrtEpDynamicOptionsQnnHtpPerformanceMode = "ep.dynamic.qnn_htp_performance_mode";
|
||||
|
||||
// Enables the session to record information about the subgraphs/nodes assigned to execution providers.
|
||||
// When enabled, an application may call Session_GetEpGraphAssignmentInfo() to retrieve the information.
|
||||
//
|
||||
// Option values:
|
||||
// - "0": Recording of EP graph assignment information is disabled. [DEFAULT]
|
||||
// - "1": Recording of EP graph assignment information is enabled.
|
||||
static const char* const kOrtSessionOptionsRecordEpGraphAssignmentInfo = "session.record_ep_graph_assignment_info";
|
||||
|
||||
@@ -1,731 +0,0 @@
|
||||
// Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the MIT License.
|
||||
|
||||
// This file contains the training c apis.
|
||||
|
||||
#pragma once
|
||||
#include <stdbool.h>
|
||||
#include "onnxruntime_c_api.h"
|
||||
|
||||
/** \page training_c_cpp_api Training C & C++ APIs
|
||||
*
|
||||
* Training C and C++ APIs are an extension of the \ref c_cpp_api "onnxruntime core C and C++ APIs" and should be used in conjunction with them.
|
||||
*
|
||||
* In order to train a model with onnxruntime, the following training artifacts must be generated:
|
||||
* - The training onnx model
|
||||
* - The checkpoint file
|
||||
* - The optimizer onnx model
|
||||
* - The eval onnx model model (optional)
|
||||
*
|
||||
* These training artifacts can be generated as part of an offline step using the python [utilities](https://github.com/microsoft/onnxruntime/blob/main/orttraining/orttraining/python/training/onnxblock/README.md) made available in the `onnxruntime-training` python package.
|
||||
*
|
||||
* After these artifacts have been generated, the C and C++ utilities listed in this documentation can be leveraged to perform training.
|
||||
*
|
||||
* If any problem is encountered, please create an [issue](https://github.com/microsoft/onnxruntime/issues/new) with your scenario and requirements, and we will be sure to respond and follow up on the request.
|
||||
*
|
||||
* <h1>Training C API</h1>
|
||||
*
|
||||
* ::OrtTrainingApi - Training C API functions.
|
||||
*
|
||||
* This C structure contains functions that enable users to perform training with onnxruntime.
|
||||
*
|
||||
* _Sample Code_:
|
||||
*
|
||||
* ```c
|
||||
* #include <onnxruntime_training_api.h>
|
||||
*
|
||||
* OrtApi* g_ort_api = OrtGetApiBase()->GetApi(ORT_API_VERSION);
|
||||
* OrtTrainingApi* g_ort_training_api = g_ort_api->GetTrainingApi(ORT_API_VERSION);
|
||||
*
|
||||
* OrtEnv* env = NULL;
|
||||
* g_ort_api->CreateEnv(logging_level, logid, &env);
|
||||
* OrtSessionOptions* session_options = NULL;
|
||||
* g_ort_api->CreateSessionOptions(&session_options);
|
||||
*
|
||||
* OrtCheckpointState* state = NULL;
|
||||
* g_ort_training_api->LoadCheckpoint(path_to_checkpoint, &state);
|
||||
*
|
||||
* OrtTrainingSession* training_session = NULL;
|
||||
* g_ort_training_api->CreateTrainingSession(env, session_options, training_model_path,
|
||||
* state, eval_model_path, optimizer_model_path,
|
||||
* &training_session);
|
||||
* // Training loop
|
||||
* {
|
||||
* g_ort_training_api->TrainStep(...);
|
||||
* g_ort_training_api->OptimizerStep(...);
|
||||
* g_ort_training_api->LazyResetGrad(...);
|
||||
* }
|
||||
*
|
||||
* g_ort_training_api->ExportModelForInferencing(training_session, inference_model_path, ...);
|
||||
* g_ort_training_api->SaveCheckpoint(state, path_to_checkpoint, false);
|
||||
*
|
||||
* g_ort_training_api->ReleaseTrainingSession(training_session);
|
||||
* g_ort_training_api->ReleaseCheckpointState(state);
|
||||
* ```
|
||||
*
|
||||
* > **Note**
|
||||
* > The ::OrtCheckpointState contains the entire training state that the ::OrtTrainingSession uses. As a result, the training session must always have access to the state. That is to say, the ::OrtCheckpointState instance must outlive the lifetime of the ::OrtTrainingSession instance.
|
||||
*
|
||||
* <h1>Training C++ API</h1>
|
||||
*
|
||||
* @ref TrainingCpp - Training C++ API classes and functions.
|
||||
*
|
||||
* These C++ classes and functions enable users to perform training with onnxruntime.
|
||||
*
|
||||
* _Sample Code_:
|
||||
*
|
||||
* ```cc
|
||||
* #include <onnxruntime_training_cxx_api.h>
|
||||
*
|
||||
* Ort::Env env;
|
||||
* Ort::SessionOptions session_options;
|
||||
*
|
||||
* auto state = Ort::CheckpointState::LoadCheckpoint(path_to_checkpoint);
|
||||
* auto training_session = Ort::TrainingSession(env, session_options, state, training_model_path,
|
||||
* eval_model_path, optimizer_model_path);
|
||||
*
|
||||
* // Training Loop
|
||||
* {
|
||||
* training_session.TrainStep(...);
|
||||
* training_session.OptimizerStep(...);
|
||||
* training_session.LazyResetGrad(...);
|
||||
* }
|
||||
*
|
||||
* training_session->ExportModelForInferencing(inference_model_path, ...);
|
||||
* Ort::CheckpointState::SaveCheckpoint(state, path_to_checkpoint, false);
|
||||
* ```
|
||||
* > **Note**
|
||||
* > The ::Ort::CheckpointState contains the entire training state that the ::Ort::TrainingSession uses. As a result, the training session must always have access to the state. That is to say, the ::Ort::CheckpointState instance must outlive the lifetime of the ::Ort::TrainingSession instance.
|
||||
*/
|
||||
|
||||
/** @defgroup TrainingC Ort Training C API
|
||||
* @{
|
||||
*/
|
||||
ORT_RUNTIME_CLASS(TrainingSession); // Type that enables performing training for the given user models.
|
||||
ORT_RUNTIME_CLASS(CheckpointState); // Type that holds the training states for the training session.
|
||||
|
||||
/** \brief Type of property to be added to or returned from the ::OrtCheckpointState.
|
||||
*/
|
||||
typedef enum OrtPropertyType {
|
||||
OrtIntProperty = 0,
|
||||
OrtFloatProperty = 1,
|
||||
OrtStringProperty = 2,
|
||||
} OrtPropertyType;
|
||||
|
||||
/** \brief The Training C API that holds onnxruntime training function pointers
|
||||
*
|
||||
* All the Training C API functions are defined inside this structure as pointers to functions.
|
||||
* Call OrtApi::GetTrainingApi to get a pointer to this struct.
|
||||
*
|
||||
* \nosubgrouping
|
||||
*/
|
||||
struct OrtTrainingApi {
|
||||
/// \name Accessing The Training Session State
|
||||
/// @{
|
||||
|
||||
/** \brief Load a checkpoint state from a file on disk into checkpoint_state.
|
||||
*
|
||||
* This function will parse a checkpoint file, pull relevant data and load the training
|
||||
* state into the checkpoint_state. This checkpoint state can then be used to create the
|
||||
* training session by invoking OrtTrainingApi::CreateTrainingSession. By doing so, the training
|
||||
* session will resume training from the given checkpoint state.
|
||||
* \note Note that the training session created with a checkpoint state uses this state to store the entire
|
||||
* training state (including model parameters, its gradients, the optimizer states and the properties).
|
||||
* As a result, it is required that the checkpoint state outlive the lifetime of the training session.
|
||||
* \note Note that the checkpoint file can be either the complete checkpoint or the nominal checkpoint.
|
||||
*
|
||||
* \param[in] checkpoint_path Path to the checkpoint file
|
||||
* \param[out] checkpoint_state Checkpoint state that contains the states of the training session.
|
||||
*
|
||||
* \snippet{doc} snippets.dox OrtStatus Return Value
|
||||
*
|
||||
*/
|
||||
ORT_API2_STATUS(LoadCheckpoint, _In_ const ORTCHAR_T* checkpoint_path,
|
||||
_Outptr_ OrtCheckpointState** checkpoint_state);
|
||||
|
||||
/** \brief Save the given state to a checkpoint file on disk.
|
||||
*
|
||||
* This function serializes the provided checkpoint state to a file on disk.
|
||||
* This checkpoint can later be loaded by invoking OrtTrainingApi::LoadCheckpoint to resume
|
||||
* training from this snapshot of the state.
|
||||
*
|
||||
* \param[in] checkpoint_state The checkpoint state to save.
|
||||
* \param[in] checkpoint_path Path to the checkpoint file.
|
||||
* \param[in] include_optimizer_state Flag to indicate whether to save the optimizer state or not.
|
||||
*
|
||||
* \snippet{doc} snippets.dox OrtStatus Return Value
|
||||
*
|
||||
*/
|
||||
ORT_API2_STATUS(SaveCheckpoint, _In_ OrtCheckpointState* checkpoint_state, _In_ const ORTCHAR_T* checkpoint_path,
|
||||
const bool include_optimizer_state);
|
||||
|
||||
/// @}
|
||||
|
||||
/// \name Implementing The Training Loop
|
||||
/// @{
|
||||
/** \brief Create a training session that can be used to begin or resume training.
|
||||
*
|
||||
* This function creates a training session based on the env and session options provided that can
|
||||
* begin or resume training from a given checkpoint state for the given onnx models.
|
||||
* The checkpoint state represents the parameters of the training session which will be moved
|
||||
* to the device specified by the user through the session options (if necessary).
|
||||
* The training session requires four training artifacts
|
||||
* - The training onnx model
|
||||
* - The evaluation onnx model (optional)
|
||||
* - The optimizer onnx model
|
||||
* - The checkpoint file
|
||||
*
|
||||
* These artifacts can be generated using the `onnxruntime-training` python [utility](https://github.com/microsoft/onnxruntime/blob/main/orttraining/orttraining/python/training/onnxblock/README.md).
|
||||
*
|
||||
* \param[in] env Environment to be used for the training session.
|
||||
* \param[in] options Session options that the user can customize for this training session.
|
||||
* \param[in] checkpoint_state Training states that the training session uses as a starting point for training.
|
||||
* \param[in] train_model_path Model to be used to perform training.
|
||||
* \param[in] eval_model_path Model to be used to perform evaluation.
|
||||
* \param[in] optimizer_model_path Model to be used to perform gradient descent.
|
||||
* \param[out] out Created training session.
|
||||
*
|
||||
* \snippet{doc} snippets.dox OrtStatus Return Value
|
||||
*
|
||||
*/
|
||||
ORT_API2_STATUS(CreateTrainingSession, _In_ const OrtEnv* env, _In_ const OrtSessionOptions* options,
|
||||
_Inout_ OrtCheckpointState* checkpoint_state, _In_ const ORTCHAR_T* train_model_path,
|
||||
_In_ const ORTCHAR_T* eval_model_path, _In_ const ORTCHAR_T* optimizer_model_path,
|
||||
_Outptr_result_maybenull_ OrtTrainingSession** out);
|
||||
|
||||
/** \brief Create a training session that can be used to begin or resume training.
|
||||
* This api provides a way to load all the training artifacts from buffers instead of files.
|
||||
*
|
||||
* \param[in] env Environment to be used for the training session.
|
||||
* \param[in] options Session options that the user can customize for this training session.
|
||||
* \param[in] checkpoint_state Training states that the training session uses as a starting point for training.
|
||||
* \param[in] train_model_data Buffer containing the model data to be used to perform training
|
||||
* \param[in] train_data_length Length of the buffer containing train_model_data
|
||||
* \param[in] eval_model_data Buffer containing the model data to be used to perform evaluation
|
||||
* \param[in] eval_data_length Length of the buffer containing eval_model_data
|
||||
* \param[in] optim_model_data Buffer containing the model data to be used to perform weight update
|
||||
* \param[in] optim_data_length Length of the buffer containing optim_model_data
|
||||
* \param[out] out Created training session.
|
||||
*
|
||||
*/
|
||||
ORT_API2_STATUS(CreateTrainingSessionFromBuffer, _In_ const OrtEnv* env,
|
||||
_In_ const OrtSessionOptions* options, _Inout_ OrtCheckpointState* checkpoint_state,
|
||||
_In_ const void* train_model_data, size_t train_data_length,
|
||||
_In_ const void* eval_model_data, size_t eval_data_length,
|
||||
_In_ const void* optim_model_data, size_t optim_data_length,
|
||||
_Outptr_result_maybenull_ OrtTrainingSession** out);
|
||||
|
||||
/// @}
|
||||
|
||||
/// \name Model IO Information
|
||||
/// @{
|
||||
|
||||
/** \brief Retrieves the number of user outputs in the training model.
|
||||
*
|
||||
* This function returns the number of outputs of the training model so that the user can
|
||||
* allocate space for the number of outputs when OrtTrainingApi::TrainStep is invoked.
|
||||
*
|
||||
* \param[in] sess The `this` pointer to the training session.
|
||||
* \param[out] out Number of user outputs in the training model.
|
||||
*
|
||||
* \snippet{doc} snippets.dox OrtStatus Return Value
|
||||
*
|
||||
*/
|
||||
ORT_API2_STATUS(TrainingSessionGetTrainingModelOutputCount, _In_ const OrtTrainingSession* sess, _Out_ size_t* out);
|
||||
|
||||
/** \brief Retrieves the number of user outputs in the eval model.
|
||||
*
|
||||
* This function returns the number of outputs of the eval model so that the user can
|
||||
* allocate space for the number of outputs when OrtTrainingApi::EvalStep is invoked.
|
||||
*
|
||||
* \param[in] sess The `this` pointer to the training session.
|
||||
* \param[out] out Number of user outputs in the eval model.
|
||||
*
|
||||
* \snippet{doc} snippets.dox OrtStatus Return Value
|
||||
*
|
||||
*/
|
||||
ORT_API2_STATUS(TrainingSessionGetEvalModelOutputCount, _In_ const OrtTrainingSession* sess, _Out_ size_t* out);
|
||||
|
||||
/** \brief Retrieves the names of user outputs in the training model.
|
||||
*
|
||||
* This function returns the names of outputs of the training model that can be associated with the OrtValue(s)
|
||||
* returned by the OrtTrainingApi::TrainStep function.
|
||||
*
|
||||
* \param[in] sess The `this` pointer to the training session.
|
||||
* \param[in] index Index of the output name requested.
|
||||
* \param[in] allocator Allocator to use to allocate the memory for the name.
|
||||
* \param[out] output Name of the training model output at the given index.
|
||||
*
|
||||
* \snippet{doc} snippets.dox OrtStatus Return Value
|
||||
*
|
||||
*/
|
||||
ORT_API2_STATUS(TrainingSessionGetTrainingModelOutputName, _In_ const OrtTrainingSession* sess, size_t index, _Inout_ OrtAllocator* allocator, _Outptr_ char** output);
|
||||
|
||||
/** \brief Retrieves the names of user outputs in the eval model.
|
||||
*
|
||||
* This function returns the names of outputs of the eval model that can be associated with the OrtValue(s) returned
|
||||
* by the OrtTrainingApi::EvalStep function.
|
||||
*
|
||||
* \param[in] sess The `this` pointer to the training session.
|
||||
* \param[in] index Index of the output name requested.
|
||||
* \param[in] allocator Allocator to use to allocate the memory for the name.
|
||||
* \param[out] output Name of the eval model output at the given index.
|
||||
*
|
||||
* \snippet{doc} snippets.dox OrtStatus Return Value
|
||||
*
|
||||
*/
|
||||
ORT_API2_STATUS(TrainingSessionGetEvalModelOutputName, _In_ const OrtTrainingSession* sess, size_t index, _Inout_ OrtAllocator* allocator, _Outptr_ char** output);
|
||||
|
||||
/// @}
|
||||
|
||||
/// \name Implementing The Training Loop
|
||||
/// @{
|
||||
|
||||
/** \brief Reset the gradients of all trainable parameters to zero lazily.
|
||||
*
|
||||
* This function sets the internal state of the training session such that the gradients of the trainable
|
||||
* parameters in the OrtCheckpointState will be scheduled to be reset just before the new gradients are
|
||||
* computed on the next invocation of the next OrtTrainingApi::TrainStep.
|
||||
*
|
||||
* \param[in] session The `this` pointer to the training session.
|
||||
*
|
||||
* \snippet{doc} snippets.dox OrtStatus Return Value
|
||||
*
|
||||
*/
|
||||
ORT_API2_STATUS(LazyResetGrad, _Inout_ OrtTrainingSession* session);
|
||||
|
||||
/** \brief Computes the outputs of the training model and the gradients of the trainable parameters for the given inputs
|
||||
*
|
||||
* This function performs a training step that computes the outputs of the training model and the gradients
|
||||
* of the trainable parameters for the given inputs. The train step is performed based on the training model
|
||||
* that was provided to the training session.
|
||||
* The OrtTrainingApi::TrainStep is equivalent of running forward propagation and backward propagation in a single
|
||||
* step.
|
||||
* The gradients computed are stored inside the training session state so they can be later consumed
|
||||
* by the OrtTrainingApi::OptimizerStep function.
|
||||
* The gradients can be lazily reset by invoking the OrtTrainingApi::LazyResetGrad function.
|
||||
*
|
||||
* \param[in] sess The `this` pointer to the training session.
|
||||
* \param[in] run_options Run options for this training step.
|
||||
* \param[in] inputs_len Number of user inputs to the training model.
|
||||
* \param[in] inputs The user inputs to the training model.
|
||||
* \param[in] outputs_len Number of user outputs expected from this training step.
|
||||
* \param[out] outputs User outputs computed by train step.
|
||||
*
|
||||
* \snippet{doc} snippets.dox OrtStatus Return Value
|
||||
*
|
||||
*/
|
||||
ORT_API2_STATUS(TrainStep, _Inout_ OrtTrainingSession* sess, _In_opt_ const OrtRunOptions* run_options,
|
||||
_In_ size_t inputs_len, _In_reads_(inputs_len) const OrtValue* const* inputs,
|
||||
_In_ size_t outputs_len, _Inout_updates_all_(outputs_len) OrtValue** outputs);
|
||||
|
||||
/** \brief Computes the outputs for the eval model for the given inputs
|
||||
*
|
||||
* This function performs an eval step that computes the outputs of the eval model for the given inputs.
|
||||
* The eval step is performed based on the eval model that was provided to the training session.
|
||||
*
|
||||
* \param[in] sess The `this` pointer to the training session.
|
||||
* \param[in] run_options Run options for this eval step.
|
||||
* \param[in] inputs_len Number of user inputs to the eval model.
|
||||
* \param[in] inputs The user inputs to the eval model.
|
||||
* \param[in] outputs_len Number of user outputs expected from this eval step.
|
||||
* \param[out] outputs User outputs computed by eval step.
|
||||
*
|
||||
* \snippet{doc} snippets.dox OrtStatus Return Value
|
||||
*
|
||||
*/
|
||||
ORT_API2_STATUS(EvalStep, _In_ const OrtTrainingSession* sess, _In_opt_ const OrtRunOptions* run_options,
|
||||
_In_ size_t inputs_len, _In_reads_(inputs_len) const OrtValue* const* inputs,
|
||||
_In_ size_t outputs_len, _Inout_updates_all_(outputs_len) OrtValue** outputs);
|
||||
|
||||
/** \brief Sets the learning rate for this training session.
|
||||
*
|
||||
* This function allows users to set the learning rate for the training session. The current
|
||||
* learning rate is maintained by the training session and can be overwritten by invoking
|
||||
* this function with the desired learning rate. This function should not be used when a valid
|
||||
* learning rate scheduler is registered. It should be used either to set the learning rate
|
||||
* derived from a custom learning rate scheduler or to set a constant learning rate to be used
|
||||
* throughout the training session.
|
||||
* \note Please note that this function does not set the initial learning rate that may be needed
|
||||
* by the predefined learning rate schedulers. To set the initial learning rate for learning
|
||||
* rate schedulers, please look at the function OrtTrainingApi::RegisterLinearLRScheduler.
|
||||
*
|
||||
* \param[in] sess The `this` pointer to the training session.
|
||||
* \param[in] learning_rate Desired learning rate to be set.
|
||||
*
|
||||
* \snippet{doc} snippets.dox OrtStatus Return Value
|
||||
*
|
||||
*/
|
||||
ORT_API2_STATUS(SetLearningRate, _Inout_ OrtTrainingSession* sess, _In_ float learning_rate);
|
||||
|
||||
/** \brief Gets the current learning rate for this training session.
|
||||
*
|
||||
* This function allows users to get the learning rate for the training session. The current
|
||||
* learning rate is maintained by the training session, and users can query it for the purpose
|
||||
* of implementing their own learning rate schedulers.
|
||||
*
|
||||
* \param[in] sess The `this` pointer to the training session.
|
||||
* \param[out] learning_rate Learning rate currently in use by the training session.
|
||||
*
|
||||
* \snippet{doc} snippets.dox OrtStatus Return Value
|
||||
*
|
||||
*/
|
||||
ORT_API2_STATUS(GetLearningRate, _Inout_ OrtTrainingSession* sess, _Out_ float* learning_rate);
|
||||
|
||||
/** \brief Performs the weight updates for the trainable parameters using the optimizer model.
|
||||
*
|
||||
* This function performs the weight update step that updates the trainable parameters such that they
|
||||
* take a step in the direction of their gradients (gradient descent). The optimizer step is performed
|
||||
* based on the optimizer model that was provided to the training session.
|
||||
* The updated parameters are stored inside the training state so that they can be used by the next
|
||||
* OrtTrainingApi::TrainStep function call.
|
||||
*
|
||||
* \param[in] sess The `this` pointer to the training session.
|
||||
* \param[in] run_options Run options for this optimizer step.
|
||||
*
|
||||
* \snippet{doc} snippets.dox OrtStatus Return Value
|
||||
*
|
||||
*/
|
||||
ORT_API2_STATUS(OptimizerStep, _Inout_ OrtTrainingSession* sess,
|
||||
_In_opt_ const OrtRunOptions* run_options);
|
||||
|
||||
/** \brief Registers a linear learning rate scheduler for the training session.
|
||||
*
|
||||
* Register a linear learning rate scheduler that decays the learning rate by linearly updated
|
||||
* multiplicative factor from the initial learning rate set on the training session to 0. The decay
|
||||
* is performed after the initial warm up phase where the learning rate is linearly incremented
|
||||
* from 0 to the initial learning rate provided.
|
||||
*
|
||||
* \param[in] sess The `this` pointer to the training session.
|
||||
* \param[in] warmup_step_count Warmup steps for LR warmup.
|
||||
* \param[in] total_step_count Total step count.
|
||||
* \param[in] initial_lr The initial learning rate to be used by the training session.
|
||||
*
|
||||
* \snippet{doc} snippets.dox OrtStatus Return Value
|
||||
*
|
||||
*/
|
||||
ORT_API2_STATUS(RegisterLinearLRScheduler, _Inout_ OrtTrainingSession* sess, _In_ const int64_t warmup_step_count,
|
||||
_In_ const int64_t total_step_count, _In_ const float initial_lr);
|
||||
|
||||
/** \brief Update the learning rate based on the registered learing rate scheduler.
|
||||
*
|
||||
* Takes a scheduler step that updates the learning rate that is being used by the training session.
|
||||
* This function should typically be called before invoking the optimizer step for each round,
|
||||
* or as determined necessary to update the learning rate being used by the training session.
|
||||
* \note Please note that a valid predefined learning rate scheduler must be first registered to invoke this
|
||||
* function.
|
||||
*
|
||||
* \param[in] sess The `this` pointer to the training session.
|
||||
*
|
||||
* \snippet{doc} snippets.dox OrtStatus Return Value
|
||||
*
|
||||
*/
|
||||
ORT_API2_STATUS(SchedulerStep, _Inout_ OrtTrainingSession* sess);
|
||||
|
||||
/// @}
|
||||
|
||||
/// \name Accessing The Training Session State
|
||||
/// @{
|
||||
/** \brief Retrieves the size of all the parameters.
|
||||
*
|
||||
* Calculates the total number of primitive (datatype of the parameters) elements of all the parameters in the
|
||||
* training state.
|
||||
* When trainable_only argument is true, the size is calculated for trainable params only.
|
||||
*
|
||||
* \param[in] sess The `this` pointer to the training session.
|
||||
* \param[out] out Size of all parameter elements.
|
||||
* \param[in] trainable_only Whether to skip non-trainable parameters
|
||||
*
|
||||
* \snippet{doc} snippets.dox OrtStatus Return Value
|
||||
*
|
||||
*/
|
||||
ORT_API2_STATUS(GetParametersSize, _Inout_ OrtTrainingSession* sess, _Out_ size_t* out, bool trainable_only);
|
||||
|
||||
/** \brief Copy all parameters to a contiguous buffer held by the argument parameters_buffer
|
||||
*
|
||||
* The parameters_buffer has to be of the size given by GetParametersSize api call,
|
||||
* with matching setting for the argument trainable_only. All the target parameters must be of the same
|
||||
* datatype. The OrtValue must be pre-allocated onto
|
||||
* the desired device. This is a complementary function to OrtTrainingApi::CopyBufferToParameters.
|
||||
* Parameter ordering is preserved.
|
||||
* User is responsible for allocating and freeing the resources used by the parameters_buffer.
|
||||
*
|
||||
* \param[in] sess The `this` pointer to the training session.
|
||||
* \param[in] trainable_only Whether to skip non-trainable parameters
|
||||
* \param[out] parameters_buffer The pre-allocated OrtValue buffer to copy onto.
|
||||
*
|
||||
* \snippet{doc} snippets.dox OrtStatus Return Value
|
||||
*
|
||||
*/
|
||||
ORT_API2_STATUS(CopyParametersToBuffer, _Inout_ OrtTrainingSession* sess,
|
||||
_Inout_ OrtValue* parameters_buffer, bool trainable_only);
|
||||
|
||||
/** \brief Copy parameter values from the given contiguous buffer held by parameters_buffer to the training state
|
||||
*
|
||||
* The parameters_buffer argument has to be of the size given by OrtTrainingApi::GetParametersSize api call,
|
||||
* with matching setting for trainable_only argument. All the target parameters must be of the same
|
||||
* datatype. This is a complementary function to OrtTrainingApi::CopyParametersToBuffer
|
||||
* and can be used to load updated buffer values onto the training state.
|
||||
* Parameter ordering is preserved.
|
||||
* User is responsible for allocating and freeing the resources used by the parameters_buffer.
|
||||
* In case the training session was created with a nominal checkpoint, invoking this function is required
|
||||
* to load the updated parameters onto the checkpoint to complete it.
|
||||
*
|
||||
* \param[in] sess The `this` pointer to the training session.
|
||||
* \param[in] trainable_only Whether to skip non-trainable parameters
|
||||
* \param[out] parameters_buffer The pre-allocated OrtValue buffer to copy from.
|
||||
*
|
||||
* \snippet{doc} snippets.dox OrtStatus Return Value
|
||||
*
|
||||
*/
|
||||
ORT_API2_STATUS(CopyBufferToParameters, _Inout_ OrtTrainingSession* sess,
|
||||
_Inout_ OrtValue* parameters_buffer, bool trainable_only);
|
||||
|
||||
/// @}
|
||||
|
||||
/// \name Release Training Resources
|
||||
/// @{
|
||||
|
||||
/** \brief Frees up the memory used up by the training session.
|
||||
*
|
||||
* This function frees up any memory that was allocated in the training session. The training
|
||||
* session can no longer be used after this call.
|
||||
*
|
||||
*/
|
||||
ORT_CLASS_RELEASE(TrainingSession);
|
||||
|
||||
/** \brief Frees up the memory used up by the checkpoint state.
|
||||
*
|
||||
* This function frees up any memory that was allocated in the checkpoint state. The checkpoint
|
||||
* state can no longer be used after this call.
|
||||
* \note Note that the checkpoint state must be released only after the training session has been released.
|
||||
*
|
||||
*/
|
||||
ORT_CLASS_RELEASE(CheckpointState);
|
||||
|
||||
/// @}
|
||||
|
||||
/// \name Prepare For Inferencing
|
||||
/// @{
|
||||
/** \brief Export a model that can be used for inferencing.
|
||||
*
|
||||
* If the training session was provided with an eval model, the training session can generate
|
||||
* an inference model if it knows the inference graph outputs. The input inference graph outputs
|
||||
* are used to prune the eval model so that the inference model's outputs align with the provided outputs.
|
||||
* The exported model is saved at the path provided and can be used for inferencing with InferenceSession.
|
||||
* \note Note that the function re-loads the eval model from the path provided to OrtTrainingApi::CreateTrainingSession
|
||||
* and expects that this path still be valid.
|
||||
*
|
||||
* \param[in] sess The `this` pointer to the training session.
|
||||
* \param[in] inference_model_path Path where the inference model should be serialized to.
|
||||
* \param[in] graph_outputs_len Size of the graph output names array.
|
||||
* \param[in] graph_output_names Names of the outputs that are needed in the inference model.
|
||||
*
|
||||
* \snippet{doc} snippets.dox OrtStatus Return Value
|
||||
*
|
||||
*/
|
||||
ORT_API2_STATUS(ExportModelForInferencing, _Inout_ OrtTrainingSession* sess,
|
||||
_In_ const ORTCHAR_T* inference_model_path, size_t graph_outputs_len,
|
||||
_In_reads_(graph_outputs_len) const char* const* graph_output_names);
|
||||
|
||||
/// @}
|
||||
|
||||
/// \name Training Utilities
|
||||
/// @{
|
||||
/** \brief Sets the seed used for random number generation in Onnxruntime.
|
||||
*
|
||||
* Use this function to generate reproducible results. It should be noted that completely reproducible
|
||||
* results are not guaranteed.
|
||||
*
|
||||
* \param[in] seed The seed to be set.
|
||||
*
|
||||
* \snippet{doc} snippets.dox OrtStatus Return Value
|
||||
*
|
||||
*/
|
||||
ORT_API2_STATUS(SetSeed, _In_ const int64_t seed);
|
||||
|
||||
/// @}
|
||||
|
||||
/// \name Model IO Information
|
||||
/// @{
|
||||
/** \brief Retrieves the number of user inputs in the training model.
|
||||
*
|
||||
* This function returns the number of inputs of the training model so that the user can accordingly
|
||||
* allocate the OrtValue(s) provided to the OrtTrainingApi::TrainStep function.
|
||||
*
|
||||
* \param[in] sess The `this` pointer to the training session.
|
||||
* \param[out] out Number of user inputs in the training model.
|
||||
*
|
||||
* \snippet{doc} snippets.dox OrtStatus Return Value
|
||||
*
|
||||
*/
|
||||
ORT_API2_STATUS(TrainingSessionGetTrainingModelInputCount, _In_ const OrtTrainingSession* sess, _Out_ size_t* out);
|
||||
|
||||
/** \brief Retrieves the number of user inputs in the eval model.
|
||||
*
|
||||
* This function returns the number of inputs of the eval model so that the user can accordingly
|
||||
* allocate the OrtValue(s) provided to the OrtTrainingApi::EvalStep function.
|
||||
*
|
||||
* \param[in] sess The `this` pointer to the training session.
|
||||
* \param[out] out Number of user inputs in the eval model.
|
||||
*
|
||||
* \snippet{doc} snippets.dox OrtStatus Return Value
|
||||
*
|
||||
*/
|
||||
ORT_API2_STATUS(TrainingSessionGetEvalModelInputCount, _In_ const OrtTrainingSession* sess, _Out_ size_t* out);
|
||||
|
||||
/** \brief Retrieves the name of the user input at given index in the training model.
|
||||
*
|
||||
* This function returns the names of inputs of the training model that can be associated with the
|
||||
* OrtValue(s) provided to the OrtTrainingApi::TrainStep function.
|
||||
*
|
||||
* \param[in] sess The `this` pointer to the training session.
|
||||
* \param[in] index The index of the training model input name requested.
|
||||
* \param[in] allocator The allocator to use to allocate the memory for the requested name.
|
||||
* \param[out] output Name of the user input for the training model at the given index.
|
||||
*
|
||||
* \snippet{doc} snippets.dox OrtStatus Return Value
|
||||
*
|
||||
*/
|
||||
ORT_API2_STATUS(TrainingSessionGetTrainingModelInputName, _In_ const OrtTrainingSession* sess, size_t index,
|
||||
_In_ OrtAllocator* allocator, _Outptr_ char** output);
|
||||
|
||||
/** \brief Retrieves the name of the user input at given index in the eval model.
|
||||
*
|
||||
* This function returns the names of inputs of the eval model that can be associated with the OrtValue(s) provided
|
||||
* to the OrtTrainingApi::EvalStep function.
|
||||
*
|
||||
* \param[in] sess The `this` pointer to the training session.
|
||||
* \param[in] index The index of the eval model input name requested.
|
||||
* \param[in] allocator The allocator to use to allocate the memory for the requested name.
|
||||
* \param[out] output Name of the user input for the eval model at the given index.
|
||||
*
|
||||
* \snippet{doc} snippets.dox OrtStatus Return Value
|
||||
*
|
||||
*/
|
||||
ORT_API2_STATUS(TrainingSessionGetEvalModelInputName, _In_ const OrtTrainingSession* sess, size_t index,
|
||||
_In_ OrtAllocator* allocator, _Outptr_ char** output);
|
||||
|
||||
/// @}
|
||||
|
||||
/// \name Accessing The Training Session State
|
||||
/// @{
|
||||
|
||||
/** \brief Adds or updates the given property to/in the checkpoint state.
|
||||
*
|
||||
* Runtime properties such as epoch, training step, best score, and others can be added to the checkpoint
|
||||
* state by the user by calling this function with the corresponding property name and value.
|
||||
* The given property name must be unique to be able to successfully add the property.
|
||||
*
|
||||
* \param[in] checkpoint_state The checkpoint state which should hold the property.
|
||||
* \param[in] property_name Name of the property being added or updated.
|
||||
* \param[in] property_type Type of the property associated with the given name.
|
||||
* \param[in] property_value Property value associated with the given name.
|
||||
*
|
||||
* \snippet{doc} snippets.dox OrtStatus Return Value
|
||||
*
|
||||
*/
|
||||
ORT_API2_STATUS(AddProperty, _Inout_ OrtCheckpointState* checkpoint_state,
|
||||
_In_ const char* property_name, _In_ enum OrtPropertyType property_type,
|
||||
_In_ void* property_value);
|
||||
|
||||
/** \brief Gets the property value associated with the given name from the checkpoint state.
|
||||
*
|
||||
* Gets the property value from an existing entry in the checkpoint state. The property must
|
||||
* exist in the checkpoint state to be able to retrieve it successfully.
|
||||
*
|
||||
* \param[in] checkpoint_state The checkpoint state that is currently holding the property.
|
||||
* \param[in] property_name Name of the property being retrieved.
|
||||
* \param[in] allocator Allocator used to allocate the memory for the property_value.
|
||||
* \param[out] property_type Type of the property associated with the given name.
|
||||
* \param[out] property_value Property value associated with the given name.
|
||||
*
|
||||
* \snippet{doc} snippets.dox OrtStatus Return Value
|
||||
*
|
||||
*/
|
||||
ORT_API2_STATUS(GetProperty, _In_ const OrtCheckpointState* checkpoint_state,
|
||||
_In_ const char* property_name, _Inout_ OrtAllocator* allocator,
|
||||
_Out_ enum OrtPropertyType* property_type, _Outptr_ void** property_value);
|
||||
|
||||
/// @}
|
||||
|
||||
/// \name Accessing The Training Session State
|
||||
/// @{
|
||||
|
||||
/** \brief Load a checkpoint state from a buffer into checkpoint_state.
|
||||
*
|
||||
* This function will parse a checkpoint bytes buffer, pull relevant data and load the training
|
||||
* state into the checkpoint_state. This checkpoint state can then be used to create the
|
||||
* training session by invoking OrtTrainingApi::CreateTrainingSession. By doing so, the training
|
||||
* session will resume training from the given checkpoint state.
|
||||
* \note Note that the training session created with a checkpoint state uses this state to store the entire
|
||||
* training state (including model parameters, its gradients, the optimizer states and the properties).
|
||||
* As a result, it is required that the checkpoint state outlive the lifetime of the training session.
|
||||
*
|
||||
* \param[in] checkpoint_buffer Path to the checkpoint bytes buffer.
|
||||
* \param[in] num_bytes Number of bytes in the checkpoint buffer.
|
||||
* \param[out] checkpoint_state Checkpoint state that contains the states of the training session.
|
||||
*
|
||||
* \snippet{doc} snippets.dox OrtStatus Return Value
|
||||
*
|
||||
*/
|
||||
ORT_API2_STATUS(LoadCheckpointFromBuffer, _In_ const void* checkpoint_buffer,
|
||||
_In_ const size_t num_bytes, _Outptr_ OrtCheckpointState** checkpoint_state);
|
||||
|
||||
/** \brief Retrieves the type and shape information of the parameter associated with the given parameter name.
|
||||
*
|
||||
* This function retrieves the type and shape of the parameter associated with the given parameter name.
|
||||
* The parameter must exist in the checkpoint state to be able to retrieve its type and shape information successfully.
|
||||
*
|
||||
* \param[in] checkpoint_state The checkpoint state.
|
||||
* \param[in] parameter_name Name of the parameter being retrieved.
|
||||
* \param[out] parameter_type_and_shape The type and shape of the parameter being retrieved.
|
||||
*
|
||||
* \snippet{doc} snippets.dox OrtStatus Return Value
|
||||
*
|
||||
*/
|
||||
ORT_API2_STATUS(GetParameterTypeAndShape, _In_ const OrtCheckpointState* checkpoint_state,
|
||||
_In_ const char* parameter_name, _Outptr_ OrtTensorTypeAndShapeInfo** parameter_type_and_shape);
|
||||
|
||||
/** \brief Updates the data associated with the model parameter in the checkpoint state for the given parameter name.
|
||||
*
|
||||
* This function updates a model parameter in the checkpoint state with the given parameter data.
|
||||
* The training session must be already created with the checkpoint state that contains the parameter
|
||||
* being updated. The given parameter is copied over to the registered device for the training session.
|
||||
* The parameter must exist in the checkpoint state to be able to update it successfully.
|
||||
*
|
||||
* \param[in] checkpoint_state The checkpoint state.
|
||||
* \param[in] parameter_name Name of the parameter being updated.
|
||||
* \param[in] parameter The parameter data that should replace the existing parameter data.
|
||||
*
|
||||
* \snippet{doc} snippets.dox OrtStatus Return Value
|
||||
*
|
||||
*/
|
||||
ORT_API2_STATUS(UpdateParameter, _Inout_ OrtCheckpointState* checkpoint_state,
|
||||
_In_ const char* parameter_name, _In_ OrtValue* parameter);
|
||||
|
||||
/** \brief Gets the data associated with the model parameter from the checkpoint state for the given parameter name.
|
||||
*
|
||||
* This function retrieves the model parameter data from the checkpoint state for the given parameter name.
|
||||
* The parameter is copied over and returned as an OrtValue. The training session must be already created
|
||||
* with the checkpoint state that contains the parameter being retrieved.
|
||||
* The parameter must exist in the checkpoint state to be able to retrieve it successfully.
|
||||
*
|
||||
* \param[in] checkpoint_state The checkpoint state.
|
||||
* \param[in] parameter_name Name of the parameter being retrieved.
|
||||
* \param[in] allocator Allocator used to allocate the memory for the parameter.
|
||||
* \param[out] parameter The parameter data that is retrieved from the checkpoint state.
|
||||
*
|
||||
* \snippet{doc} snippets.dox OrtStatus Return Value
|
||||
*
|
||||
*/
|
||||
ORT_API2_STATUS(GetParameter, _In_ const OrtCheckpointState* checkpoint_state,
|
||||
_In_ const char* parameter_name, _Inout_ OrtAllocator* allocator,
|
||||
_Outptr_ OrtValue** parameter);
|
||||
|
||||
/// @}
|
||||
};
|
||||
|
||||
typedef struct OrtTrainingApi OrtTrainingApi;
|
||||
|
||||
/// @}
|
||||
@@ -1,418 +0,0 @@
|
||||
// Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the MIT License.
|
||||
|
||||
#pragma once
|
||||
#include "onnxruntime_training_c_api.h"
|
||||
#include <optional>
|
||||
#include <variant>
|
||||
|
||||
namespace Ort::detail {
|
||||
|
||||
#define ORT_DECLARE_TRAINING_RELEASE(NAME) \
|
||||
void OrtRelease(Ort##NAME* ptr);
|
||||
|
||||
// These release methods must be forward declared before including onnxruntime_cxx_api.h
|
||||
// otherwise class Base won't be aware of them
|
||||
ORT_DECLARE_TRAINING_RELEASE(CheckpointState);
|
||||
ORT_DECLARE_TRAINING_RELEASE(TrainingSession);
|
||||
|
||||
} // namespace Ort::detail
|
||||
|
||||
#include "onnxruntime_cxx_api.h"
|
||||
|
||||
namespace Ort {
|
||||
|
||||
/// <summary>
|
||||
/// This function returns the C training api struct with the pointers to the ort training C functions.
|
||||
/// If using C++, please use the class instances instead of invoking the C functions directly.
|
||||
/// </summary>
|
||||
/// <returns>OrtTrainingApi struct with ort training C function pointers.</returns>
|
||||
inline const OrtTrainingApi& GetTrainingApi() { return *GetApi().GetTrainingApi(ORT_API_VERSION); }
|
||||
|
||||
namespace detail {
|
||||
|
||||
#define ORT_DEFINE_TRAINING_RELEASE(NAME) \
|
||||
inline void OrtRelease(Ort##NAME* ptr) { GetTrainingApi().Release##NAME(ptr); }
|
||||
|
||||
ORT_DEFINE_TRAINING_RELEASE(CheckpointState);
|
||||
ORT_DEFINE_TRAINING_RELEASE(TrainingSession);
|
||||
|
||||
#undef ORT_DECLARE_TRAINING_RELEASE
|
||||
#undef ORT_DEFINE_TRAINING_RELEASE
|
||||
|
||||
} // namespace detail
|
||||
|
||||
using Property = std::variant<int64_t, float, std::string>;
|
||||
|
||||
/**
|
||||
* \defgroup TrainingCpp Ort Training C++ API
|
||||
* @{
|
||||
*/
|
||||
|
||||
/** \brief Holds the state of the training session.
|
||||
*
|
||||
* This class holds the entire training session state that includes model parameters, their gradients,
|
||||
* optimizer parameters, and user properties. The Ort::TrainingSession leverages the Ort::CheckpointState
|
||||
* by accessing and updating the contained training state.
|
||||
* \note Note that the training session created with a checkpoint state uses this state to store the entire
|
||||
* training state (including model parameters, its gradients, the optimizer states and the properties).
|
||||
* The Ort::TrainingSession does not hold a copy of the Ort::CheckpointState and as a result, it is required
|
||||
* that the checkpoint state outlive the lifetime of the training session.
|
||||
* \note Note that the checkpoint state can be either the complete checkpoint state or the nominal checkpoint
|
||||
* state depending on the version provided while loading the checkpoint.
|
||||
*
|
||||
*/
|
||||
class CheckpointState : public detail::Base<OrtCheckpointState> {
|
||||
private:
|
||||
CheckpointState(OrtCheckpointState* checkpoint_state) { p_ = checkpoint_state; }
|
||||
|
||||
public:
|
||||
// Construct the checkpoint state by loading the checkpoint by calling LoadCheckpoint
|
||||
CheckpointState() = delete;
|
||||
|
||||
/// \name Accessing The Training Session State
|
||||
/// @{
|
||||
|
||||
/** \brief Load a checkpoint state from a file on disk into checkpoint_state.
|
||||
*
|
||||
* This function will parse a checkpoint file, pull relevant data and load the training
|
||||
* state and return an instance of Ort::CheckpointState. This checkpoint state can then be used to create the
|
||||
* training session by instantiating Ort::TrainingSession. By doing so, the training session will resume
|
||||
* training from the given checkpoint state.
|
||||
*
|
||||
* \param[in] path_to_checkpoint Path to the checkpoint file
|
||||
* \return Ort::CheckpointState object which holds the state of the training session parameters.
|
||||
*
|
||||
*/
|
||||
static CheckpointState LoadCheckpoint(const std::basic_string<ORTCHAR_T>& path_to_checkpoint);
|
||||
|
||||
/** \brief Load a checkpoint state from a buffer.
|
||||
*
|
||||
* This function will parse a checkpoint buffer, pull relevant data and load the training
|
||||
* state and return an instance of Ort::CheckpointState. This checkpoint state can then be used to create the
|
||||
* training session by instantiating Ort::TrainingSession. By doing so, the training session will resume
|
||||
* training from the given checkpoint state.
|
||||
*
|
||||
* \param[in] buffer Buffer containing the checkpoint data.
|
||||
* \return Ort::CheckpointState object which holds the state of the training session parameters.
|
||||
*
|
||||
*/
|
||||
static CheckpointState LoadCheckpointFromBuffer(const std::vector<uint8_t>& buffer);
|
||||
|
||||
/** \brief Save the given state to a checkpoint file on disk.
|
||||
*
|
||||
* This function serializes the provided checkpoint state to a file on disk.
|
||||
* This checkpoint can later be loaded by invoking Ort::CheckpointState::LoadCheckpoint to resume
|
||||
* training from this snapshot of the state.
|
||||
*
|
||||
* \param[in] checkpoint_state The checkpoint state to save.
|
||||
* \param[in] path_to_checkpoint Path to the checkpoint file.
|
||||
* \param[in] include_optimizer_state Flag to indicate whether to save the optimizer state or not.
|
||||
*
|
||||
*/
|
||||
static void SaveCheckpoint(const CheckpointState& checkpoint_state,
|
||||
const std::basic_string<ORTCHAR_T>& path_to_checkpoint,
|
||||
const bool include_optimizer_state = false);
|
||||
|
||||
/** \brief Adds or updates the given property to/in the checkpoint state.
|
||||
*
|
||||
* Runtime properties such as epoch, training step, best score, and others can be added to the checkpoint
|
||||
* state by the user by calling this function with the corresponding property name and value.
|
||||
* The given property name must be unique to be able to successfully add the property.
|
||||
*
|
||||
* \param[in] property_name Name of the property being added or updated.
|
||||
* \param[in] property_value Property value associated with the given name.
|
||||
*
|
||||
*/
|
||||
void AddProperty(const std::string& property_name, const Property& property_value);
|
||||
|
||||
/** \brief Gets the property value associated with the given name from the checkpoint state.
|
||||
*
|
||||
* Gets the property value from an existing entry in the checkpoint state. The property must
|
||||
* exist in the checkpoint state to be able to retrieve it successfully.
|
||||
*
|
||||
* \param[in] property_name Name of the property being retrieved.
|
||||
* \return Property value associated with the given property name.
|
||||
*
|
||||
*/
|
||||
Property GetProperty(const std::string& property_name);
|
||||
|
||||
/** \brief Updates the data associated with the model parameter in the checkpoint state for the given parameter name.
|
||||
*
|
||||
* This function updates a model parameter in the checkpoint state with the given parameter data.
|
||||
* The training session must be already created with the checkpoint state that contains the parameter
|
||||
* being updated. The given parameter is copied over to the registered device for the training session.
|
||||
* The parameter must exist in the checkpoint state to be able to update it successfully.
|
||||
*
|
||||
* \param[in] parameter_name Name of the parameter being updated.
|
||||
* \param[in] parameter The parameter data that should replace the existing parameter data.
|
||||
*
|
||||
*/
|
||||
void UpdateParameter(const std::string& parameter_name, const Value& parameter);
|
||||
|
||||
/** \brief Gets the data associated with the model parameter from the checkpoint state for the given parameter name.
|
||||
*
|
||||
* This function retrieves the model parameter data from the checkpoint state for the given parameter name.
|
||||
* The parameter is copied over to the provided OrtValue. The training session must be already created
|
||||
* with the checkpoint state that contains the parameter being retrieved.
|
||||
* The parameter must exist in the checkpoint state to be able to retrieve it successfully.
|
||||
*
|
||||
* \param[in] parameter_name Name of the parameter being retrieved.
|
||||
* \return The parameter data that is retrieved from the checkpoint state.
|
||||
*
|
||||
*/
|
||||
Value GetParameter(const std::string& parameter_name);
|
||||
|
||||
/// @}
|
||||
};
|
||||
|
||||
/** \brief Trainer class that provides training, evaluation and optimizer methods for training an ONNX models.
|
||||
*
|
||||
* The training session requires four training artifacts
|
||||
* - The training onnx model
|
||||
* - The evaluation onnx model (optional)
|
||||
* - The optimizer onnx model
|
||||
* - The checkpoint file
|
||||
*
|
||||
* These artifacts can be generated using the `onnxruntime-training` python [utility](https://github.com/microsoft/onnxruntime/blob/main/orttraining/orttraining/python/training/onnxblock/README.md).
|
||||
*
|
||||
*/
|
||||
class TrainingSession : public detail::Base<OrtTrainingSession> {
|
||||
private:
|
||||
size_t training_model_output_count_, eval_model_output_count_;
|
||||
|
||||
public:
|
||||
/// \name Constructing the Training Session
|
||||
/// @{
|
||||
/** \brief Create a training session that can be used to begin or resume training.
|
||||
*
|
||||
* This constructor instantiates the training session based on the env and session options provided that can
|
||||
* begin or resume training from a given checkpoint state for the given onnx models.
|
||||
* The checkpoint state represents the parameters of the training session which will be moved
|
||||
* to the device specified by the user through the session options (if necessary).
|
||||
*
|
||||
* \param[in] env Env to be used for the training session.
|
||||
* \param[in] session_options SessionOptions that the user can customize for this training session.
|
||||
* \param[in] checkpoint_state Training states that the training session uses as a starting point for training.
|
||||
* \param[in] train_model_path Model to be used to perform training.
|
||||
* \param[in] eval_model_path Model to be used to perform evaluation.
|
||||
* \param[in] optimizer_model_path Model to be used to perform gradient descent.
|
||||
*
|
||||
*/
|
||||
TrainingSession(const Env& env, const SessionOptions& session_options, CheckpointState& checkpoint_state,
|
||||
const std::basic_string<ORTCHAR_T>& train_model_path,
|
||||
const std::optional<std::basic_string<ORTCHAR_T>>& eval_model_path = std::nullopt,
|
||||
const std::optional<std::basic_string<ORTCHAR_T>>& optimizer_model_path = std::nullopt);
|
||||
|
||||
/** \brief Create a training session that can be used to begin or resume training.
|
||||
* This constructor allows the users to load the models from buffers instead of files.
|
||||
*
|
||||
* \param[in] env Env to be used for the training session.
|
||||
* \param[in] session_options SessionOptions that the user can customize for this training session.
|
||||
* \param[in] checkpoint_state Training states that the training session uses as a starting point for training.
|
||||
* \param[in] train_model_data Buffer containing training model data.
|
||||
* \param[in] eval_model_data Buffer containing evaluation model data.
|
||||
* \param[in] optim_model_data Buffer containing optimizer model (used for performing weight/parameter update).
|
||||
*
|
||||
*/
|
||||
TrainingSession(const Env& env, const SessionOptions& session_options, CheckpointState& checkpoint_state,
|
||||
const std::vector<uint8_t>& train_model_data, const std::vector<uint8_t>& eval_model_data = {},
|
||||
const std::vector<uint8_t>& optim_model_data = {});
|
||||
/// @}
|
||||
|
||||
/// \name Implementing The Training Loop
|
||||
/// @{
|
||||
/** \brief Computes the outputs of the training model and the gradients of the trainable parameters for the given inputs
|
||||
*
|
||||
* This function performs a training step that computes the outputs of the training model and the gradients
|
||||
* of the trainable parameters for the given inputs. The train step is performed based on the training model
|
||||
* that was provided to the training session.
|
||||
* The Ort::TrainingSession::TrainStep is equivalent of running forward propagation and backward propagation in a single
|
||||
* step.
|
||||
* The gradients computed are stored inside the training session state so they can be later consumed
|
||||
* by the Ort::TrainingSession::OptimizerStep function.
|
||||
* The gradients can be lazily reset by invoking the Ort::TrainingSession::LazyResetGrad function.
|
||||
*
|
||||
* \param[in] input_values The user inputs to the training model.
|
||||
* \return A std::vector of Ort::Value objects that represents the output of the forward pass of the training model.
|
||||
*
|
||||
*
|
||||
*/
|
||||
std::vector<Value> TrainStep(const std::vector<Value>& input_values);
|
||||
|
||||
/** \brief Reset the gradients of all trainable parameters to zero lazily.
|
||||
*
|
||||
* This function sets the internal state of the training session such that the gradients of the trainable
|
||||
* parameters in the OrtCheckpointState will be scheduled to be reset just before the new gradients are
|
||||
* computed on the next invocation of the next Ort::TrainingSession::TrainStep.
|
||||
*
|
||||
*/
|
||||
void LazyResetGrad();
|
||||
|
||||
/** \brief Computes the outputs for the eval model for the given inputs
|
||||
*
|
||||
* This function performs an eval step that computes the outputs of the eval model for the given inputs.
|
||||
* The eval step is performed based on the eval model that was provided to the training session.
|
||||
*
|
||||
* \param[in] input_values The user inputs to the eval model.
|
||||
* \return A std::vector of Ort::Value objects that represents the output of the eval pass.
|
||||
*
|
||||
*/
|
||||
std::vector<Value> EvalStep(const std::vector<Value>& input_values);
|
||||
|
||||
/** \brief Sets the learning rate for this training session.
|
||||
*
|
||||
* This function allows users to set the learning rate for the training session. The current
|
||||
* learning rate is maintained by the training session and can be overwritten by invoking
|
||||
* this function with the desired learning rate. This function should not be used when a valid
|
||||
* learning rate scheduler is registered. It should be used either to set the learning rate
|
||||
* derived from a custom learning rate scheduler or to set a constant learning rate to be used
|
||||
* throughout the training session.
|
||||
* \note Please note that this function does not set the initial learning rate that may be needed
|
||||
* by the predefined learning rate schedulers. To set the initial learning rate for learning
|
||||
* rate schedulers, please look at the function Ort::TrainingSession::RegisterLinearLRScheduler.
|
||||
*
|
||||
* \param[in] learning_rate Desired learning rate to be set.
|
||||
*
|
||||
*/
|
||||
void SetLearningRate(float learning_rate);
|
||||
|
||||
/** \brief Gets the current learning rate for this training session.
|
||||
*
|
||||
* This function allows users to get the learning rate for the training session. The current
|
||||
* learning rate is maintained by the training session, and users can query it for the purpose
|
||||
* of implementing their own learning rate schedulers.
|
||||
*
|
||||
* \return float representing the current learning rate.
|
||||
*
|
||||
*/
|
||||
float GetLearningRate() const;
|
||||
|
||||
/** \brief Registers a linear learning rate scheduler for the training session.
|
||||
*
|
||||
* Register a linear learning rate scheduler that decays the learning rate by linearly updated
|
||||
* multiplicative factor from the initial learning rate set on the training session to 0. The decay
|
||||
* is performed after the initial warm up phase where the learning rate is linearly incremented
|
||||
* from 0 to the initial learning rate provided.
|
||||
*
|
||||
* \param[in] warmup_step_count Warmup steps for LR warmup.
|
||||
* \param[in] total_step_count Total step count.
|
||||
* \param[in] initial_lr The initial learning rate to be used by the training session.
|
||||
*
|
||||
*/
|
||||
void RegisterLinearLRScheduler(int64_t warmup_step_count, int64_t total_step_count,
|
||||
float initial_lr);
|
||||
|
||||
/** \brief Update the learning rate based on the registered learing rate scheduler.
|
||||
*
|
||||
* Takes a scheduler step that updates the learning rate that is being used by the training session.
|
||||
* This function should typically be called before invoking the optimizer step for each round,
|
||||
* or as determined necessary to update the learning rate being used by the training session.
|
||||
* \note Please note that a valid predefined learning rate scheduler must be first registered to invoke this
|
||||
* function.
|
||||
*
|
||||
*/
|
||||
void SchedulerStep();
|
||||
|
||||
/** \brief Performs the weight updates for the trainable parameters using the optimizer model.
|
||||
*
|
||||
* This function performs the weight update step that updates the trainable parameters such that they
|
||||
* take a step in the direction of their gradients (gradient descent). The optimizer step is performed
|
||||
* based on the optimizer model that was provided to the training session.
|
||||
* The updated parameters are stored inside the training state so that they can be used by the next
|
||||
* Ort::TrainingSession::TrainStep function call.
|
||||
*
|
||||
*/
|
||||
void OptimizerStep();
|
||||
|
||||
/// @}
|
||||
|
||||
/// \name Prepare For Inferencing
|
||||
/// @{
|
||||
|
||||
/** \brief Export a model that can be used for inferencing.
|
||||
*
|
||||
* If the training session was provided with an eval model, the training session can generate
|
||||
* an inference model if it knows the inference graph outputs. The input inference graph outputs
|
||||
* are used to prune the eval model so that the inference model's outputs align with the provided outputs.
|
||||
* The exported model is saved at the path provided and can be used for inferencing with Ort::Session.
|
||||
* \note Note that the function re-loads the eval model from the path provided to Ort::TrainingSession
|
||||
* and expects that this path still be valid.
|
||||
*
|
||||
* \param[in] inference_model_path Path where the inference model should be serialized to.
|
||||
* \param[in] graph_output_names Names of the outputs that are needed in the inference model.
|
||||
*
|
||||
*/
|
||||
void ExportModelForInferencing(const std::basic_string<ORTCHAR_T>& inference_model_path,
|
||||
const std::vector<std::string>& graph_output_names);
|
||||
|
||||
/// @}
|
||||
|
||||
/// \name Model IO Information
|
||||
/// @{
|
||||
/** \brief Retrieves the names of the user inputs for the training and eval models.
|
||||
*
|
||||
* This function returns the names of inputs of the training or eval model that can be associated
|
||||
* with the Ort::Value(s) provided to the Ort::TrainingSession::TrainStep or Ort::TrainingSession::EvalStep
|
||||
* function.
|
||||
*
|
||||
* \param[in] training Whether the training model input names are requested or eval model input names.
|
||||
* \return Graph input names for either the training model or the eval model.
|
||||
*
|
||||
*/
|
||||
std::vector<std::string> InputNames(const bool training);
|
||||
|
||||
/** \brief Retrieves the names of the user outputs for the training and eval models.
|
||||
*
|
||||
* This function returns the names of outputs of the training or eval model that can be associated
|
||||
* with the Ort::Value(s) returned by the Ort::TrainingSession::TrainStep or Ort::TrainingSession::EvalStep
|
||||
* function.
|
||||
*
|
||||
* \param[in] training Whether the training model output names are requested or eval model output names.
|
||||
* \return Graph output names for either the training model or the eval model.
|
||||
*
|
||||
*/
|
||||
std::vector<std::string> OutputNames(const bool training);
|
||||
|
||||
/// @}
|
||||
|
||||
/// \name Accessing The Training Session State
|
||||
/// @{
|
||||
|
||||
/** \brief Returns a contiguous buffer that holds a copy of all training state parameters
|
||||
*
|
||||
* \param[in] only_trainable Whether to only copy trainable parameters or to copy all parameters.
|
||||
* \return Contiguous buffer to the model parameters.
|
||||
*
|
||||
*/
|
||||
Value ToBuffer(const bool only_trainable);
|
||||
|
||||
/** \brief Loads the training session model parameters from a contiguous buffer
|
||||
*
|
||||
* In case the training session was created with a nominal checkpoint, invoking this function is required
|
||||
* to load the updated parameters onto the checkpoint to complete it.
|
||||
*
|
||||
* \param[in] buffer Contiguous buffer to load the parameters from.
|
||||
*/
|
||||
void FromBuffer(Value& buffer);
|
||||
|
||||
/// @}
|
||||
};
|
||||
|
||||
/// \name Training Utilities
|
||||
/// @{
|
||||
/** \brief This function sets the seed for generating random numbers.
|
||||
*
|
||||
* Use this function to generate reproducible results. It should be noted that completely
|
||||
* reproducible results are not guaranteed.
|
||||
*
|
||||
* \param[in] seed Manual seed to use for random number generation.
|
||||
*/
|
||||
void SetSeed(const int64_t seed);
|
||||
/// @}
|
||||
|
||||
/// @}
|
||||
|
||||
} // namespace Ort
|
||||
|
||||
#include "onnxruntime_training_cxx_inline.h"
|
||||
@@ -1,295 +0,0 @@
|
||||
// Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the MIT License.
|
||||
|
||||
#pragma once
|
||||
#include "onnxruntime_training_c_api.h"
|
||||
#include "onnxruntime_cxx_api.h"
|
||||
|
||||
namespace Ort {
|
||||
|
||||
inline TrainingSession::TrainingSession(const Env& env, const SessionOptions& session_options,
|
||||
CheckpointState& checkpoint_state,
|
||||
const std::basic_string<ORTCHAR_T>& train_model_path,
|
||||
const std::optional<std::basic_string<ORTCHAR_T>>& eval_model_path,
|
||||
const std::optional<std::basic_string<ORTCHAR_T>>& optimizer_model_path) {
|
||||
ThrowOnError(GetTrainingApi().CreateTrainingSession(
|
||||
env, session_options, checkpoint_state,
|
||||
train_model_path.c_str(),
|
||||
eval_model_path.has_value() ? eval_model_path.value().c_str() : nullptr,
|
||||
optimizer_model_path.has_value() ? optimizer_model_path.value().c_str() : nullptr,
|
||||
&p_));
|
||||
|
||||
ThrowOnError(GetTrainingApi().TrainingSessionGetTrainingModelOutputCount(p_, &training_model_output_count_));
|
||||
|
||||
ThrowOnError(GetTrainingApi().TrainingSessionGetEvalModelOutputCount(p_, &eval_model_output_count_));
|
||||
}
|
||||
|
||||
inline TrainingSession::TrainingSession(const Env& env, const SessionOptions& session_options,
|
||||
CheckpointState& checkpoint_state,
|
||||
const std::vector<uint8_t>& train_model_data,
|
||||
const std::vector<uint8_t>& eval_model_data,
|
||||
const std::vector<uint8_t>& optim_model_data) {
|
||||
ThrowOnError(GetTrainingApi().CreateTrainingSessionFromBuffer(
|
||||
env, session_options, checkpoint_state,
|
||||
train_model_data.data(), train_model_data.size(),
|
||||
eval_model_data.data(), eval_model_data.size(),
|
||||
optim_model_data.data(), optim_model_data.size(),
|
||||
&p_));
|
||||
|
||||
ThrowOnError(GetTrainingApi().TrainingSessionGetTrainingModelOutputCount(p_, &training_model_output_count_));
|
||||
|
||||
ThrowOnError(GetTrainingApi().TrainingSessionGetEvalModelOutputCount(p_, &eval_model_output_count_));
|
||||
}
|
||||
|
||||
inline std::vector<Value> TrainingSession::TrainStep(const std::vector<Value>& input_values) {
|
||||
std::vector<Value> output_values;
|
||||
output_values.reserve(training_model_output_count_);
|
||||
for (size_t i = 0; i < training_model_output_count_; i++) output_values.emplace_back(nullptr);
|
||||
auto ort_input_values = reinterpret_cast<const OrtValue* const*>(input_values.data());
|
||||
auto ort_output_values = reinterpret_cast<OrtValue**>(output_values.data());
|
||||
RunOptions run_options;
|
||||
ThrowOnError(GetTrainingApi().TrainStep(
|
||||
p_, run_options, input_values.size(), ort_input_values,
|
||||
training_model_output_count_, ort_output_values));
|
||||
|
||||
return output_values;
|
||||
}
|
||||
|
||||
inline void TrainingSession::LazyResetGrad() {
|
||||
ThrowOnError(GetTrainingApi().LazyResetGrad(p_));
|
||||
}
|
||||
|
||||
inline std::vector<Value> TrainingSession::EvalStep(const std::vector<Value>& input_values) {
|
||||
std::vector<Value> output_values;
|
||||
output_values.reserve(eval_model_output_count_);
|
||||
for (size_t i = 0; i < eval_model_output_count_; i++) output_values.emplace_back(nullptr);
|
||||
auto ort_input_values = reinterpret_cast<const OrtValue* const*>(input_values.data());
|
||||
auto ort_output_values = reinterpret_cast<OrtValue**>(output_values.data());
|
||||
RunOptions run_options;
|
||||
ThrowOnError(GetTrainingApi().EvalStep(
|
||||
p_, run_options, input_values.size(), ort_input_values,
|
||||
eval_model_output_count_, ort_output_values));
|
||||
|
||||
return output_values;
|
||||
}
|
||||
|
||||
inline void TrainingSession::SetLearningRate(float learning_rate) {
|
||||
ThrowOnError(GetTrainingApi().SetLearningRate(p_, learning_rate));
|
||||
}
|
||||
|
||||
inline float TrainingSession::GetLearningRate() const {
|
||||
float learning_rate = 0;
|
||||
ThrowOnError(GetTrainingApi().GetLearningRate(p_, &learning_rate));
|
||||
return learning_rate;
|
||||
}
|
||||
|
||||
inline void TrainingSession::RegisterLinearLRScheduler(int64_t warmup_step_count, int64_t total_step_count,
|
||||
float initial_lr) {
|
||||
ThrowOnError(GetTrainingApi().RegisterLinearLRScheduler(p_, warmup_step_count, total_step_count,
|
||||
initial_lr));
|
||||
}
|
||||
|
||||
inline void TrainingSession::SchedulerStep() {
|
||||
ThrowOnError(GetTrainingApi().SchedulerStep(p_));
|
||||
}
|
||||
|
||||
inline void TrainingSession::OptimizerStep() {
|
||||
RunOptions run_options;
|
||||
ThrowOnError(GetTrainingApi().OptimizerStep(p_, run_options));
|
||||
}
|
||||
|
||||
inline std::vector<std::string> TrainingSession::InputNames(const bool training) {
|
||||
auto& input_count_function = training ? GetTrainingApi().TrainingSessionGetTrainingModelInputCount
|
||||
: GetTrainingApi().TrainingSessionGetEvalModelInputCount;
|
||||
auto& input_name_function = training ? GetTrainingApi().TrainingSessionGetTrainingModelInputName
|
||||
: GetTrainingApi().TrainingSessionGetEvalModelInputName;
|
||||
|
||||
size_t input_count = 0;
|
||||
ThrowOnError(input_count_function(p_, &input_count));
|
||||
std::vector<std::string> input_names(input_count);
|
||||
AllocatorWithDefaultOptions allocator;
|
||||
for (size_t index = 0; index < input_count; ++index) {
|
||||
char* input_name;
|
||||
ThrowOnError(input_name_function(p_, index, allocator, &input_name));
|
||||
input_names[index] = std::string(input_name);
|
||||
allocator.Free(input_name);
|
||||
}
|
||||
|
||||
return input_names;
|
||||
}
|
||||
|
||||
inline std::vector<std::string> TrainingSession::OutputNames(const bool training) {
|
||||
auto& output_count_function = training ? GetTrainingApi().TrainingSessionGetTrainingModelOutputCount
|
||||
: GetTrainingApi().TrainingSessionGetEvalModelOutputCount;
|
||||
auto& output_name_function = training ? GetTrainingApi().TrainingSessionGetTrainingModelOutputName
|
||||
: GetTrainingApi().TrainingSessionGetEvalModelOutputName;
|
||||
|
||||
size_t output_count = 0;
|
||||
ThrowOnError(output_count_function(p_, &output_count));
|
||||
std::vector<std::string> output_names(output_count);
|
||||
AllocatorWithDefaultOptions allocator;
|
||||
for (size_t index = 0; index < output_count; ++index) {
|
||||
char* output_name;
|
||||
ThrowOnError(output_name_function(p_, index, allocator, &output_name));
|
||||
output_names[index] = std::string(output_name);
|
||||
allocator.Free(output_name);
|
||||
}
|
||||
|
||||
return output_names;
|
||||
}
|
||||
|
||||
inline Value TrainingSession::ToBuffer(const bool only_trainable) {
|
||||
size_t buffer_size = 0U;
|
||||
ThrowOnError(GetTrainingApi().GetParametersSize(p_, &buffer_size, only_trainable));
|
||||
|
||||
std::array<int64_t, 1> buffer_shape{static_cast<int64_t>(buffer_size)};
|
||||
|
||||
AllocatorWithDefaultOptions allocator;
|
||||
Value buffer = Value::CreateTensor(allocator, buffer_shape.data(), 1U,
|
||||
ONNXTensorElementDataType::ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT);
|
||||
|
||||
ThrowOnError(GetTrainingApi().CopyParametersToBuffer(p_, buffer, only_trainable));
|
||||
|
||||
return buffer;
|
||||
}
|
||||
|
||||
inline void TrainingSession::FromBuffer(Value& buffer) {
|
||||
if (!buffer.IsTensor()) {
|
||||
ThrowStatus(Status("Incorrect buffer received. Expected a tensor buffer.", OrtErrorCode::ORT_INVALID_ARGUMENT));
|
||||
}
|
||||
|
||||
auto tensor_info = buffer.GetTensorTypeAndShapeInfo();
|
||||
auto buffer_shape = tensor_info.GetShape();
|
||||
|
||||
if (buffer_shape.size() != 1U) {
|
||||
ThrowStatus(Status("Incorrect buffer received. Expected a contiguous tensor buffer.",
|
||||
OrtErrorCode::ORT_INVALID_ARGUMENT));
|
||||
}
|
||||
|
||||
auto buffer_size = buffer_shape.front();
|
||||
|
||||
size_t session_buffer_size = 0U;
|
||||
ThrowOnError(GetTrainingApi().GetParametersSize(p_, &session_buffer_size, false));
|
||||
|
||||
if (buffer_size == static_cast<int64_t>(session_buffer_size)) {
|
||||
ThrowOnError(GetTrainingApi().CopyBufferToParameters(p_, buffer, false));
|
||||
return;
|
||||
}
|
||||
|
||||
size_t session_buffer_size_trainable_only = 0U;
|
||||
ThrowOnError(GetTrainingApi().GetParametersSize(p_, &session_buffer_size_trainable_only, true));
|
||||
|
||||
if (buffer_size == static_cast<int64_t>(session_buffer_size_trainable_only)) {
|
||||
ThrowOnError(GetTrainingApi().CopyBufferToParameters(p_, buffer, true));
|
||||
return;
|
||||
} else {
|
||||
ThrowStatus(Status("Incorrect buffer size received.", OrtErrorCode::ORT_INVALID_ARGUMENT));
|
||||
}
|
||||
}
|
||||
|
||||
inline CheckpointState CheckpointState::LoadCheckpoint(const std::basic_string<ORTCHAR_T>& path_to_checkpoint) {
|
||||
OrtCheckpointState* checkpoint_state;
|
||||
ThrowOnError(GetTrainingApi().LoadCheckpoint(path_to_checkpoint.c_str(), &checkpoint_state));
|
||||
return CheckpointState(checkpoint_state);
|
||||
}
|
||||
|
||||
inline CheckpointState CheckpointState::LoadCheckpointFromBuffer(const std::vector<uint8_t>& buffer) {
|
||||
OrtCheckpointState* checkpoint_state;
|
||||
ThrowOnError(GetTrainingApi().LoadCheckpointFromBuffer(buffer.data(), buffer.size(), &checkpoint_state));
|
||||
return CheckpointState(checkpoint_state);
|
||||
}
|
||||
|
||||
inline void CheckpointState::SaveCheckpoint(const CheckpointState& checkpoint_states,
|
||||
const std::basic_string<ORTCHAR_T>& path_to_checkpoint,
|
||||
const bool include_optimizer_state) {
|
||||
ThrowOnError(GetTrainingApi().SaveCheckpoint(checkpoint_states, path_to_checkpoint.c_str(),
|
||||
include_optimizer_state));
|
||||
}
|
||||
|
||||
inline void TrainingSession::ExportModelForInferencing(const std::basic_string<ORTCHAR_T>& inference_model_path,
|
||||
const std::vector<std::string>& graph_output_names) {
|
||||
std::vector<const char*> output_names;
|
||||
output_names.reserve(graph_output_names.size());
|
||||
for (const auto& output_name : graph_output_names) {
|
||||
output_names.push_back(output_name.c_str());
|
||||
}
|
||||
ThrowOnError(GetTrainingApi().ExportModelForInferencing(
|
||||
p_, inference_model_path.c_str(), graph_output_names.size(), output_names.data()));
|
||||
}
|
||||
|
||||
inline void SetSeed(const int64_t seed) {
|
||||
ThrowOnError(GetTrainingApi().SetSeed(seed));
|
||||
}
|
||||
|
||||
inline void CheckpointState::AddProperty(const std::string& property_name, const Property& property_value) {
|
||||
if (std::holds_alternative<int64_t>(property_value)) {
|
||||
int64_t value = std::get<int64_t>(property_value);
|
||||
void* value_p = &value;
|
||||
ThrowOnError(GetTrainingApi().AddProperty(p_, property_name.c_str(), OrtPropertyType::OrtIntProperty, value_p));
|
||||
} else if (std::holds_alternative<float>(property_value)) {
|
||||
float value = std::get<float>(property_value);
|
||||
void* value_p = &value;
|
||||
ThrowOnError(GetTrainingApi().AddProperty(p_, property_name.c_str(), OrtPropertyType::OrtFloatProperty, value_p));
|
||||
} else if (std::holds_alternative<std::string>(property_value)) {
|
||||
std::string value = std::get<std::string>(property_value);
|
||||
auto buffer = std::make_unique<char[]>(value.length() + 1);
|
||||
memcpy(buffer.get(), value.c_str(), value.length());
|
||||
// AddProperty takes a char* and calls PropertyBag::AddProperty which takes a std::string. The data will be
|
||||
// copied at that point so buffer can free the local allocation once the call is made.
|
||||
ThrowOnError(GetTrainingApi().AddProperty(p_, property_name.c_str(), OrtPropertyType::OrtStringProperty,
|
||||
buffer.get()));
|
||||
} else {
|
||||
ThrowStatus(Status("Unknown property type received.", OrtErrorCode::ORT_INVALID_ARGUMENT));
|
||||
}
|
||||
}
|
||||
|
||||
inline Property CheckpointState::GetProperty(const std::string& property_name) {
|
||||
void* property_value = nullptr;
|
||||
OrtPropertyType property_type;
|
||||
|
||||
AllocatorWithDefaultOptions allocator;
|
||||
ThrowOnError(GetTrainingApi().GetProperty(p_, property_name.c_str(), allocator, &property_type, &property_value));
|
||||
|
||||
Property property;
|
||||
|
||||
switch (property_type) {
|
||||
case OrtPropertyType::OrtIntProperty: {
|
||||
auto value_p = reinterpret_cast<int64_t*>(property_value);
|
||||
property = *value_p;
|
||||
allocator.Free(property_value);
|
||||
break;
|
||||
}
|
||||
case OrtPropertyType::OrtFloatProperty: {
|
||||
auto value_p = reinterpret_cast<float*>(property_value);
|
||||
property = *value_p;
|
||||
allocator.Free(property_value);
|
||||
break;
|
||||
}
|
||||
case OrtPropertyType::OrtStringProperty: {
|
||||
auto value_p = reinterpret_cast<char*>(property_value);
|
||||
property = std::string(value_p);
|
||||
allocator.Free(property_value);
|
||||
break;
|
||||
}
|
||||
default: {
|
||||
ThrowStatus(Status("Unknown property type received.", OrtErrorCode::ORT_INVALID_ARGUMENT));
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return property;
|
||||
}
|
||||
|
||||
inline void CheckpointState::UpdateParameter(const std::string& parameter_name, const Value& parameter) {
|
||||
ThrowOnError(GetTrainingApi().UpdateParameter(p_, parameter_name.c_str(), parameter));
|
||||
}
|
||||
|
||||
inline Value CheckpointState::GetParameter(const std::string& parameter_name) {
|
||||
AllocatorWithDefaultOptions allocator;
|
||||
OrtValue* parameter;
|
||||
ThrowOnError(GetTrainingApi().GetParameter(p_, parameter_name.c_str(), allocator, ¶meter));
|
||||
|
||||
return Value{parameter};
|
||||
}
|
||||
|
||||
} // namespace Ort
|
||||
Reference in New Issue
Block a user