From b89b7c1b128de3ab773e4abfd7a1629ad0767c95 Mon Sep 17 00:00:00 2001 From: Danny Date: Wed, 8 Jul 2026 12:43:20 +0700 Subject: [PATCH 1/4] [Go] Add Go bindings for ONNX Runtime C API Idiomatic Go wrapper for the ORT C API via CGO, matching the pattern of existing language bindings (rust/, java/, csharp/). Covers session management, typed/untyped tensor I/O, string tensors, IO binding, model metadata, run options, session options with CUDA and TensorRT execution providers, context cancellation, and concurrent inference. Tested with real Qwen3-Embedding-0.6B ONNX INT8 model. --- go/.gitignore | 1 + go/.golangci.yml | 32 + go/go.mod | 3 + go/onnxruntime/cshim.c | 488 ++ go/onnxruntime/cshim.h | 194 + go/onnxruntime/doc.go | 10 + go/onnxruntime/errors.go | 93 + go/onnxruntime/iobinding.go | 194 + go/onnxruntime/iobinding_test.go | 101 + go/onnxruntime/load_unix.go | 57 + go/onnxruntime/load_windows.go | 27 + go/onnxruntime/metadata.go | 154 + go/onnxruntime/metadata_test.go | 70 + go/onnxruntime/onnxruntime_c_api.h | 8682 +++++++++++++++++++++++ go/onnxruntime/onnxruntime_ep_c_api.h | 3102 ++++++++ go/onnxruntime/onnxruntime_error_code.h | 92 + go/onnxruntime/options.go | 227 + go/onnxruntime/options_test.go | 135 + go/onnxruntime/ort.go | 170 + go/onnxruntime/ort_test.go | 96 + go/onnxruntime/qwen3_test.go | 192 + go/onnxruntime/run_options.go | 63 + go/onnxruntime/run_options_test.go | 64 + go/onnxruntime/session.go | 497 ++ go/onnxruntime/session_test.go | 331 + go/onnxruntime/string_tensor.go | 116 + go/onnxruntime/string_tensor_test.go | 84 + go/onnxruntime/tensor.go | 331 + go/onnxruntime/tensor_test.go | 183 + go/onnxruntime/types.go | 129 + go/testdata/add_f32.onnx | Bin 0 -> 100 bytes go/testdata/gen_models.py | 82 + go/testdata/kvconcat.onnx | Bin 0 -> 181 bytes go/testdata/matmul_dynamic.onnx | Bin 0 -> 116 bytes 34 files changed, 16000 insertions(+) create mode 100644 go/.gitignore create mode 100644 go/.golangci.yml create mode 100644 go/go.mod create mode 100644 go/onnxruntime/cshim.c create mode 100644 go/onnxruntime/cshim.h create mode 100644 go/onnxruntime/doc.go create mode 100644 go/onnxruntime/errors.go create mode 100644 go/onnxruntime/iobinding.go create mode 100644 go/onnxruntime/iobinding_test.go create mode 100644 go/onnxruntime/load_unix.go create mode 100644 go/onnxruntime/load_windows.go create mode 100644 go/onnxruntime/metadata.go create mode 100644 go/onnxruntime/metadata_test.go create mode 100644 go/onnxruntime/onnxruntime_c_api.h create mode 100644 go/onnxruntime/onnxruntime_ep_c_api.h create mode 100644 go/onnxruntime/onnxruntime_error_code.h create mode 100644 go/onnxruntime/options.go create mode 100644 go/onnxruntime/options_test.go create mode 100644 go/onnxruntime/ort.go create mode 100644 go/onnxruntime/ort_test.go create mode 100644 go/onnxruntime/qwen3_test.go create mode 100644 go/onnxruntime/run_options.go create mode 100644 go/onnxruntime/run_options_test.go create mode 100644 go/onnxruntime/session.go create mode 100644 go/onnxruntime/session_test.go create mode 100644 go/onnxruntime/string_tensor.go create mode 100644 go/onnxruntime/string_tensor_test.go create mode 100644 go/onnxruntime/tensor.go create mode 100644 go/onnxruntime/tensor_test.go create mode 100644 go/onnxruntime/types.go create mode 100644 go/testdata/add_f32.onnx create mode 100644 go/testdata/gen_models.py create mode 100644 go/testdata/kvconcat.onnx create mode 100644 go/testdata/matmul_dynamic.onnx diff --git a/go/.gitignore b/go/.gitignore new file mode 100644 index 0000000000000..304506af20618 --- /dev/null +++ b/go/.gitignore @@ -0,0 +1 @@ +.ort-lib/ diff --git a/go/.golangci.yml b/go/.golangci.yml new file mode 100644 index 0000000000000..6659ce74583a8 --- /dev/null +++ b/go/.golangci.yml @@ -0,0 +1,32 @@ +version: "2" + +run: + modules-download-mode: readonly + timeout: 10m + +linters: + enable: + - gocritic + - govet + - errcheck + - staticcheck + - unused + - ineffassign + - gosec + settings: + gocritic: + enabled-checks: + - evalOrder + disabled-checks: + - ifElseChain + - dupSubExpr + gosec: + excludes: + # CGO pointer passing is intentional in this binding library. + - G103 + exclusions: + rules: + - linters: + - errcheck + - gosec + path: _test\.go diff --git a/go/go.mod b/go/go.mod new file mode 100644 index 0000000000000..46a56cf101a03 --- /dev/null +++ b/go/go.mod @@ -0,0 +1,3 @@ +module github.com/microsoft/onnxruntime/go + +go 1.26 diff --git a/go/onnxruntime/cshim.c b/go/onnxruntime/cshim.c new file mode 100644 index 0000000000000..82354e59a5321 --- /dev/null +++ b/go/onnxruntime/cshim.c @@ -0,0 +1,488 @@ +#include "cshim.h" + +static const OrtApi *g_api = NULL; + +int ort_init_api(void *get_api_base_fn) { + const OrtApiBase *base = ((const OrtApiBase *(*)(void))get_api_base_fn)(); + if (base == NULL) return 1; + g_api = base->GetApi(ORT_GO_API_VERSION); + return g_api == NULL ? 2 : 0; +} + +// Environment + +OrtStatusPtr ort_CreateEnv(OrtLoggingLevel level, const char *logid, OrtEnv **out) { + return g_api->CreateEnv(level, logid, out); +} + +void ort_ReleaseEnv(OrtEnv *env) { + g_api->ReleaseEnv(env); +} + +// Session options + +OrtStatusPtr ort_CreateSessionOptions(OrtSessionOptions **out) { + return g_api->CreateSessionOptions(out); +} + +void ort_ReleaseSessionOptions(OrtSessionOptions *opts) { + g_api->ReleaseSessionOptions(opts); +} + +OrtStatusPtr ort_SetIntraOpNumThreads(OrtSessionOptions *opts, int n) { + return g_api->SetIntraOpNumThreads(opts, n); +} + +OrtStatusPtr ort_SetInterOpNumThreads(OrtSessionOptions *opts, int n) { + return g_api->SetInterOpNumThreads(opts, n); +} + +OrtStatusPtr ort_SetSessionGraphOptimizationLevel(OrtSessionOptions *opts, GraphOptimizationLevel level) { + return g_api->SetSessionGraphOptimizationLevel(opts, level); +} + +OrtStatusPtr ort_AddSessionConfigEntry(OrtSessionOptions *opts, const char *key, const char *value) { + return g_api->AddSessionConfigEntry(opts, key, value); +} + +OrtStatusPtr ort_SessionOptionsAppendExecutionProvider( + OrtSessionOptions *opts, const char *provider_name, + const char *const *keys, const char *const *values, size_t num_keys) { + return g_api->SessionOptionsAppendExecutionProvider(opts, provider_name, keys, values, num_keys); +} + +// CUDA V2 provider options + +OrtStatusPtr ort_CreateCUDAProviderOptions(OrtCUDAProviderOptionsV2 **out) { + return g_api->CreateCUDAProviderOptions(out); +} + +OrtStatusPtr ort_UpdateCUDAProviderOptions(OrtCUDAProviderOptionsV2 *opts, + const char *const *keys, const char *const *values, size_t num_keys) { + return g_api->UpdateCUDAProviderOptions(opts, keys, values, num_keys); +} + +OrtStatusPtr ort_SessionOptionsAppendExecutionProvider_CUDA_V2( + OrtSessionOptions *opts, const OrtCUDAProviderOptionsV2 *cuda_opts) { + return g_api->SessionOptionsAppendExecutionProvider_CUDA_V2(opts, cuda_opts); +} + +void ort_ReleaseCUDAProviderOptions(OrtCUDAProviderOptionsV2 *opts) { + g_api->ReleaseCUDAProviderOptions(opts); +} + +// TensorRT V2 provider options + +OrtStatusPtr ort_CreateTensorRTProviderOptions(OrtTensorRTProviderOptionsV2 **out) { + return g_api->CreateTensorRTProviderOptions(out); +} + +OrtStatusPtr ort_UpdateTensorRTProviderOptions(OrtTensorRTProviderOptionsV2 *opts, + const char *const *keys, const char *const *values, size_t num_keys) { + return g_api->UpdateTensorRTProviderOptions(opts, keys, values, num_keys); +} + +OrtStatusPtr ort_SessionOptionsAppendExecutionProvider_TensorRT_V2( + OrtSessionOptions *opts, const OrtTensorRTProviderOptionsV2 *trt_opts) { + return g_api->SessionOptionsAppendExecutionProvider_TensorRT_V2(opts, trt_opts); +} + +void ort_ReleaseTensorRTProviderOptions(OrtTensorRTProviderOptionsV2 *opts) { + g_api->ReleaseTensorRTProviderOptions(opts); +} + +// Session + +OrtStatusPtr ort_CreateSession(const OrtEnv *env, const char *model_path, + const OrtSessionOptions *opts, OrtSession **out) { + return g_api->CreateSession(env, model_path, opts, out); +} + +OrtStatusPtr ort_CreateSessionFromArray(const OrtEnv *env, const void *model_data, + size_t model_data_length, const OrtSessionOptions *opts, OrtSession **out) { + return g_api->CreateSessionFromArray(env, model_data, model_data_length, opts, out); +} + +void ort_ReleaseSession(OrtSession *session) { + g_api->ReleaseSession(session); +} + +// Session introspection + +OrtStatusPtr ort_SessionGetInputCount(const OrtSession *session, size_t *out) { + return g_api->SessionGetInputCount(session, out); +} + +OrtStatusPtr ort_SessionGetOutputCount(const OrtSession *session, size_t *out) { + return g_api->SessionGetOutputCount(session, out); +} + +OrtStatusPtr ort_SessionGetInputName(const OrtSession *session, size_t index, + OrtAllocator *allocator, char **out) { + return g_api->SessionGetInputName(session, index, allocator, out); +} + +OrtStatusPtr ort_SessionGetOutputName(const OrtSession *session, size_t index, + OrtAllocator *allocator, char **out) { + return g_api->SessionGetOutputName(session, index, allocator, out); +} + +OrtStatusPtr ort_SessionGetInputTypeInfo(const OrtSession *session, size_t index, + OrtTypeInfo **out) { + return g_api->SessionGetInputTypeInfo(session, index, out); +} + +OrtStatusPtr ort_SessionGetOutputTypeInfo(const OrtSession *session, size_t index, + OrtTypeInfo **out) { + return g_api->SessionGetOutputTypeInfo(session, index, out); +} + +// Type info + +OrtStatusPtr ort_CastTypeInfoToTensorInfo(const OrtTypeInfo *type_info, + const OrtTensorTypeAndShapeInfo **out) { + return g_api->CastTypeInfoToTensorInfo(type_info, out); +} + +OrtStatusPtr ort_GetOnnxTypeFromTypeInfo(const OrtTypeInfo *type_info, enum ONNXType *out) { + return g_api->GetOnnxTypeFromTypeInfo(type_info, out); +} + +void ort_ReleaseTypeInfo(OrtTypeInfo *info) { + g_api->ReleaseTypeInfo(info); +} + +// Tensor type and shape info + +OrtStatusPtr ort_GetTensorElementType(const OrtTensorTypeAndShapeInfo *info, + enum ONNXTensorElementDataType *out) { + return g_api->GetTensorElementType(info, out); +} + +OrtStatusPtr ort_GetDimensionsCount(const OrtTensorTypeAndShapeInfo *info, size_t *out) { + return g_api->GetDimensionsCount(info, out); +} + +OrtStatusPtr ort_GetDimensions(const OrtTensorTypeAndShapeInfo *info, + int64_t *dim_values, size_t dim_values_length) { + return g_api->GetDimensions(info, dim_values, dim_values_length); +} + +OrtStatusPtr ort_GetTensorShapeElementCount(const OrtTensorTypeAndShapeInfo *info, size_t *out) { + return g_api->GetTensorShapeElementCount(info, out); +} + +void ort_ReleaseTensorTypeAndShapeInfo(OrtTensorTypeAndShapeInfo *info) { + g_api->ReleaseTensorTypeAndShapeInfo(info); +} + +// Tensor creation and data access + +OrtStatusPtr ort_CreateTensorWithDataAsOrtValue(const OrtMemoryInfo *info, + void *p_data, size_t p_data_len, const int64_t *shape, size_t shape_len, + ONNXTensorElementDataType type, OrtValue **out) { + return g_api->CreateTensorWithDataAsOrtValue(info, p_data, p_data_len, shape, shape_len, type, out); +} + +OrtStatusPtr ort_CreateTensorAsOrtValue(OrtAllocator *allocator, + const int64_t *shape, size_t shape_len, ONNXTensorElementDataType type, OrtValue **out) { + return g_api->CreateTensorAsOrtValue(allocator, shape, shape_len, type, out); +} + +OrtStatusPtr ort_GetTensorMutableData(OrtValue *value, void **out) { + return g_api->GetTensorMutableData(value, out); +} + +OrtStatusPtr ort_GetTensorTypeAndShape(const OrtValue *value, + OrtTensorTypeAndShapeInfo **out) { + return g_api->GetTensorTypeAndShape(value, out); +} + +OrtStatusPtr ort_IsTensor(const OrtValue *value, int *out) { + return g_api->IsTensor(value, out); +} + +void ort_ReleaseValue(OrtValue *value) { + g_api->ReleaseValue(value); +} + +// Memory and allocator + +OrtStatusPtr ort_CreateCpuMemoryInfo(enum OrtAllocatorType type, + enum OrtMemType mem_type, OrtMemoryInfo **out) { + return g_api->CreateCpuMemoryInfo(type, mem_type, out); +} + +void ort_ReleaseMemoryInfo(OrtMemoryInfo *info) { + g_api->ReleaseMemoryInfo(info); +} + +OrtStatusPtr ort_GetAllocatorWithDefaultOptions(OrtAllocator **out) { + return g_api->GetAllocatorWithDefaultOptions(out); +} + +// Run + +OrtStatusPtr ort_Run(OrtSession *session, const OrtRunOptions *run_options, + const char *const *input_names, const OrtValue *const *inputs, size_t input_len, + const char *const *output_names, size_t output_names_len, OrtValue **outputs) { + return g_api->Run(session, run_options, input_names, inputs, input_len, + output_names, output_names_len, outputs); +} + +// Run options + +OrtStatusPtr ort_CreateRunOptions(OrtRunOptions **out) { + return g_api->CreateRunOptions(out); +} + +void ort_ReleaseRunOptions(OrtRunOptions *opts) { + g_api->ReleaseRunOptions(opts); +} + +OrtStatusPtr ort_RunOptionsSetTerminate(OrtRunOptions *opts) { + return g_api->RunOptionsSetTerminate(opts); +} + +OrtStatusPtr ort_RunOptionsUnsetTerminate(OrtRunOptions *opts) { + return g_api->RunOptionsUnsetTerminate(opts); +} + +OrtStatusPtr ort_RunOptionsSetRunLogVerbosityLevel(OrtRunOptions *opts, int level) { + return g_api->RunOptionsSetRunLogVerbosityLevel(opts, level); +} + +OrtStatusPtr ort_RunOptionsSetRunLogSeverityLevel(OrtRunOptions *opts, int level) { + return g_api->RunOptionsSetRunLogSeverityLevel(opts, level); +} + +OrtStatusPtr ort_RunOptionsSetRunTag(OrtRunOptions *opts, const char *tag) { + return g_api->RunOptionsSetRunTag(opts, tag); +} + +OrtStatusPtr ort_AddRunConfigEntry(OrtRunOptions *opts, const char *key, const char *value) { + return g_api->AddRunConfigEntry(opts, key, value); +} + +// Model metadata + +OrtStatusPtr ort_SessionGetModelMetadata(const OrtSession *session, OrtModelMetadata **out) { + return g_api->SessionGetModelMetadata(session, out); +} + +OrtStatusPtr ort_ModelMetadataGetProducerName(const OrtModelMetadata *meta, + OrtAllocator *allocator, char **out) { + return g_api->ModelMetadataGetProducerName(meta, allocator, out); +} + +OrtStatusPtr ort_ModelMetadataGetGraphName(const OrtModelMetadata *meta, + OrtAllocator *allocator, char **out) { + return g_api->ModelMetadataGetGraphName(meta, allocator, out); +} + +OrtStatusPtr ort_ModelMetadataGetDomain(const OrtModelMetadata *meta, + OrtAllocator *allocator, char **out) { + return g_api->ModelMetadataGetDomain(meta, allocator, out); +} + +OrtStatusPtr ort_ModelMetadataGetDescription(const OrtModelMetadata *meta, + OrtAllocator *allocator, char **out) { + return g_api->ModelMetadataGetDescription(meta, allocator, out); +} + +OrtStatusPtr ort_ModelMetadataGetGraphDescription(const OrtModelMetadata *meta, + OrtAllocator *allocator, char **out) { + return g_api->ModelMetadataGetGraphDescription(meta, allocator, out); +} + +OrtStatusPtr ort_ModelMetadataGetVersion(const OrtModelMetadata *meta, int64_t *out) { + return g_api->ModelMetadataGetVersion(meta, out); +} + +OrtStatusPtr ort_ModelMetadataGetCustomMetadataMapKeys(const OrtModelMetadata *meta, + OrtAllocator *allocator, char ***keys, int64_t *count) { + return g_api->ModelMetadataGetCustomMetadataMapKeys(meta, allocator, keys, count); +} + +OrtStatusPtr ort_ModelMetadataLookupCustomMetadataMap(const OrtModelMetadata *meta, + OrtAllocator *allocator, const char *key, char **out) { + return g_api->ModelMetadataLookupCustomMetadataMap(meta, allocator, key, out); +} + +void ort_ReleaseModelMetadata(OrtModelMetadata *meta) { + g_api->ReleaseModelMetadata(meta); +} + +// IO Binding + +OrtStatusPtr ort_CreateIoBinding(OrtSession *session, OrtIoBinding **out) { + return g_api->CreateIoBinding(session, out); +} + +OrtStatusPtr ort_BindInput(OrtIoBinding *binding, const char *name, const OrtValue *val) { + return g_api->BindInput(binding, name, val); +} + +OrtStatusPtr ort_BindOutput(OrtIoBinding *binding, const char *name, const OrtValue *val) { + return g_api->BindOutput(binding, name, val); +} + +OrtStatusPtr ort_BindOutputToDevice(OrtIoBinding *binding, const char *name, + const OrtMemoryInfo *mem_info) { + return g_api->BindOutputToDevice(binding, name, mem_info); +} + +OrtStatusPtr ort_RunWithBinding(OrtSession *session, const OrtRunOptions *run_options, + const OrtIoBinding *binding) { + return g_api->RunWithBinding(session, run_options, binding); +} + +OrtStatusPtr ort_GetBoundOutputNames(const OrtIoBinding *binding, OrtAllocator *allocator, + char **buffer, size_t **lengths, size_t *count) { + return g_api->GetBoundOutputNames(binding, allocator, buffer, lengths, count); +} + +OrtStatusPtr ort_GetBoundOutputValues(const OrtIoBinding *binding, OrtAllocator *allocator, + OrtValue ***output, size_t *count) { + return g_api->GetBoundOutputValues(binding, allocator, output, count); +} + +void ort_ClearBoundInputs(OrtIoBinding *binding) { + g_api->ClearBoundInputs(binding); +} + +void ort_ClearBoundOutputs(OrtIoBinding *binding) { + g_api->ClearBoundOutputs(binding); +} + +void ort_ReleaseIoBinding(OrtIoBinding *binding) { + g_api->ReleaseIoBinding(binding); +} + +// String tensors + +OrtStatusPtr ort_FillStringTensor(OrtValue *value, const char *const *s, size_t s_len) { + return g_api->FillStringTensor(value, s, s_len); +} + +OrtStatusPtr ort_GetStringTensorDataLength(const OrtValue *value, size_t *len) { + return g_api->GetStringTensorDataLength(value, len); +} + +OrtStatusPtr ort_GetStringTensorContent(const OrtValue *value, void *s, size_t s_len, + size_t *offsets, size_t offsets_len) { + return g_api->GetStringTensorContent(value, s, s_len, offsets, offsets_len); +} + +OrtStatusPtr ort_GetStringTensorElementLength(const OrtValue *value, size_t index, size_t *out) { + return g_api->GetStringTensorElementLength(value, index, out); +} + +OrtStatusPtr ort_GetStringTensorElement(const OrtValue *value, size_t s_len, size_t index, void *s) { + return g_api->GetStringTensorElement(value, s_len, index, s); +} + +// Additional session options + +OrtStatusPtr ort_CloneSessionOptions(const OrtSessionOptions *in, OrtSessionOptions **out) { + return g_api->CloneSessionOptions(in, out); +} + +OrtStatusPtr ort_EnableMemPattern(OrtSessionOptions *opts) { + return g_api->EnableMemPattern(opts); +} + +OrtStatusPtr ort_DisableMemPattern(OrtSessionOptions *opts) { + return g_api->DisableMemPattern(opts); +} + +OrtStatusPtr ort_EnableCpuMemArena(OrtSessionOptions *opts) { + return g_api->EnableCpuMemArena(opts); +} + +OrtStatusPtr ort_DisableCpuMemArena(OrtSessionOptions *opts) { + return g_api->DisableCpuMemArena(opts); +} + +OrtStatusPtr ort_EnableProfiling(OrtSessionOptions *opts, const char *prefix) { + return g_api->EnableProfiling(opts, prefix); +} + +OrtStatusPtr ort_DisableProfiling(OrtSessionOptions *opts) { + return g_api->DisableProfiling(opts); +} + +OrtStatusPtr ort_AddFreeDimensionOverride(OrtSessionOptions *opts, + const char *dim_denotation, int64_t dim_value) { + return g_api->AddFreeDimensionOverride(opts, dim_denotation, dim_value); +} + +OrtStatusPtr ort_AddFreeDimensionOverrideByName(OrtSessionOptions *opts, + const char *dim_name, int64_t dim_value) { + return g_api->AddFreeDimensionOverrideByName(opts, dim_name, dim_value); +} + +OrtStatusPtr ort_SetSessionExecutionMode(OrtSessionOptions *opts, ExecutionMode mode) { + return g_api->SetSessionExecutionMode(opts, mode); +} + +OrtStatusPtr ort_AddInitializer(OrtSessionOptions *opts, const char *name, const OrtValue *val) { + return g_api->AddInitializer(opts, name, val); +} + +// Session profiling + +OrtStatusPtr ort_SessionEndProfiling(OrtSession *session, OrtAllocator *allocator, char **out) { + return g_api->SessionEndProfiling(session, allocator, out); +} + +// Value type + +OrtStatusPtr ort_GetValueType(const OrtValue *value, enum ONNXType *out) { + return g_api->GetValueType(value, out); +} + +OrtStatusPtr ort_GetValueCount(const OrtValue *value, size_t *out) { + return g_api->GetValueCount(value, out); +} + +OrtStatusPtr ort_GetValue(const OrtValue *value, int index, OrtAllocator *allocator, OrtValue **out) { + return g_api->GetValue(value, index, allocator, out); +} + +// Memory info + +OrtStatusPtr ort_CreateMemoryInfo(const char *name, enum OrtAllocatorType type, + int id, enum OrtMemType mem_type, OrtMemoryInfo **out) { + return g_api->CreateMemoryInfo(name, type, id, mem_type, out); +} + +// Error handling + +OrtErrorCode ort_GetErrorCode(const OrtStatus *status) { + return g_api->GetErrorCode(status); +} + +const char *ort_GetErrorMessage(const OrtStatus *status) { + return g_api->GetErrorMessage(status); +} + +void ort_ReleaseStatus(OrtStatus *status) { + g_api->ReleaseStatus(status); +} + +// Provider enumeration + +OrtStatusPtr ort_GetAvailableProviders(char ***out, int *count) { + return g_api->GetAvailableProviders(out, count); +} + +OrtStatusPtr ort_ReleaseAvailableProviders(char **ptr, int count) { + return g_api->ReleaseAvailableProviders(ptr, count); +} + +// Allocator free + +void ort_AllocatorFree(OrtAllocator *allocator, void *ptr) { + allocator->Free(allocator, ptr); +} diff --git a/go/onnxruntime/cshim.h b/go/onnxruntime/cshim.h new file mode 100644 index 0000000000000..41935d1bc008c --- /dev/null +++ b/go/onnxruntime/cshim.h @@ -0,0 +1,194 @@ +#ifndef ORT_GO_CSHIM_H +#define ORT_GO_CSHIM_H + +#include "onnxruntime_c_api.h" + +// Minimum ORT API version required by these bindings. +// Covers all functions used, including UpdateCUDAProviderOptions (1.16). +#define ORT_GO_API_VERSION 17 + +// Initialize the global OrtApi pointer from a resolved OrtGetApiBase function. +// Returns 0 on success, 1 if apiBase is NULL, 2 if GetApi returns NULL. +int ort_init_api(void *get_api_base_fn); + +// Environment +OrtStatusPtr ort_CreateEnv(OrtLoggingLevel level, const char *logid, OrtEnv **out); +void ort_ReleaseEnv(OrtEnv *env); + +// Session options +OrtStatusPtr ort_CreateSessionOptions(OrtSessionOptions **out); +void ort_ReleaseSessionOptions(OrtSessionOptions *opts); +OrtStatusPtr ort_SetIntraOpNumThreads(OrtSessionOptions *opts, int n); +OrtStatusPtr ort_SetInterOpNumThreads(OrtSessionOptions *opts, int n); +OrtStatusPtr ort_SetSessionGraphOptimizationLevel(OrtSessionOptions *opts, GraphOptimizationLevel level); +OrtStatusPtr ort_AddSessionConfigEntry(OrtSessionOptions *opts, const char *key, const char *value); +OrtStatusPtr ort_SessionOptionsAppendExecutionProvider( + OrtSessionOptions *opts, const char *provider_name, + const char *const *keys, const char *const *values, size_t num_keys); + +// CUDA V2 provider options +OrtStatusPtr ort_CreateCUDAProviderOptions(OrtCUDAProviderOptionsV2 **out); +OrtStatusPtr ort_UpdateCUDAProviderOptions(OrtCUDAProviderOptionsV2 *opts, + const char *const *keys, const char *const *values, size_t num_keys); +OrtStatusPtr ort_SessionOptionsAppendExecutionProvider_CUDA_V2( + OrtSessionOptions *opts, const OrtCUDAProviderOptionsV2 *cuda_opts); +void ort_ReleaseCUDAProviderOptions(OrtCUDAProviderOptionsV2 *opts); + +// TensorRT V2 provider options +OrtStatusPtr ort_CreateTensorRTProviderOptions(OrtTensorRTProviderOptionsV2 **out); +OrtStatusPtr ort_UpdateTensorRTProviderOptions(OrtTensorRTProviderOptionsV2 *opts, + const char *const *keys, const char *const *values, size_t num_keys); +OrtStatusPtr ort_SessionOptionsAppendExecutionProvider_TensorRT_V2( + OrtSessionOptions *opts, const OrtTensorRTProviderOptionsV2 *trt_opts); +void ort_ReleaseTensorRTProviderOptions(OrtTensorRTProviderOptionsV2 *opts); + +// Session +OrtStatusPtr ort_CreateSession(const OrtEnv *env, const char *model_path, + const OrtSessionOptions *opts, OrtSession **out); +OrtStatusPtr ort_CreateSessionFromArray(const OrtEnv *env, const void *model_data, + size_t model_data_length, const OrtSessionOptions *opts, OrtSession **out); +void ort_ReleaseSession(OrtSession *session); + +// Session introspection +OrtStatusPtr ort_SessionGetInputCount(const OrtSession *session, size_t *out); +OrtStatusPtr ort_SessionGetOutputCount(const OrtSession *session, size_t *out); +OrtStatusPtr ort_SessionGetInputName(const OrtSession *session, size_t index, + OrtAllocator *allocator, char **out); +OrtStatusPtr ort_SessionGetOutputName(const OrtSession *session, size_t index, + OrtAllocator *allocator, char **out); +OrtStatusPtr ort_SessionGetInputTypeInfo(const OrtSession *session, size_t index, + OrtTypeInfo **out); +OrtStatusPtr ort_SessionGetOutputTypeInfo(const OrtSession *session, size_t index, + OrtTypeInfo **out); + +// Type info +OrtStatusPtr ort_CastTypeInfoToTensorInfo(const OrtTypeInfo *type_info, + const OrtTensorTypeAndShapeInfo **out); +OrtStatusPtr ort_GetOnnxTypeFromTypeInfo(const OrtTypeInfo *type_info, enum ONNXType *out); +void ort_ReleaseTypeInfo(OrtTypeInfo *info); + +// Tensor type and shape info +OrtStatusPtr ort_GetTensorElementType(const OrtTensorTypeAndShapeInfo *info, + enum ONNXTensorElementDataType *out); +OrtStatusPtr ort_GetDimensionsCount(const OrtTensorTypeAndShapeInfo *info, size_t *out); +OrtStatusPtr ort_GetDimensions(const OrtTensorTypeAndShapeInfo *info, + int64_t *dim_values, size_t dim_values_length); +OrtStatusPtr ort_GetTensorShapeElementCount(const OrtTensorTypeAndShapeInfo *info, size_t *out); +void ort_ReleaseTensorTypeAndShapeInfo(OrtTensorTypeAndShapeInfo *info); + +// Tensor creation and data access +OrtStatusPtr ort_CreateTensorWithDataAsOrtValue(const OrtMemoryInfo *info, + void *p_data, size_t p_data_len, const int64_t *shape, size_t shape_len, + ONNXTensorElementDataType type, OrtValue **out); +OrtStatusPtr ort_CreateTensorAsOrtValue(OrtAllocator *allocator, + const int64_t *shape, size_t shape_len, ONNXTensorElementDataType type, OrtValue **out); +OrtStatusPtr ort_GetTensorMutableData(OrtValue *value, void **out); +OrtStatusPtr ort_GetTensorTypeAndShape(const OrtValue *value, + OrtTensorTypeAndShapeInfo **out); +OrtStatusPtr ort_IsTensor(const OrtValue *value, int *out); +void ort_ReleaseValue(OrtValue *value); + +// Memory and allocator +OrtStatusPtr ort_CreateCpuMemoryInfo(enum OrtAllocatorType type, + enum OrtMemType mem_type, OrtMemoryInfo **out); +void ort_ReleaseMemoryInfo(OrtMemoryInfo *info); +OrtStatusPtr ort_GetAllocatorWithDefaultOptions(OrtAllocator **out); + +// Run +OrtStatusPtr ort_Run(OrtSession *session, const OrtRunOptions *run_options, + const char *const *input_names, const OrtValue *const *inputs, size_t input_len, + const char *const *output_names, size_t output_names_len, OrtValue **outputs); + +// Run options +OrtStatusPtr ort_CreateRunOptions(OrtRunOptions **out); +void ort_ReleaseRunOptions(OrtRunOptions *opts); +OrtStatusPtr ort_RunOptionsSetTerminate(OrtRunOptions *opts); +OrtStatusPtr ort_RunOptionsUnsetTerminate(OrtRunOptions *opts); +OrtStatusPtr ort_RunOptionsSetRunLogVerbosityLevel(OrtRunOptions *opts, int level); +OrtStatusPtr ort_RunOptionsSetRunLogSeverityLevel(OrtRunOptions *opts, int level); +OrtStatusPtr ort_RunOptionsSetRunTag(OrtRunOptions *opts, const char *tag); +OrtStatusPtr ort_AddRunConfigEntry(OrtRunOptions *opts, const char *key, const char *value); + +// Model metadata +OrtStatusPtr ort_SessionGetModelMetadata(const OrtSession *session, OrtModelMetadata **out); +OrtStatusPtr ort_ModelMetadataGetProducerName(const OrtModelMetadata *meta, + OrtAllocator *allocator, char **out); +OrtStatusPtr ort_ModelMetadataGetGraphName(const OrtModelMetadata *meta, + OrtAllocator *allocator, char **out); +OrtStatusPtr ort_ModelMetadataGetDomain(const OrtModelMetadata *meta, + OrtAllocator *allocator, char **out); +OrtStatusPtr ort_ModelMetadataGetDescription(const OrtModelMetadata *meta, + OrtAllocator *allocator, char **out); +OrtStatusPtr ort_ModelMetadataGetGraphDescription(const OrtModelMetadata *meta, + OrtAllocator *allocator, char **out); +OrtStatusPtr ort_ModelMetadataGetVersion(const OrtModelMetadata *meta, int64_t *out); +OrtStatusPtr ort_ModelMetadataGetCustomMetadataMapKeys(const OrtModelMetadata *meta, + OrtAllocator *allocator, char ***keys, int64_t *count); +OrtStatusPtr ort_ModelMetadataLookupCustomMetadataMap(const OrtModelMetadata *meta, + OrtAllocator *allocator, const char *key, char **out); +void ort_ReleaseModelMetadata(OrtModelMetadata *meta); + +// IO Binding +OrtStatusPtr ort_CreateIoBinding(OrtSession *session, OrtIoBinding **out); +OrtStatusPtr ort_BindInput(OrtIoBinding *binding, const char *name, const OrtValue *val); +OrtStatusPtr ort_BindOutput(OrtIoBinding *binding, const char *name, const OrtValue *val); +OrtStatusPtr ort_BindOutputToDevice(OrtIoBinding *binding, const char *name, + const OrtMemoryInfo *mem_info); +OrtStatusPtr ort_RunWithBinding(OrtSession *session, const OrtRunOptions *run_options, + const OrtIoBinding *binding); +OrtStatusPtr ort_GetBoundOutputNames(const OrtIoBinding *binding, OrtAllocator *allocator, + char **buffer, size_t **lengths, size_t *count); +OrtStatusPtr ort_GetBoundOutputValues(const OrtIoBinding *binding, OrtAllocator *allocator, + OrtValue ***output, size_t *count); +void ort_ClearBoundInputs(OrtIoBinding *binding); +void ort_ClearBoundOutputs(OrtIoBinding *binding); +void ort_ReleaseIoBinding(OrtIoBinding *binding); + +// String tensors +OrtStatusPtr ort_FillStringTensor(OrtValue *value, const char *const *s, size_t s_len); +OrtStatusPtr ort_GetStringTensorDataLength(const OrtValue *value, size_t *len); +OrtStatusPtr ort_GetStringTensorContent(const OrtValue *value, void *s, size_t s_len, + size_t *offsets, size_t offsets_len); +OrtStatusPtr ort_GetStringTensorElementLength(const OrtValue *value, size_t index, size_t *out); +OrtStatusPtr ort_GetStringTensorElement(const OrtValue *value, size_t s_len, size_t index, void *s); + +// Additional session options +OrtStatusPtr ort_CloneSessionOptions(const OrtSessionOptions *in, OrtSessionOptions **out); +OrtStatusPtr ort_EnableMemPattern(OrtSessionOptions *opts); +OrtStatusPtr ort_DisableMemPattern(OrtSessionOptions *opts); +OrtStatusPtr ort_EnableCpuMemArena(OrtSessionOptions *opts); +OrtStatusPtr ort_DisableCpuMemArena(OrtSessionOptions *opts); +OrtStatusPtr ort_EnableProfiling(OrtSessionOptions *opts, const char *prefix); +OrtStatusPtr ort_DisableProfiling(OrtSessionOptions *opts); +OrtStatusPtr ort_AddFreeDimensionOverride(OrtSessionOptions *opts, + const char *dim_denotation, int64_t dim_value); +OrtStatusPtr ort_AddFreeDimensionOverrideByName(OrtSessionOptions *opts, + const char *dim_name, int64_t dim_value); +OrtStatusPtr ort_SetSessionExecutionMode(OrtSessionOptions *opts, ExecutionMode mode); +OrtStatusPtr ort_AddInitializer(OrtSessionOptions *opts, const char *name, const OrtValue *val); + +// Session profiling +OrtStatusPtr ort_SessionEndProfiling(OrtSession *session, OrtAllocator *allocator, char **out); + +// Value type +OrtStatusPtr ort_GetValueType(const OrtValue *value, enum ONNXType *out); +OrtStatusPtr ort_GetValueCount(const OrtValue *value, size_t *out); +OrtStatusPtr ort_GetValue(const OrtValue *value, int index, OrtAllocator *allocator, OrtValue **out); + +// Memory info +OrtStatusPtr ort_CreateMemoryInfo(const char *name, enum OrtAllocatorType type, + int id, enum OrtMemType mem_type, OrtMemoryInfo **out); + +// Error handling +OrtErrorCode ort_GetErrorCode(const OrtStatus *status); +const char *ort_GetErrorMessage(const OrtStatus *status); +void ort_ReleaseStatus(OrtStatus *status); + +// Provider enumeration +OrtStatusPtr ort_GetAvailableProviders(char ***out, int *count); +OrtStatusPtr ort_ReleaseAvailableProviders(char **ptr, int count); + +// Allocator free (for freeing names returned by SessionGetInputName etc.) +void ort_AllocatorFree(OrtAllocator *allocator, void *ptr); + +#endif // ORT_GO_CSHIM_H diff --git a/go/onnxruntime/doc.go b/go/onnxruntime/doc.go new file mode 100644 index 0000000000000..95da67ff29635 --- /dev/null +++ b/go/onnxruntime/doc.go @@ -0,0 +1,10 @@ +// Package onnxruntime provides Go bindings for the ONNX Runtime C API. +// +// Call SetSharedLibraryPath and Init before using any other function. +// Call Shutdown when done to release resources. +// +// A Session wraps an ORT inference session. It is safe for concurrent +// use: multiple goroutines may call Run on the same Session simultaneously. +// Each Tensor, SessionOptions, and other handle must not be used concurrently +// unless documented otherwise. +package onnxruntime diff --git a/go/onnxruntime/errors.go b/go/onnxruntime/errors.go new file mode 100644 index 0000000000000..e5d71c0e72b42 --- /dev/null +++ b/go/onnxruntime/errors.go @@ -0,0 +1,93 @@ +package onnxruntime + +/* +#include "cshim.h" +*/ +import "C" +import ( + "fmt" + "unsafe" +) + +// ErrorCode mirrors OrtErrorCode from the C API. +type ErrorCode int + +const ( + ErrorCodeOK ErrorCode = 0 + ErrorCodeFail ErrorCode = 1 + ErrorCodeInvalidArgument ErrorCode = 2 + ErrorCodeNoSuchFile ErrorCode = 3 + ErrorCodeNoModel ErrorCode = 4 + ErrorCodeEngineError ErrorCode = 5 + ErrorCodeRuntimeException ErrorCode = 6 + ErrorCodeInvalidProtobuf ErrorCode = 7 + ErrorCodeModelLoaded ErrorCode = 8 + ErrorCodeNotImplemented ErrorCode = 9 + ErrorCodeInvalidGraph ErrorCode = 10 + ErrorCodeEPFail ErrorCode = 11 +) + +func (c ErrorCode) String() string { + switch c { + case ErrorCodeOK: + return "OK" + case ErrorCodeFail: + return "Fail" + case ErrorCodeInvalidArgument: + return "InvalidArgument" + case ErrorCodeNoSuchFile: + return "NoSuchFile" + case ErrorCodeNoModel: + return "NoModel" + case ErrorCodeEngineError: + return "EngineError" + case ErrorCodeRuntimeException: + return "RuntimeException" + case ErrorCodeInvalidProtobuf: + return "InvalidProtobuf" + case ErrorCodeModelLoaded: + return "ModelLoaded" + case ErrorCodeNotImplemented: + return "NotImplemented" + case ErrorCodeInvalidGraph: + return "InvalidGraph" + case ErrorCodeEPFail: + return "EPFail" + default: + return fmt.Sprintf("ErrorCode(%d)", int(c)) + } +} + +// OrtError represents an error from the ONNX Runtime C API. +type OrtError struct { + Code ErrorCode + Msg string +} + +func (e *OrtError) Error() string { + return fmt.Sprintf("onnxruntime: %s: %s", e.Code, e.Msg) +} + +func checkStatus(status *C.OrtStatus) error { + if status == nil { + return nil + } + code := ErrorCode(C.ort_GetErrorCode(status)) + msg := C.GoString(C.ort_GetErrorMessage(status)) + C.ort_ReleaseStatus(status) + return &OrtError{Code: code, Msg: msg} +} + +func wrapErr(context string, err error) error { + if err == nil { + return nil + } + return fmt.Errorf("ort: %s: %w", context, err) +} + +var errNotInitialized = fmt.Errorf("ort: not initialized; call Init() first") +var errShutdown = fmt.Errorf("ort: already shut down") + +func statusToErr(status unsafe.Pointer) error { + return checkStatus((*C.OrtStatus)(status)) +} diff --git a/go/onnxruntime/iobinding.go b/go/onnxruntime/iobinding.go new file mode 100644 index 0000000000000..a13358420cccd --- /dev/null +++ b/go/onnxruntime/iobinding.go @@ -0,0 +1,194 @@ +package onnxruntime + +/* +#include "cshim.h" +#include +*/ +import "C" +import "unsafe" + +type AllocatorType int + +const ( + AllocatorTypeInvalid AllocatorType = -1 + AllocatorTypeDevice AllocatorType = 0 + AllocatorTypeArena AllocatorType = 1 +) + +type MemType int + +const ( + MemTypeCPUInput MemType = -2 + MemTypeCPU MemType = -1 + MemTypeDefault MemType = 0 +) + +type MemoryInfo struct { + handle *C.OrtMemoryInfo +} + +func NewCPUMemoryInfo() (*MemoryInfo, error) { + if err := checkInit(); err != nil { + return nil, err + } + var info *C.OrtMemoryInfo + if err := checkStatus(C.ort_CreateCpuMemoryInfo(C.OrtDeviceAllocator, C.OrtMemTypeDefault, &info)); err != nil { + return nil, wrapErr("create cpu memory info", err) + } + return &MemoryInfo{handle: info}, nil +} + +func NewMemoryInfo(name string, allocatorType AllocatorType, id int, memType MemType) (*MemoryInfo, error) { + if err := checkInit(); err != nil { + return nil, err + } + cName := C.CString(name) + defer C.free(unsafe.Pointer(cName)) + var info *C.OrtMemoryInfo + if err := checkStatus(C.ort_CreateMemoryInfo( + cName, + C.enum_OrtAllocatorType(allocatorType), + C.int(id), + C.enum_OrtMemType(memType), + &info, + )); err != nil { + return nil, wrapErr("create memory info", err) + } + return &MemoryInfo{handle: info}, nil +} + +func (m *MemoryInfo) Close() error { + if m.handle != nil { + C.ort_ReleaseMemoryInfo(m.handle) + m.handle = nil + } + return nil +} + +type IOBinding struct { + handle *C.OrtIoBinding + session *Session +} + +func NewIOBinding(session *Session) (*IOBinding, error) { + if err := checkInit(); err != nil { + return nil, err + } + var binding *C.OrtIoBinding + if err := checkStatus(C.ort_CreateIoBinding(session.handle, &binding)); err != nil { + return nil, wrapErr("create io binding", err) + } + return &IOBinding{handle: binding, session: session}, nil +} + +func (b *IOBinding) BindInput(name string, value *Tensor) error { + cName := C.CString(name) + defer C.free(unsafe.Pointer(cName)) + return wrapErr("bind input", checkStatus(C.ort_BindInput(b.handle, cName, value.value))) +} + +func (b *IOBinding) BindOutput(name string, value *Tensor) error { + cName := C.CString(name) + defer C.free(unsafe.Pointer(cName)) + return wrapErr("bind output", checkStatus(C.ort_BindOutput(b.handle, cName, value.value))) +} + +func (b *IOBinding) BindOutputToDevice(name string, memInfo *MemoryInfo) error { + cName := C.CString(name) + defer C.free(unsafe.Pointer(cName)) + return wrapErr("bind output to device", checkStatus(C.ort_BindOutputToDevice(b.handle, cName, memInfo.handle))) +} + +func (b *IOBinding) Run(opts *RunOptions) error { + b.session.mu.RLock() + defer b.session.mu.RUnlock() + + var runOpts *C.OrtRunOptions + if opts != nil { + runOpts = opts.handle + } + return wrapErr("run with binding", checkStatus(C.ort_RunWithBinding(b.session.handle, runOpts, b.handle))) +} + +func (b *IOBinding) OutputNames() ([]string, error) { + var allocator *C.OrtAllocator + if err := checkStatus(C.ort_GetAllocatorWithDefaultOptions(&allocator)); err != nil { + return nil, wrapErr("get allocator", err) + } + + var buffer *C.char + var lengths *C.size_t + var count C.size_t + if err := checkStatus(C.ort_GetBoundOutputNames(b.handle, allocator, &buffer, &lengths, &count)); err != nil { + return nil, wrapErr("get bound output names", err) + } + defer C.ort_AllocatorFree(allocator, unsafe.Pointer(buffer)) + defer C.ort_AllocatorFree(allocator, unsafe.Pointer(lengths)) + + n := int(count) + if n == 0 { + return nil, nil + } + + lens := unsafe.Slice((*C.size_t)(unsafe.Pointer(lengths)), n) + names := make([]string, n) + ptr := (*C.char)(unsafe.Pointer(buffer)) + for i := 0; i < n; i++ { + l := int(lens[i]) + names[i] = C.GoStringN(ptr, C.int(l)) + ptr = (*C.char)(unsafe.Pointer(uintptr(unsafe.Pointer(ptr)) + uintptr(l))) + } + return names, nil +} + +func (b *IOBinding) OutputValues() ([]*Tensor, error) { + var allocator *C.OrtAllocator + if err := checkStatus(C.ort_GetAllocatorWithDefaultOptions(&allocator)); err != nil { + return nil, wrapErr("get allocator", err) + } + + var values **C.OrtValue + var count C.size_t + if err := checkStatus(C.ort_GetBoundOutputValues(b.handle, allocator, &values, &count)); err != nil { + return nil, wrapErr("get bound output values", err) + } + defer C.ort_AllocatorFree(allocator, unsafe.Pointer(values)) + + n := int(count) + if n == 0 { + return nil, nil + } + + ptrs := unsafe.Slice((**C.OrtValue)(unsafe.Pointer(values)), n) + tensors := make([]*Tensor, n) + for i := 0; i < n; i++ { + t, err := wrapOutputTensor(ptrs[i]) + if err != nil { + for j := 0; j < i; j++ { + _ = tensors[j].Close() + } + for j := i; j < n; j++ { + C.ort_ReleaseValue(ptrs[j]) + } + return nil, wrapErr("wrap bound output", err) + } + tensors[i] = t + } + return tensors, nil +} + +func (b *IOBinding) ClearInputs() { + C.ort_ClearBoundInputs(b.handle) +} + +func (b *IOBinding) ClearOutputs() { + C.ort_ClearBoundOutputs(b.handle) +} + +func (b *IOBinding) Close() error { + if b.handle != nil { + C.ort_ReleaseIoBinding(b.handle) + b.handle = nil + } + return nil +} diff --git a/go/onnxruntime/iobinding_test.go b/go/onnxruntime/iobinding_test.go new file mode 100644 index 0000000000000..0c5755d02c06d --- /dev/null +++ b/go/onnxruntime/iobinding_test.go @@ -0,0 +1,101 @@ +package onnxruntime + +import ( + "testing" +) + +func TestIOBindingBasic(t *testing.T) { + sess, err := NewSession(testdataPath("add_f32.onnx"), nil) + if err != nil { + t.Fatal(err) + } + defer sess.Close() + + binding, err := NewIOBinding(sess) + if err != nil { + t.Fatal(err) + } + defer binding.Close() + + a, _ := CreateTensor[float32]([]int64{2, 3}, []float32{1, 2, 3, 4, 5, 6}) + b, _ := CreateTensor[float32]([]int64{2, 3}, []float32{10, 20, 30, 40, 50, 60}) + defer a.Close() + defer b.Close() + + if err := binding.BindInput("A", a); err != nil { + t.Fatal(err) + } + if err := binding.BindInput("B", b); err != nil { + t.Fatal(err) + } + + memInfo, err := NewCPUMemoryInfo() + if err != nil { + t.Fatal(err) + } + defer memInfo.Close() + + if err := binding.BindOutputToDevice("C", memInfo); err != nil { + t.Fatal(err) + } + + if err := binding.Run(nil); err != nil { + t.Fatal(err) + } + + outputs, err := binding.OutputValues() + if err != nil { + t.Fatal(err) + } + defer func() { + for _, o := range outputs { + o.Close() + } + }() + + if len(outputs) != 1 { + t.Fatalf("expected 1 output, got %d", len(outputs)) + } + + data, err := TensorData[float32](outputs[0]) + if err != nil { + t.Fatal(err) + } + if data[0] != 11 { + t.Errorf("expected 11, got %f", data[0]) + } +} + +func TestMemoryInfo(t *testing.T) { + mem, err := NewCPUMemoryInfo() + if err != nil { + t.Fatal(err) + } + mem.Close() + mem.Close() +} + +func TestNewMemoryInfo(t *testing.T) { + mem, err := NewMemoryInfo("Cpu", AllocatorTypeDevice, 0, MemTypeDefault) + if err != nil { + t.Fatal(err) + } + defer mem.Close() +} + +func TestIOBindingClear(t *testing.T) { + sess, err := NewSession(testdataPath("add_f32.onnx"), nil) + if err != nil { + t.Fatal(err) + } + defer sess.Close() + + binding, err := NewIOBinding(sess) + if err != nil { + t.Fatal(err) + } + defer binding.Close() + + binding.ClearInputs() + binding.ClearOutputs() +} diff --git a/go/onnxruntime/load_unix.go b/go/onnxruntime/load_unix.go new file mode 100644 index 0000000000000..ee18efe4225b3 --- /dev/null +++ b/go/onnxruntime/load_unix.go @@ -0,0 +1,57 @@ +//go:build !windows + +package onnxruntime + +/* +#cgo LDFLAGS: -ldl +#include +#include +#include "cshim.h" + +static void *ort_dlopen(const char *path) { + return dlopen(path, RTLD_NOW | RTLD_LOCAL); +} + +static void *ort_dlsym(void *handle, const char *symbol) { + return dlsym(handle, symbol); +} + +static const char *ort_dlerror(void) { + return dlerror(); +} +*/ +import "C" +import ( + "fmt" + "runtime" + "unsafe" +) + +func platformLibraryName() string { + if runtime.GOOS == "darwin" { + return "libonnxruntime.dylib" + } + return "libonnxruntime.so" +} + +func loadLibrary(path string) (unsafe.Pointer, error) { + cpath := C.CString(path) + defer C.free(unsafe.Pointer(cpath)) + + handle := C.ort_dlopen(cpath) + if handle == nil { + errMsg := C.GoString(C.ort_dlerror()) + return nil, fmt.Errorf("dlopen %s: %s", path, errMsg) + } + + sym := C.CString("OrtGetApiBase") + defer C.free(unsafe.Pointer(sym)) + + fn := C.ort_dlsym(handle, sym) + if fn == nil { + errMsg := C.GoString(C.ort_dlerror()) + return nil, fmt.Errorf("dlsym OrtGetApiBase: %s", errMsg) + } + + return fn, nil +} diff --git a/go/onnxruntime/load_windows.go b/go/onnxruntime/load_windows.go new file mode 100644 index 0000000000000..510ac952f7f74 --- /dev/null +++ b/go/onnxruntime/load_windows.go @@ -0,0 +1,27 @@ +//go:build windows + +package onnxruntime + +import ( + "fmt" + "syscall" + "unsafe" +) + +func platformLibraryName() string { + return "onnxruntime.dll" +} + +func loadLibrary(path string) (unsafe.Pointer, error) { + dll, err := syscall.LoadDLL(path) + if err != nil { + return nil, fmt.Errorf("LoadDLL %s: %w", path, err) + } + + proc, err := dll.FindProc("OrtGetApiBase") + if err != nil { + return nil, fmt.Errorf("FindProc OrtGetApiBase: %w", err) + } + + return unsafe.Pointer(proc.Addr()), nil +} diff --git a/go/onnxruntime/metadata.go b/go/onnxruntime/metadata.go new file mode 100644 index 0000000000000..24a325bcde707 --- /dev/null +++ b/go/onnxruntime/metadata.go @@ -0,0 +1,154 @@ +package onnxruntime + +/* +#include "cshim.h" +#include +*/ +import "C" +import ( + "fmt" + "unsafe" +) + +type ModelMetadata struct { + handle *C.OrtModelMetadata + closed bool +} + +func (s *Session) ModelMetadata() (*ModelMetadata, error) { + s.mu.RLock() + defer s.mu.RUnlock() + + if s.closed { + return nil, fmt.Errorf("ort: model metadata: session is closed") + } + + var meta *C.OrtModelMetadata + if err := checkStatus(C.ort_SessionGetModelMetadata(s.handle, &meta)); err != nil { + return nil, wrapErr("model metadata", err) + } + return &ModelMetadata{handle: meta}, nil +} + +func (m *ModelMetadata) ProducerName() (string, error) { + return m.getString(func(alloc *C.OrtAllocator, out **C.char) error { + return checkStatus(C.ort_ModelMetadataGetProducerName(m.handle, alloc, out)) + }, "producer name") +} + +func (m *ModelMetadata) GraphName() (string, error) { + return m.getString(func(alloc *C.OrtAllocator, out **C.char) error { + return checkStatus(C.ort_ModelMetadataGetGraphName(m.handle, alloc, out)) + }, "graph name") +} + +func (m *ModelMetadata) Domain() (string, error) { + return m.getString(func(alloc *C.OrtAllocator, out **C.char) error { + return checkStatus(C.ort_ModelMetadataGetDomain(m.handle, alloc, out)) + }, "domain") +} + +func (m *ModelMetadata) Description() (string, error) { + return m.getString(func(alloc *C.OrtAllocator, out **C.char) error { + return checkStatus(C.ort_ModelMetadataGetDescription(m.handle, alloc, out)) + }, "description") +} + +func (m *ModelMetadata) GraphDescription() (string, error) { + return m.getString(func(alloc *C.OrtAllocator, out **C.char) error { + return checkStatus(C.ort_ModelMetadataGetGraphDescription(m.handle, alloc, out)) + }, "graph description") +} + +func (m *ModelMetadata) Version() (int64, error) { + var version C.int64_t + if err := checkStatus(C.ort_ModelMetadataGetVersion(m.handle, &version)); err != nil { + return 0, wrapErr("metadata version", err) + } + return int64(version), nil +} + +func (m *ModelMetadata) CustomMetadataKeys() ([]string, error) { + alloc, err := defaultAllocator() + if err != nil { + return nil, wrapErr("metadata custom keys", err) + } + + var cKeys **C.char + var count C.int64_t + if err := checkStatus(C.ort_ModelMetadataGetCustomMetadataMapKeys(m.handle, alloc, &cKeys, &count)); err != nil { + return nil, wrapErr("metadata custom keys", err) + } + + n := int(count) + if n == 0 { + return nil, nil + } + + ptrs := unsafe.Slice((**C.char)(unsafe.Pointer(cKeys)), n) + keys := make([]string, n) + for i := 0; i < n; i++ { + keys[i] = C.GoString(ptrs[i]) + C.ort_AllocatorFree(alloc, unsafe.Pointer(ptrs[i])) + } + C.ort_AllocatorFree(alloc, unsafe.Pointer(cKeys)) + + return keys, nil +} + +func (m *ModelMetadata) LookupCustomMetadata(key string) (string, error) { + alloc, err := defaultAllocator() + if err != nil { + return "", wrapErr("metadata lookup", err) + } + + cKey := C.CString(key) + defer C.free(unsafe.Pointer(cKey)) + + var cVal *C.char + if err := checkStatus(C.ort_ModelMetadataLookupCustomMetadataMap(m.handle, alloc, cKey, &cVal)); err != nil { + return "", wrapErr("metadata lookup", err) + } + + if cVal == nil { + return "", nil + } + + val := C.GoString(cVal) + C.ort_AllocatorFree(alloc, unsafe.Pointer(cVal)) + return val, nil +} + +func (m *ModelMetadata) Close() error { + if m.closed { + return nil + } + m.closed = true + C.ort_ReleaseModelMetadata(m.handle) + m.handle = nil + return nil +} + +func (m *ModelMetadata) getString(fn func(*C.OrtAllocator, **C.char) error, context string) (string, error) { + alloc, err := defaultAllocator() + if err != nil { + return "", wrapErr("metadata "+context, err) + } + + var cStr *C.char + if err := fn(alloc, &cStr); err != nil { + return "", wrapErr("metadata "+context, err) + } + + str := C.GoString(cStr) + C.ort_AllocatorFree(alloc, unsafe.Pointer(cStr)) + return str, nil +} + +func defaultAllocator() (*C.OrtAllocator, error) { + var alloc *C.OrtAllocator + if err := checkStatus(C.ort_GetAllocatorWithDefaultOptions(&alloc)); err != nil { + return nil, err + } + return alloc, nil +} diff --git a/go/onnxruntime/metadata_test.go b/go/onnxruntime/metadata_test.go new file mode 100644 index 0000000000000..b62b49da0de31 --- /dev/null +++ b/go/onnxruntime/metadata_test.go @@ -0,0 +1,70 @@ +package onnxruntime + +import ( + "testing" +) + +func TestModelMetadata(t *testing.T) { + sess, err := NewSession(testdataPath("add_f32.onnx"), nil) + if err != nil { + t.Fatal(err) + } + defer sess.Close() + + meta, err := sess.ModelMetadata() + if err != nil { + t.Fatal(err) + } + defer meta.Close() + + _, err = meta.ProducerName() + if err != nil { + t.Errorf("ProducerName: %v", err) + } + + _, err = meta.GraphName() + if err != nil { + t.Errorf("GraphName: %v", err) + } + + _, err = meta.Domain() + if err != nil { + t.Errorf("Domain: %v", err) + } + + _, err = meta.Description() + if err != nil { + t.Errorf("Description: %v", err) + } + + _, err = meta.Version() + if err != nil { + t.Errorf("Version: %v", err) + } + + keys, err := meta.CustomMetadataKeys() + if err != nil { + t.Errorf("CustomMetadataKeys: %v", err) + } + t.Logf("metadata: keys=%v", keys) + + _, err = meta.LookupCustomMetadata("nonexistent_key") + if err != nil { + t.Errorf("LookupCustomMetadata for missing key should not error: %v", err) + } +} + +func TestModelMetadataDoubleClose(t *testing.T) { + sess, err := NewSession(testdataPath("add_f32.onnx"), nil) + if err != nil { + t.Fatal(err) + } + defer sess.Close() + + meta, err := sess.ModelMetadata() + if err != nil { + t.Fatal(err) + } + meta.Close() + meta.Close() +} diff --git a/go/onnxruntime/onnxruntime_c_api.h b/go/onnxruntime/onnxruntime_c_api.h new file mode 100644 index 0000000000000..34a15076c8a7f --- /dev/null +++ b/go/onnxruntime/onnxruntime_c_api.h @@ -0,0 +1,8682 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +// See docs\c_cxx\README.md on generating the Doxygen documentation from this file + +/** \mainpage ONNX Runtime + * + * ONNX Runtime is a high-performance inference and training graph execution engine for deep learning models. + * + * ONNX Runtime's C, C++ APIs offer an easy to use interface to onboard and execute onnx models. + * - \subpage c_cpp_api "Core C, C++ APIs" + * - \subpage training_c_cpp_api "Training C, C++ APIs for on-device training" + * + * \page c_cpp_api Core C, C++ APIs + *

C

+ * + * ::OrtApi - Click here to go to the structure with all C API functions. + * + *

C++

+ * + * ::Ort - Click here to go to the namespace holding all of the C++ wrapper classes + * + * It is a set of header only wrapper classes around the C API. The goal is to turn the C style return value error codes into C++ exceptions, and to + * automate memory management through standard C++ RAII principles. + * + * \addtogroup Global + * ONNX Runtime C API + * @{ + */ + +#pragma once +#include +#include +#include +#include + +#include "onnxruntime_error_code.h" + +/** \brief The API version defined in this header + * + * This value is used by some API functions to behave as this version of the header expects. + */ +#define ORT_API_VERSION 29 + +#ifdef __cplusplus +extern "C" { +#endif + +//! @} +// SAL2 Definitions +#ifndef _MSC_VER +#define _In_ +#define _In_z_ +#define _In_opt_ +#define _In_opt_z_ +#define _Out_ +#define _Out_opt_ +#define _Outptr_ +#define _Outptr_opt_ +#define _Inout_ +#define _Inout_opt_ +#define _Frees_ptr_opt_ +#define _Ret_maybenull_ +#define _Ret_notnull_ +#define _Check_return_ +#define _Outptr_result_maybenull_ +#define _Outptr_result_maybenull_z_ +#define _In_reads_(X) +#define _In_reads_opt_ +#define _Inout_updates_(X) +#define _Out_writes_(X) +#define _Out_writes_opt_(X) +#define _Inout_updates_all_(X) +#define _Out_writes_bytes_all_(X) +#define _Out_writes_all_(X) +#define _Success_(X) +#define _Outptr_result_buffer_maybenull_(X) +#define ORT_ALL_ARGS_NONNULL __attribute__((nonnull)) +#else +#include +#define ORT_ALL_ARGS_NONNULL +#endif + +#ifdef _WIN32 +// Define ORT_DLL_IMPORT if your program is dynamically linked to Ort. +// dllexport is not used, we use a .def file. +#ifdef ORT_DLL_IMPORT +#define ORT_EXPORT __declspec(dllimport) +#else +#define ORT_EXPORT +#endif +#define ORT_API_CALL __stdcall +#define ORT_MUST_USE_RESULT +#define ORTCHAR_T wchar_t +#else +// Make symbols visible on non-Windows platforms. The visibility attribute is +// needed when ORT is built as a shared library without a version script +// (e.g. when compiled within another project's build system). On non-Apple +// platforms, the default ORT build uses a generated version script +// (tools/ci_build/gen_def.py) that exports the needed symbols, so this was +// previously only enabled for __APPLE__. Expanding to __GNUC__ (GCC/Clang) +// covers additional embedding scenarios while remaining harmless when a +// version script is also in use. +#if defined(__GNUC__) +#define ORT_EXPORT __attribute__((visibility("default"))) +#else +#define ORT_EXPORT +#endif +#define ORT_API_CALL +#define ORT_MUST_USE_RESULT __attribute__((warn_unused_result)) +#define ORTCHAR_T char +#endif + +/// ORTCHAR_T, ORT_TSTR are reserved specifically for path handling. +/// All other strings are UTF-8 encoded, use char and std::string +#ifndef ORT_TSTR +#ifdef _WIN32 +#define ORT_TSTR(X) L##X +// When X is a macro, L##X is not defined. In this case, we need to use ORT_TSTR_ON_MACRO. +#define ORT_TSTR_ON_MACRO(X) L"" X +#else +#define ORT_TSTR(X) X +#define ORT_TSTR_ON_MACRO(X) X +#endif +#endif + +// On Windows, ORT_FILE is a wchar_t version of the __FILE__ macro. +// Otherwise, ORT_FILE is equivalent to __FILE__. +#ifndef ORT_FILE +#define ORT_FILE_INTERNAL(x) ORT_TSTR(x) +#define ORT_FILE ORT_FILE_INTERNAL(__FILE__) +#endif + +// Any pointer marked with _In_ or _Out_, cannot be NULL. + +// Windows users should use unicode paths when possible to bypass the MAX_PATH limitation +// Every pointer marked with _In_ or _Out_, cannot be NULL. Caller should ensure that. +// for ReleaseXXX(...) functions, they can accept NULL pointer. + +#ifdef __cplusplus +// For any compiler with C++11 support, MSVC 2015 and greater, or Clang version supporting noexcept. +// Such complex condition is needed because compilers set __cplusplus value differently. +#ifndef __has_feature +#define __has_feature(x) 0 +#endif +#if ((__cplusplus >= 201103L) || (_MSC_VER >= 1900) || (defined(__has_feature) && __has_feature(cxx_noexcept))) +#define NO_EXCEPTION noexcept +#else +#define NO_EXCEPTION throw() +#endif +#else +#define NO_EXCEPTION +#endif + +// __VA_ARGS__ on Windows and Linux are different +#define ORT_API(RETURN_TYPE, NAME, ...) RETURN_TYPE ORT_API_CALL NAME(__VA_ARGS__) NO_EXCEPTION + +#define ORT_API_T(RETURN_TYPE, NAME, ...) \ + RETURN_TYPE(ORT_API_CALL* NAME)(__VA_ARGS__) NO_EXCEPTION + +#define ORT_API_STATUS(NAME, ...) \ + _Success_(return == 0) _Check_return_ _Ret_maybenull_ OrtStatusPtr ORT_API_CALL NAME(__VA_ARGS__) \ + NO_EXCEPTION ORT_MUST_USE_RESULT + +// XXX: Unfortunately, SAL annotations are known to not work with function pointers +#define ORT_API2_STATUS(NAME, ...) \ + _Check_return_ _Ret_maybenull_ OrtStatusPtr(ORT_API_CALL* NAME)(__VA_ARGS__) NO_EXCEPTION ORT_MUST_USE_RESULT + +// Used in *.cc files. Almost as same as ORT_API_STATUS, except without ORT_MUST_USE_RESULT and ORT_EXPORT +#define ORT_API_STATUS_IMPL(NAME, ...) \ + _Success_(return == 0) _Check_return_ _Ret_maybenull_ OrtStatusPtr ORT_API_CALL NAME(__VA_ARGS__) NO_EXCEPTION + +#define ORT_CLASS_RELEASE(X) void(ORT_API_CALL * Release##X)(_Frees_ptr_opt_ Ort##X * input) + +#ifdef __DOXYGEN__ +#undef ORT_API_STATUS +#define ORT_API_STATUS(NAME, ...) OrtStatus* NAME(__VA_ARGS__) +#undef ORT_API2_STATUS +#define ORT_API2_STATUS(NAME, ...) OrtStatus* NAME(__VA_ARGS__) +#undef ORT_CLASS_RELEASE +#define ORT_CLASS_RELEASE(X) void Release##X(Ort##X* input) +#undef NO_EXCEPTION +#define NO_EXCEPTION +#endif +/** \addtogroup Global + * ONNX Runtime C API + * @{ + */ + +/** Copied from TensorProto::DataType + * Currently, Ort doesn't support complex64, complex128 + */ +typedef enum ONNXTensorElementDataType { + ONNX_TENSOR_ELEMENT_DATA_TYPE_UNDEFINED, + ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT, // maps to c type float + ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT8, // maps to c type uint8_t + ONNX_TENSOR_ELEMENT_DATA_TYPE_INT8, // maps to c type int8_t + ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT16, // maps to c type uint16_t + ONNX_TENSOR_ELEMENT_DATA_TYPE_INT16, // maps to c type int16_t + ONNX_TENSOR_ELEMENT_DATA_TYPE_INT32, // maps to c type int32_t + ONNX_TENSOR_ELEMENT_DATA_TYPE_INT64, // maps to c type int64_t + ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING, // maps to c++ type std::string + ONNX_TENSOR_ELEMENT_DATA_TYPE_BOOL, + ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT16, + ONNX_TENSOR_ELEMENT_DATA_TYPE_DOUBLE, // maps to c type double + ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT32, // maps to c type uint32_t + ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT64, // maps to c type uint64_t + ONNX_TENSOR_ELEMENT_DATA_TYPE_COMPLEX64, // complex with float32 real and imaginary components + ONNX_TENSOR_ELEMENT_DATA_TYPE_COMPLEX128, // complex with float64 real and imaginary components + ONNX_TENSOR_ELEMENT_DATA_TYPE_BFLOAT16, // Non-IEEE floating-point format based on IEEE754 single-precision + // float 8 types were introduced in onnx 1.14, see https://onnx.ai/onnx/technical/float8.html + ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT8E4M3FN, // Non-IEEE floating-point format based on IEEE754 single-precision + ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT8E4M3FNUZ, // Non-IEEE floating-point format based on IEEE754 single-precision + ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT8E5M2, // Non-IEEE floating-point format based on IEEE754 single-precision + ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT8E5M2FNUZ, // Non-IEEE floating-point format based on IEEE754 single-precision + // Int4 types were introduced in ONNX 1.16. See https://onnx.ai/onnx/technical/int4.html + ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT4, // maps to a pair of packed uint4 values (size == 1 byte) + ONNX_TENSOR_ELEMENT_DATA_TYPE_INT4, // maps to a pair of packed int4 values (size == 1 byte) + // Float4 types were introduced in ONNX 1.18. See https://onnx.ai/onnx/technical/float4.html + ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT4E2M1, // maps to a pair of packed float4 values (size == 1 byte) + // Int2 types were introduced in ONNX 1.20. See https://onnx.ai/onnx/technical/int2.html + ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT2, // maps to 4 packed uint2 values (size == 1 byte) + ONNX_TENSOR_ELEMENT_DATA_TYPE_INT2, // maps to 4 packed int2 values (size == 1 byte) + // Float8E8M0 type introduced in ONNX 1.21. 8-bit float with 8 exponent bits, 0 mantissa bits, no sign bit. + ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT8E8M0, // Non-IEEE floating-point format, all values are powers of two +} ONNXTensorElementDataType; + +// Synced with onnx TypeProto oneof +typedef enum ONNXType { + ONNX_TYPE_UNKNOWN, + ONNX_TYPE_TENSOR, + ONNX_TYPE_SEQUENCE, + ONNX_TYPE_MAP, + ONNX_TYPE_OPAQUE, + ONNX_TYPE_SPARSETENSOR, + ONNX_TYPE_OPTIONAL +} ONNXType; + +// These types are synced with internal +// SparseFormatFlags +typedef enum OrtSparseFormat { + ORT_SPARSE_UNDEFINED = 0, + ORT_SPARSE_COO = 0x1, + ORT_SPARSE_CSRC = 0x2, + ORT_SPARSE_BLOCK_SPARSE = 0x4 +} OrtSparseFormat; + +// Enum allows to query sparse tensor indices +enum OrtSparseIndicesFormat { + ORT_SPARSE_COO_INDICES, + ORT_SPARSE_CSR_INNER_INDICES, + ORT_SPARSE_CSR_OUTER_INDICES, + ORT_SPARSE_BLOCK_SPARSE_INDICES +}; + +/** \brief Logging severity levels + * + * In typical API usage, specifying a logging severity level specifies the minimum severity of log messages to show. + */ +typedef enum OrtLoggingLevel { + ORT_LOGGING_LEVEL_VERBOSE, ///< Verbose informational messages (least severe). + ORT_LOGGING_LEVEL_INFO, ///< Informational messages. + ORT_LOGGING_LEVEL_WARNING, ///< Warning messages. + ORT_LOGGING_LEVEL_ERROR, ///< Error messages. + ORT_LOGGING_LEVEL_FATAL, ///< Fatal error messages (most severe). +} OrtLoggingLevel; + +typedef enum OrtOpAttrType { + ORT_OP_ATTR_UNDEFINED = 0, + ORT_OP_ATTR_INT, + ORT_OP_ATTR_INTS, + ORT_OP_ATTR_FLOAT, + ORT_OP_ATTR_FLOATS, + ORT_OP_ATTR_STRING, + ORT_OP_ATTR_STRINGS, + ORT_OP_ATTR_GRAPH, + ORT_OP_ATTR_TENSOR, +} OrtOpAttrType; + +//! @} +#define ORT_RUNTIME_CLASS(X) \ + struct Ort##X; \ + typedef struct Ort##X Ort##X + +/** \addtogroup Global + * ONNX Runtime C API + * @{ + */ +// The actual types defined have an Ort prefix +ORT_RUNTIME_CLASS(Env); +ORT_RUNTIME_CLASS(Status); // nullptr for Status* indicates success +ORT_RUNTIME_CLASS(MemoryInfo); +ORT_RUNTIME_CLASS(IoBinding); +ORT_RUNTIME_CLASS(Session); // Don't call ReleaseSession from Dllmain (because session owns a thread pool) +ORT_RUNTIME_CLASS(Value); +ORT_RUNTIME_CLASS(RunOptions); +ORT_RUNTIME_CLASS(TypeInfo); +ORT_RUNTIME_CLASS(TensorTypeAndShapeInfo); +ORT_RUNTIME_CLASS(MapTypeInfo); +ORT_RUNTIME_CLASS(SequenceTypeInfo); +ORT_RUNTIME_CLASS(OptionalTypeInfo); +ORT_RUNTIME_CLASS(SessionOptions); +ORT_RUNTIME_CLASS(CustomOpDomain); +ORT_RUNTIME_CLASS(ModelMetadata); +ORT_RUNTIME_CLASS(ThreadPoolParams); +ORT_RUNTIME_CLASS(ThreadingOptions); +ORT_RUNTIME_CLASS(ArenaCfg); +ORT_RUNTIME_CLASS(PrepackedWeightsContainer); +ORT_RUNTIME_CLASS(TensorRTProviderOptionsV2); +ORT_RUNTIME_CLASS(NvTensorRtRtxProviderOptions); +ORT_RUNTIME_CLASS(CUDAProviderOptionsV2); +ORT_RUNTIME_CLASS(CANNProviderOptions); +ORT_RUNTIME_CLASS(DnnlProviderOptions); +ORT_RUNTIME_CLASS(Op); +ORT_RUNTIME_CLASS(OpAttr); +ORT_RUNTIME_CLASS(Logger); +ORT_RUNTIME_CLASS(ShapeInferContext); +ORT_RUNTIME_CLASS(LoraAdapter); +ORT_RUNTIME_CLASS(ValueInfo); +ORT_RUNTIME_CLASS(Node); +ORT_RUNTIME_CLASS(Graph); +ORT_RUNTIME_CLASS(Model); +ORT_RUNTIME_CLASS(ModelCompilationOptions); +ORT_RUNTIME_CLASS(HardwareDevice); +ORT_RUNTIME_CLASS(EpDevice); +ORT_RUNTIME_CLASS(KeyValuePairs); +ORT_RUNTIME_CLASS(SyncStream); // Opaque class to create an onnxruntime::Stream. +ORT_RUNTIME_CLASS(ExternalInitializerInfo); +ORT_RUNTIME_CLASS(ExternalResourceImporter); // Capability object for external resource import +ORT_RUNTIME_CLASS(ExternalMemoryHandle); // EP-imported view of shared external allocation +ORT_RUNTIME_CLASS(ExternalSemaphoreHandle); // EP-imported view of shared external semaphore +ORT_RUNTIME_CLASS(DeviceEpIncompatibilityDetails); +ORT_RUNTIME_CLASS(EpAssignedSubgraph); +ORT_RUNTIME_CLASS(EpAssignedNode); + +#ifdef _MSC_VER +typedef _Return_type_success_(return == 0) OrtStatus* OrtStatusPtr; +#else +typedef OrtStatus* OrtStatusPtr; +#endif + +/** \brief Generic function pointer type for experimental API lookup. + * + * Returned by OrtApi::GetExperimentalFunction. Cast to the correct function pointer type before calling. + * The experimental API function pointer types are defined in onnxruntime_experimental_c_api.h. + * + * This is a function pointer rather than \c void* because casting between function pointers and object + * pointers is undefined behavior in C and C++. Using a function pointer type keeps all casts well-defined. + */ +typedef void(ORT_API_CALL* OrtExperimentalFnPtr)(void); + +/** \brief Memory allocation interface + * + * Structure of function pointers that defines a memory allocator. This can be created and filled in by the user for custom allocators. + * + * When an allocator is passed to any function, be sure that the allocator object is not destroyed until the last allocated object using it is freed. + */ +typedef struct OrtAllocator { + uint32_t version; ///< Must be initialized to ORT_API_VERSION + + /// Returns a pointer to an allocated block of `size` bytes + void*(ORT_API_CALL* Alloc)(struct OrtAllocator* this_, size_t size); + + /// Free a block of memory previously allocated with OrtAllocator::Alloc + void(ORT_API_CALL* Free)(struct OrtAllocator* this_, void* p); + + /// Return a pointer to an ::OrtMemoryInfo that describes this allocator + const struct OrtMemoryInfo*(ORT_API_CALL* Info)(const struct OrtAllocator* this_); + /** + * @brief Optional allocation function to use for memory allocations made during session initialization. + * Use this function if you want to separate allocations made by ORT during Run() calls from + * those made during session initialization. This allows for separate memory management strategies for these + * allocations. + * + * \return pointer to an allocated block of `size` bytes. nullptr if size was 0 or allocation failed. + * + * \since 1.18 + */ + void*(ORT_API_CALL* Reserve)(struct OrtAllocator* this_, size_t size); + + /** + * @brief Function used to get the statistics of the allocator. + * + * Return a pointer to the OrtKeyValuePairs structure that contains the statistics of the allocator. + * The user should call OrtApi::ReleaseKeyValuePairs when done. + * + * Current known keys are: + * - Limit: Bytes limit of the allocator. -1 if no limit is set. + * - InUse: Number of bytes in use. + * - TotalAllocated: The total number of allocated bytes by the allocator. + * - MaxInUse: The maximum bytes in use. + * - NumAllocs: Number of allocations. + * - NumReserves: Number of reserves. (Number of calls to Reserve() in arena-based allocators) + * - NumArenaExtensions: Number of arena extensions (Relevant only for arena based allocators) + * - NumArenaShrinkages: Number of arena shrinkages (Relevant only for arena based allocators) + * - MaxAllocSize: The max single allocation seen. + * + * The allocator is free to add other entries as appropriate. + * + * \note Implementation of this function is optional and GetStats may be set to a nullptr. + * If the OrtAllocator is wrapping an internal ORT allocator that does not implement GetStats + * the returned OrtKeyValuePairs instance will be empty. + * + * \since 1.23 + */ + ORT_API2_STATUS(GetStats, _In_ const struct OrtAllocator* this_, _Outptr_ OrtKeyValuePairs** out); + + /** \brief Allocate using a stream. + * + * If the allocator is stream aware this performs allocation using a stream. + * + * Alloc will be used if this is nullptr. + * + * \param[in] this_ OrtAllocator instance + * \param[in] size Size of the allocation in bytes. nullptr if size was 0 or allocation failed. + * \param[in] stream The stream to allocate on. + * + * \return pointer to an allocated block of `size` bytes + * + * \note Implementation of this function is optional and AllocOnStream may be set to a nullptr. + * \since 1.23 + */ + void*(ORT_API_CALL* AllocOnStream)(struct OrtAllocator* this_, size_t size, OrtSyncStream* stream); + + /** \brief Release unused memory held by the allocator back to the system. + * + * For arena-based allocators, this frees allocation regions that are completely unused. + * For mempool-based allocators, this trims the pool to a configured minimum. + * For non-arena allocators this is a no-op. + * + * \param[in] this_ OrtAllocator instance + * + * \return nullptr on success, or an OrtStatus* on failure. + * + * \note Implementation of this function is optional and Shrink may be set to a nullptr. + * Callers must check for nullptr before invoking. + * \since 1.25 + */ + ORT_API2_STATUS(Shrink, _In_ struct OrtAllocator* this_); +} OrtAllocator; + +typedef void(ORT_API_CALL* OrtLoggingFunction)( + void* param, OrtLoggingLevel severity, const char* category, const char* logid, const char* code_location, + const char* message); + +/** \brief Graph optimization level + * + * Refer to https://www.onnxruntime.ai/docs/performance/graph-optimizations.html#graph-optimization-levels + * for an in-depth understanding of the Graph Optimization Levels. + */ +typedef enum GraphOptimizationLevel { + ORT_DISABLE_ALL = 0, + ORT_ENABLE_BASIC = 1, + ORT_ENABLE_EXTENDED = 2, + ORT_ENABLE_LAYOUT = 3, + ORT_ENABLE_ALL = 99 +} GraphOptimizationLevel; + +typedef enum ExecutionMode { + ORT_SEQUENTIAL = 0, + ORT_PARALLEL = 1, +} ExecutionMode; + +/** \brief Language projection identifiers + * /see OrtApi::SetLanguageProjection + */ +typedef enum OrtLanguageProjection { + ORT_PROJECTION_C = 0, + ORT_PROJECTION_CPLUSPLUS = 1, + ORT_PROJECTION_CSHARP = 2, + ORT_PROJECTION_PYTHON = 3, + ORT_PROJECTION_JAVA = 4, + ORT_PROJECTION_WINML = 5, + ORT_PROJECTION_NODEJS = 6, +} OrtLanguageProjection; + +struct OrtKernelInfo; +typedef struct OrtKernelInfo OrtKernelInfo; +struct OrtKernelContext; +typedef struct OrtKernelContext OrtKernelContext; +struct OrtCustomOp; +typedef struct OrtCustomOp OrtCustomOp; + +typedef enum OrtAllocatorType { + OrtInvalidAllocator = -1, + OrtDeviceAllocator = 0, + OrtArenaAllocator = 1, + OrtReadOnlyAllocator = 2, +} OrtAllocatorType; + +/** \brief Memory types for allocated memory, execution provider specific types should be extended in each provider. + */ +// Whenever this struct is updated, please also update the MakeKey function in onnxruntime / core / framework / execution_provider.cc +typedef enum OrtMemType { + /// Any CPU memory used by non-CPU execution provider + OrtMemTypeCPUInput = -2, + /// CPU accessible memory outputted by non-CPU execution provider, i.e. HOST_ACCESSIBLE + OrtMemTypeCPUOutput = -1, + /// CPU accessible memory allocated by non-CPU execution provider, i.e. HOST_ACCESSIBLE + OrtMemTypeCPU = OrtMemTypeCPUOutput, + /// The default allocator for execution provider + OrtMemTypeDefault = 0, +} OrtMemType; + +/** \brief This matches OrtDevice::MemoryType values */ +typedef enum OrtDeviceMemoryType { + OrtDeviceMemoryType_DEFAULT = 0, ///< Device memory + OrtDeviceMemoryType_HOST_ACCESSIBLE = 5, ///< Shared/pinned memory for transferring between CPU and the device +} OrtDeviceMemoryType; + +/** \brief This mimics OrtDevice type constants so they can be returned in the API + */ +typedef enum OrtMemoryInfoDeviceType { + OrtMemoryInfoDeviceType_CPU = 0, + OrtMemoryInfoDeviceType_GPU = 1, + OrtMemoryInfoDeviceType_FPGA = 2, + OrtMemoryInfoDeviceType_NPU = 3, +} OrtMemoryInfoDeviceType; + +typedef enum OrtHardwareDeviceType { + OrtHardwareDeviceType_CPU, + OrtHardwareDeviceType_GPU, + OrtHardwareDeviceType_NPU +} OrtHardwareDeviceType; + +/** \brief These are the default EP selection policies used by ORT when doing automatic EP selection. + */ +typedef enum OrtExecutionProviderDevicePolicy { + OrtExecutionProviderDevicePolicy_DEFAULT, + OrtExecutionProviderDevicePolicy_PREFER_CPU, + OrtExecutionProviderDevicePolicy_PREFER_NPU, + OrtExecutionProviderDevicePolicy_PREFER_GPU, + OrtExecutionProviderDevicePolicy_MAX_PERFORMANCE, + OrtExecutionProviderDevicePolicy_MAX_EFFICIENCY, + OrtExecutionProviderDevicePolicy_MIN_OVERALL_POWER, +} OrtExecutionProviderDevicePolicy; + +/** \brief Reasons why an execution provider might not be compatible with a device + */ +typedef enum OrtDeviceEpIncompatibilityReason { + OrtDeviceEpIncompatibility_NONE = 0, + OrtDeviceEpIncompatibility_DRIVER_INCOMPATIBLE = 1 << 0, + OrtDeviceEpIncompatibility_DEVICE_INCOMPATIBLE = 1 << 1, + OrtDeviceEpIncompatibility_MISSING_DEPENDENCY = 1 << 2, + OrtDeviceEpIncompatibility_UNKNOWN = 1 << 31 +} OrtDeviceEpIncompatibilityReason; + +/** \brief Delegate to allow providing custom OrtEpDevice selection logic + * + * This delegate is called by the EP selection code to allow the user to provide custom device selection logic. + * The user can use this to select OrtEpDevice instances from the list of available devices. + * + * \param ep_devices The list of available devices. + * \param num_devices The number of available devices. + * \param model_metadata The model metadata. + * \param runtime_metadata The runtime metadata. May be nullptr. + * \param selected Pre-allocated array to populate with selected OrtEpDevice pointers from ep_devices. + * \param max_selected The maximum number of devices that can be selected in the pre-allocated array. + Currently the maximum is 8. + * \param num_selected The number of selected devices. + * \param state Opaque pointer. Required to use the delegate from other languages like C# and python. + * + * \return OrtStatus* Selection status. Return nullptr on success. + * Use CreateStatus to provide error info. Use ORT_FAIL as the error code. + * ORT will release the OrtStatus* if not null. + */ +typedef OrtStatus*(ORT_API_CALL* EpSelectionDelegate)(_In_ const OrtEpDevice** ep_devices, + _In_ size_t num_devices, + _In_ const OrtKeyValuePairs* model_metadata, + _In_opt_ const OrtKeyValuePairs* runtime_metadata, + _Inout_ const OrtEpDevice** selected, + _In_ size_t max_selected, + _Out_ size_t* num_selected, + _In_ void* state); + +/** \brief Function called by ORT to write a buffer to a custom destination (e.g., file, stream, etc.). + * + * \param state Opaque pointer holding the user's state. + * \param buffer The buffer to write. + * \param buffer_num_bytes The size of the buffer in bytes. + * + * \return OrtStatus* Write status. Return nullptr on success. + * Use CreateStatus to provide error info. Use ORT_FAIL as the error code. + * ORT will release the OrtStatus* if not null. + */ +typedef OrtStatus*(ORT_API_CALL* OrtWriteBufferFunc)(_In_ void* state, + _In_ const void* buffer, + _In_ size_t buffer_num_bytes); + +/** \brief Function called by ORT to allow user to specify how an initializer should be saved, that is, either + * written to an external file or stored within the model. ORT calls this function for every initializer when + * generating a model. + * + * If the function implementation sets the `new_external_info` output parameter to NULL, ORT stores the initializer data + * within the generated model. + * + * Otherwise, if the function implementation sets `new_external_info` to a valid OrtExternalInitializerInfo instance, + * ORT assumes that this function stores the initializer data in a file. In this case, ORT configures the model's + * initializer to point to the location specified by the `new_external_info` output parameter. + * + * \param[in] state Opaque pointer holding the user's state. + * \param[in] initializer_name The initializer's name as a null-terminated string. + * \param[in] initializer_value OrtValue containing the initializer's data, type, and shape. + * \param[in] external_info If the initializer is originally stored in an external file, `external_info` contains + * the file path, file offset, and the data's byte size within the file. Otherwise, + * `external_info` is NULL if the initializer is not originally stored in a file. + * \param[out] new_external_info Output parameter set to a new OrtExternalInitializerInfo instance indicating the + * location where the function implementation stored the initializer data. + * The function implementation must use `OrtApi::CreateExternalInitializerInfo()` to + * create the instance. + * If the function implementation sets `new_external_info` to NULL, + * ORT stores the initializers within the model. + * + * \note ORT takes ownership of the `new_external_info` output parameter. + * + * \return OrtStatus* Write status. Return nullptr on success. + * Use CreateStatus to provide error info. Use ORT_FAIL as the error code. + * ORT will release the OrtStatus* if not null. + */ +typedef OrtStatus*(ORT_API_CALL* OrtGetInitializerLocationFunc)( + _In_ void* state, + _In_ const char* initializer_name, + _In_ const OrtValue* initializer_value, + _In_opt_ const OrtExternalInitializerInfo* external_info, + _Outptr_result_maybenull_ OrtExternalInitializerInfo** new_external_info); + +/** \brief Algorithm to use for cuDNN Convolution Op + */ +typedef enum OrtCudnnConvAlgoSearch { + OrtCudnnConvAlgoSearchExhaustive, // expensive exhaustive benchmarking using cudnnFindConvolutionForwardAlgorithmEx + OrtCudnnConvAlgoSearchHeuristic, // lightweight heuristic based search using cudnnGetConvolutionForwardAlgorithm_v7 + OrtCudnnConvAlgoSearchDefault, // default algorithm using CUDNN_CONVOLUTION_FWD_ALGO_IMPLICIT_PRECOMP_GEMM +} OrtCudnnConvAlgoSearch; + +/** \brief CUDA Provider Options + * + * \see OrtApi::SessionOptionsAppendExecutionProvider_CUDA + */ +typedef struct OrtCUDAProviderOptions { +#ifdef __cplusplus + OrtCUDAProviderOptions() + : device_id{}, + cudnn_conv_algo_search{OrtCudnnConvAlgoSearchExhaustive}, + gpu_mem_limit{SIZE_MAX}, + arena_extend_strategy{}, + do_copy_in_default_stream{1}, + has_user_compute_stream{}, + user_compute_stream{}, + default_memory_arena_cfg{}, + tunable_op_enable{false}, + tunable_op_tuning_enable{false}, + tunable_op_max_tuning_duration_ms{} {} +#endif + + /** \brief CUDA device Id + * Defaults to 0. + */ + int device_id; + + /** \brief CUDA Convolution algorithm search configuration. + * See enum OrtCudnnConvAlgoSearch for more details. + * Defaults to OrtCudnnConvAlgoSearchExhaustive. + */ + OrtCudnnConvAlgoSearch cudnn_conv_algo_search; + + /** \brief CUDA memory limit (To use all possible memory pass in maximum size_t) + * Defaults to SIZE_MAX. + * \note If a ::OrtArenaCfg has been applied, it will override this field + */ + size_t gpu_mem_limit; + + /** \brief Strategy used to grow the memory arena + * 0 = kNextPowerOfTwo
+ * 1 = kSameAsRequested
+ * Defaults to 0. + * \note If a ::OrtArenaCfg has been applied, it will override this field + */ + int arena_extend_strategy; + + /** \brief Flag indicating if copying needs to take place on the same stream as the compute stream in the CUDA EP + * 0 = Use separate streams for copying and compute. + * 1 = Use the same stream for copying and compute. + * Defaults to 1. + * WARNING: Setting this to 0 may result in data races for some models. + * Please see issue #4829 for more details. + */ + int do_copy_in_default_stream; + + /** \brief Flag indicating if there is a user provided compute stream + * Defaults to 0. + */ + int has_user_compute_stream; + + /** \brief User provided compute stream. + * If provided, please set `has_user_compute_stream` to 1. + */ + void* user_compute_stream; + + /** \brief CUDA memory arena configuration parameters + */ + OrtArenaCfg* default_memory_arena_cfg; + + /** \brief Enable TunableOp for using. + * Set it to 1/0 to enable/disable TunableOp. Otherwise, it is disabled by default. + * This option can be overridden by environment variable ORT_CUDA_TUNABLE_OP_ENABLE. + */ + int tunable_op_enable; + + /** \brief Enable TunableOp for tuning. + * Set it to 1/0 to enable/disable TunableOp tuning. Otherwise, it is disabled by default. + * This option can be overridden by environment variable ORT_CUDA_TUNABLE_OP_TUNING_ENABLE. + */ + int tunable_op_tuning_enable; + + /** \brief Max tuning duration time limit for each instance of TunableOp. + * Defaults to 0 to disable the limit. + */ + int tunable_op_max_tuning_duration_ms; + +} OrtCUDAProviderOptions; + +/** \brief ROCM Provider Options + * + * \see OrtApi::SessionOptionsAppendExecutionProvider_ROCM + */ +typedef struct OrtROCMProviderOptions { +#ifdef __cplusplus + OrtROCMProviderOptions() + : device_id{}, + miopen_conv_exhaustive_search{0}, + gpu_mem_limit{SIZE_MAX}, + arena_extend_strategy{}, + do_copy_in_default_stream{1}, + has_user_compute_stream{}, + user_compute_stream{}, + default_memory_arena_cfg{}, + enable_hip_graph{false}, + tunable_op_enable{false}, + tunable_op_tuning_enable{false}, + tunable_op_max_tuning_duration_ms{} {} +#endif + + /** \brief ROCM device Id + * Defaults to 0. + */ + int device_id; + + /** \brief ROCM MIOpen Convolution algorithm exhaustive search option. + * Defaults to 0 (false). + */ + int miopen_conv_exhaustive_search; + + /** \brief ROCM memory limit (To use all possible memory pass in maximum size_t) + * Defaults to SIZE_MAX. + * \note If a ::OrtArenaCfg has been applied, it will override this field + */ + size_t gpu_mem_limit; + + /** \brief Strategy used to grow the memory arena + * 0 = kNextPowerOfTwo
+ * 1 = kSameAsRequested
+ * Defaults to 0. + * \note If a ::OrtArenaCfg has been applied, it will override this field + */ + int arena_extend_strategy; + + /** \brief Flag indicating if copying needs to take place on the same stream as the compute stream in the ROCM EP + * 0 = Use separate streams for copying and compute. + * 1 = Use the same stream for copying and compute. + * Defaults to 1. + * WARNING: Setting this to 0 may result in data races for some models. + * Please see issue #4829 for more details. + */ + int do_copy_in_default_stream; + + /** \brief Flag indicating if there is a user provided compute stream + * Defaults to 0. + */ + int has_user_compute_stream; + + /** \brief User provided compute stream. + * If provided, please set `has_user_compute_stream` to 1. + */ + void* user_compute_stream; + + /** \brief ROCM memory arena configuration parameters + */ + OrtArenaCfg* default_memory_arena_cfg; + + int enable_hip_graph; + + /** \brief Enable TunableOp for using. + * Set it to 1/0 to enable/disable TunableOp. Otherwise, it is disabled by default. + * This option can be overridden by environment variable ORT_ROCM_TUNABLE_OP_ENABLE. + */ + int tunable_op_enable; + + /** \brief Enable TunableOp for tuning. + * Set it to 1/0 to enable/disable TunableOp tuning. Otherwise, it is disabled by default. + * This option can be overridden by environment variable ORT_ROCM_TUNABLE_OP_TUNING_ENABLE. + */ + int tunable_op_tuning_enable; + + /** \brief Max tuning duration time limit for each instance of TunableOp. + * Defaults to 0 to disable the limit. + */ + int tunable_op_max_tuning_duration_ms; + +} OrtROCMProviderOptions; + +/** \brief TensorRT Provider Options + * + * \see OrtApi::SessionOptionsAppendExecutionProvider_TensorRT + */ +typedef struct OrtTensorRTProviderOptions { + int device_id; ///< CUDA device id (0 = default device) + int has_user_compute_stream; // indicator of user specified CUDA compute stream. + void* user_compute_stream; // user specified CUDA compute stream. + int trt_max_partition_iterations; // maximum iterations for TensorRT parser to get capability + int trt_min_subgraph_size; // minimum size of TensorRT subgraphs + size_t trt_max_workspace_size; // maximum workspace size for TensorRT. + int trt_fp16_enable; // enable TensorRT FP16 precision. Default 0 = false, nonzero = true + int trt_int8_enable; // enable TensorRT INT8 precision. Default 0 = false, nonzero = true + const char* trt_int8_calibration_table_name; // TensorRT INT8 calibration table name. + int trt_int8_use_native_calibration_table; // use native TensorRT generated calibration table. Default 0 = false, nonzero = true + int trt_dla_enable; // enable DLA. Default 0 = false, nonzero = true + int trt_dla_core; // DLA core number. Default 0 + int trt_dump_subgraphs; // dump TRT subgraph. Default 0 = false, nonzero = true + int trt_engine_cache_enable; // enable engine caching. Default 0 = false, nonzero = true + const char* trt_engine_cache_path; // specify engine cache path + int trt_engine_decryption_enable; // enable engine decryption. Default 0 = false, nonzero = true + const char* trt_engine_decryption_lib_path; // specify engine decryption library path + int trt_force_sequential_engine_build; // force building TensorRT engine sequentially. Default 0 = false, nonzero = true + // This is the legacy struct and don't add new fields here. + // For new field that can be represented by string, please add it in include/onnxruntime/core/providers/tensorrt/tensorrt_provider_options.h + // For non-string field, need to create a new separate api to handle it. +} OrtTensorRTProviderOptions; + +/** \brief MIGraphX Provider Options + * + * \see OrtApi::SessionOptionsAppendExecutionProvider_MIGraphX + */ +typedef struct OrtMIGraphXProviderOptions { + int device_id; // hip device id. + int migraphx_fp16_enable; // MIGraphX FP16 precision. Default 0 = false, nonzero = true + int migraphx_fp8_enable; // MIGraphX FP8 precision. Default 0 = false, nonzero = true + int migraphx_int8_enable; // MIGraphX INT8 precision. Default 0 = false, nonzero = true + int migraphx_use_native_calibration_table; // MIGraphx INT8 cal table. Default 0 = false, nonzero = true + const char* migraphx_int8_calibration_table_name; // MIGraphx INT8 calibration table name + int migraphx_save_compiled_model; // migraphx save compiled model. Default 0 = false, nonzero = true + const char* migraphx_save_model_path; // migraphx model path name + int migraphx_load_compiled_model; // migraphx int8 cal table. Default 0 = false, nonzero = true + const char* migraphx_load_model_path; // migraphx model path name + bool migraphx_exhaustive_tune; // MIGraphX tuned compile. Default = false, nonzero = true + + /** \brief MIGraphX memory limit (To use all possible memory pass in maximum size_t) + * Defaults to SIZE_MAX. + * \note If a ::OrtArenaCfg has been applied, it will override this field + */ + size_t migraphx_mem_limit; + + /** \brief Strategy used to grow the memory arena + * 0 = kNextPowerOfTwo
+ * 1 = kSameAsRequested
+ * Defaults to 0. + * \note If a ::OrtArenaCfg has been applied, it will override this field + */ + int migraphx_arena_extend_strategy; + + // This is the legacy struct and don't add new fields here. +} OrtMIGraphXProviderOptions; + +/** \brief OpenVINO Provider Options + * \brief This Struct is frozen since ORT 1.13.0. Its maintained part of Legacy API for compatibility. + * \brief For latest OpenVINO Provider Options update to the ProviderOptions map. + * \brief Latest OpenVINO Provider Options are listed in the + * \htmlonly + * onnxruntime document. + * \endhtmlonly + * \see OrtApi::SessionOptionsAppendExecutionProvider() + */ +typedef struct OrtOpenVINOProviderOptions { +#ifdef __cplusplus + OrtOpenVINOProviderOptions() : device_type{}, + enable_npu_fast_compile{}, + device_id{}, + num_of_threads{}, + cache_dir{}, + context{}, + enable_opencl_throttling{}, + enable_dynamic_shapes{} {} +#endif + /** \brief Device type string + * + * Valid settings are one of: "CPU_FP32", "CPU_FP16", "GPU_FP32", "GPU_FP16" + */ + const char* device_type; + unsigned char enable_npu_fast_compile; ///< 0 = disabled, nonzero = enabled + const char* device_id; + size_t num_of_threads; ///< 0 = Use default number of threads + const char* cache_dir; // path is set to empty by default + void* context; + unsigned char enable_opencl_throttling; ///< 0 = disabled, nonzero = enabled + unsigned char enable_dynamic_shapes; ///< 0 = disabled, nonzero = enabled +} OrtOpenVINOProviderOptions; + +struct OrtApi; +typedef struct OrtApi OrtApi; + +struct OrtTrainingApi; +typedef struct OrtTrainingApi OrtTrainingApi; + +struct OrtModelEditorApi; +typedef struct OrtModelEditorApi OrtModelEditorApi; + +struct OrtCompileApi; +typedef struct OrtCompileApi OrtCompileApi; + +struct OrtInteropApi; +typedef struct OrtInteropApi OrtInteropApi; + +struct OrtEpApi; +typedef struct OrtEpApi OrtEpApi; + +/** \brief The helper interface to get the right version of OrtApi + * + * Get a pointer to this structure through ::OrtGetApiBase + */ +struct OrtApiBase { + /** \brief Get a pointer to the requested version of the ::OrtApi + * + * \param[in] version Must be ::ORT_API_VERSION + * \return The ::OrtApi for the version requested, nullptr will be returned if this version is unsupported, for example when using a runtime + * older than the version created with this header file. + * + * One can call GetVersionString() to get the version of the Onnxruntime library for logging + * and error reporting purposes. + */ + const OrtApi*(ORT_API_CALL* GetApi)(uint32_t version)NO_EXCEPTION; + + /** \brief Returns a null terminated string of the version of the Onnxruntime library (eg: "1.8.1") + * + * \return UTF-8 encoded version string. Do not deallocate the returned buffer. + */ + const char*(ORT_API_CALL* GetVersionString)(void)NO_EXCEPTION; +}; + +typedef struct OrtApiBase OrtApiBase; + +/** \brief The Onnxruntime library's entry point to access the C API + * + * Call this to get the a pointer to an ::OrtApiBase + */ +ORT_EXPORT const OrtApiBase* ORT_API_CALL OrtGetApiBase(void) NO_EXCEPTION; + +/** \brief Thread work loop function + * + * Onnxruntime will provide the working loop on custom thread creation + * Argument is an onnxruntime built-in type which will be provided when thread pool calls OrtCustomCreateThreadFn + */ +typedef void (*OrtThreadWorkerFn)(void* ort_worker_fn_param); + +typedef const struct OrtCustomHandleType { + char __place_holder; +}* OrtCustomThreadHandle; + +/** \brief Ort custom thread creation function + * + * The function should return a thread handle to be used in onnxruntime thread pools + * Onnxruntime will throw exception on return value of nullptr or 0, indicating that the function failed to create a thread + */ +typedef OrtCustomThreadHandle (*OrtCustomCreateThreadFn)(void* ort_custom_thread_creation_options, OrtThreadWorkerFn ort_thread_worker_fn, void* ort_worker_fn_param); + +/** \brief Custom thread join function + * + * Onnxruntime thread pool destructor will call the function to join a custom thread. + * Argument ort_custom_thread_handle is the value returned by OrtCustomCreateThreadFn + */ +typedef void (*OrtCustomJoinThreadFn)(OrtCustomThreadHandle ort_custom_thread_handle); + +/** \brief Thread pool work enqueue callback + * + * Called when work is about to be enqueued to a thread pool worker thread. + * This runs on the thread that is submitting the work. + * \param[in] user_context The user-provided context passed when configuring callbacks + * \return Callback-specific data that will be passed to the configured + * OrtThreadPoolCallbacksConfig::on_start_work and OrtThreadPoolCallbacksConfig::on_stop_work. + * May return NULL. + */ +typedef _Ret_maybenull_ void* (*OrtThreadPoolWorkEnqueueFn)(_In_opt_ void* user_context)NO_EXCEPTION; + +/** \brief Thread pool work start callback + * + * Called when a thread is about to start executing work. + * This typically runs on a worker thread, but may also run on the submitting thread + * when the work queue is full and work is executed synchronously. + * \param[in] user_context The user-provided context passed when configuring callbacks + * \param[in] enqueue_data Data returned by the corresponding OrtThreadPoolCallbacksConfig::on_enqueue call + */ +typedef void (*OrtThreadPoolWorkStartFn)(_In_opt_ void* user_context, _In_opt_ void* enqueue_data) NO_EXCEPTION; + +/** \brief Thread pool work stop callback + * + * Called when a thread has finished executing work. + * This typically runs on a worker thread, but may also run on the submitting thread + * when the work queue is full and work is executed synchronously. + * Guaranteed to be called regardless of whether the work finishes successfully or not. + * \param[in] user_context The user-provided context passed when configuring callbacks + * \param[in] enqueue_data Data returned by the corresponding OrtThreadPoolCallbacksConfig::on_enqueue call + */ +typedef void (*OrtThreadPoolWorkStopFn)(_In_opt_ void* user_context, _In_opt_ void* enqueue_data) NO_EXCEPTION; + +/** \brief Thread pool work abandon callback + * + * Called when enqueued work is abandoned (revoked or rejected) without execution. + * This allows the caller to free any resources associated with enqueue_data. + * \param[in] user_context The user-provided context passed when configuring callbacks + * \param[in] enqueue_data Data returned by the corresponding OrtThreadPoolCallbacksConfig::on_enqueue call + */ +typedef void (*OrtThreadPoolWorkAbandonFn)(_In_opt_ void* user_context, _In_opt_ void* enqueue_data) NO_EXCEPTION; + +/** \brief Configuration for thread pool work callbacks. + * + * A versioned struct that bundles all thread pool callback function pointers and a user context. + * Pass to OrtApi::SetPerSessionThreadPoolCallbacks. + * + * \note The version field must be set to ORT_API_VERSION. + * \note New fields MUST only be appended at the end of this struct. + * + * \since Version 1.25. + */ +typedef struct OrtThreadPoolCallbacksConfig { + uint32_t version; /**< Must be ORT_API_VERSION */ + OrtThreadPoolWorkEnqueueFn on_enqueue; /**< Called when work is enqueued. May be NULL. */ + OrtThreadPoolWorkStartFn on_start_work; /**< Called when work starts. May be NULL. */ + OrtThreadPoolWorkStopFn on_stop_work; /**< Called when work completes. May be NULL. */ + OrtThreadPoolWorkAbandonFn on_abandon; /**< Called when work is abandoned. May be NULL. */ + void* user_context; /**< User-provided context passed to all callbacks. May be NULL. + Must remain valid for the lifetime of the session. + Subject to concurrent invocations from multiple threads + and must be thread-safe. May affect inference performance. */ +} OrtThreadPoolCallbacksConfig; + +typedef OrtStatus*(ORT_API_CALL* RegisterCustomOpsFn)(OrtSessionOptions* options, const OrtApiBase* api); + +/** \brief Callback function for RunAsync + * + * \param[in] user_data User specific data that passed back to the callback + * \param[out] outputs On succeed, outputs host inference results, on error, the value will be nullptr + * \param[out] num_outputs Number of outputs, on error, the value will be zero + * \param[out] status On error, status will provide details + */ +typedef void (*RunAsyncCallbackFn)(void* user_data, OrtValue** outputs, size_t num_outputs, OrtStatusPtr status); + +/** \brief External memory handle type for importing GPU resources. + * + * \todo Add Linux DMA-BUF file descriptor for embedded GPU memory sharing + * + * \since Version 1.24. + */ +typedef enum OrtExternalMemoryHandleType { + ORT_EXTERNAL_MEMORY_HANDLE_TYPE_D3D12_RESOURCE = 0, /**< Shared HANDLE from ID3D12Device::CreateSharedHandle(resource) */ + ORT_EXTERNAL_MEMORY_HANDLE_TYPE_D3D12_HEAP = 1, /**< Shared HANDLE from ID3D12Device::CreateSharedHandle(heap) */ + ORT_EXTERNAL_MEMORY_HANDLE_TYPE_VK_MEMORY_WIN32 = 2, /**< Shared HANDLE from vkGetMemoryWin32HandleKHR, non-dedicated allocation */ + ORT_EXTERNAL_MEMORY_HANDLE_TYPE_VK_MEMORY_OPAQUE_FD = 3, /**< File descriptor from vkGetMemoryOpaqueFdKHR, non-dedicated allocation */ +} OrtExternalMemoryHandleType; + +/** \brief Descriptor for importing external memory. + * + * \note The version field must be set to ORT_API_VERSION. + * This ensures forward compatibility as fields may be added in future versions. + * + * \since Version 1.24. + */ +typedef struct OrtExternalMemoryDescriptor { + uint32_t version; /**< Must be ORT_API_VERSION */ + OrtExternalMemoryHandleType handle_type; /**< Type of the external memory handle */ + void* native_handle; /**< Platform-specific handle (e.g., Windows HANDLE) */ + size_t size_bytes; /**< Total size in bytes of the external allocation */ + size_t offset_bytes; /**< Offset in bytes into the allocation (default 0). + Base offset for the imported memory region. */ +} OrtExternalMemoryDescriptor; + +/** \brief External semaphore type for GPU synchronization. + * + * \since Version 1.24. + */ +typedef enum OrtExternalSemaphoreType { + ORT_EXTERNAL_SEMAPHORE_D3D12_FENCE = 0, /**< Shared HANDLE from ID3D12Device::CreateSharedHandle(fence) */ + ORT_EXTERNAL_SEMAPHORE_VK_TIMELINE_SEMAPHORE_WIN32 = 1, /**< Shared HANDLE from vkGetSemaphoreWin32HandleKHR of a VkSemaphore created as VK_SEMAPHORE_TYPE_TIMELINE */ + ORT_EXTERNAL_SEMAPHORE_VK_TIMELINE_SEMAPHORE_OPAQUE_FD = 2, /**< File descriptor from vkGetSemaphoreFdKHR of a VkSemaphore created as VK_SEMAPHORE_TYPE_TIMELINE */ +} OrtExternalSemaphoreType; + +/** \brief Descriptor for importing external semaphores. + * + * \note The version field must be set to ORT_API_VERSION. + * This ensures forward compatibility as fields may be added in future versions. + * + * \since Version 1.24. + */ +typedef struct OrtExternalSemaphoreDescriptor { + uint32_t version; /**< Must be ORT_API_VERSION */ + OrtExternalSemaphoreType type; /**< Type of the external semaphore */ + void* native_handle; /**< Platform-specific handle (e.g., Windows HANDLE) */ +} OrtExternalSemaphoreDescriptor; + +/** \brief Graphics API type for interop configuration. + * + * Specifies the graphics API used for GPU interop with the execution provider. + * This enables synchronization between graphics workloads (e.g., rendering, compute shaders) + * and ONNX Runtime inference. + * + * \since Version 1.25. + */ +typedef enum OrtGraphicsApi { + ORT_GRAPHICS_API_NONE = 0, /**< No graphics interop (default) */ + ORT_GRAPHICS_API_D3D12 = 1, /**< Direct3D 12 interop */ + ORT_GRAPHICS_API_VULKAN = 2, /**< Vulkan interop */ +} OrtGraphicsApi; + +/** \brief Configuration for initializing graphics interop on an EP factory. + * + * This structure contains all parameters needed to set up graphics interop between + * ONNX Runtime and an external graphics API (D3D12, Vulkan). The factory stores this + * configuration and uses it when creating synchronization streams. + * + * Design rationale: + * - Single init function with all required params to avoid multiple init signatures + * - Factory stores the context and uses it in stream creation + * - Supports extensibility via additional_options for future requirements + * + * Example usage for D3D12: + * \code + * const OrtInteropApi* interop_api = ort_api->GetInteropApi(); + * OrtGraphicsInteropConfig config = {0}; + * config.version = ORT_API_VERSION; + * config.graphics_api = ORT_GRAPHICS_API_D3D12; + * config.command_queue = my_d3d12_command_queue; // ID3D12CommandQueue* + * status = interop_api->InitGraphicsInteropForEpDevice(ep_device, &config); + * \endcode + * + * \note The version field must be set to ORT_API_VERSION. + * This ensures forward compatibility as fields may be added in future versions. + * + * \since Version 1.25. + */ +typedef struct OrtGraphicsInteropConfig { + uint32_t version; /**< Must be ORT_API_VERSION */ + OrtGraphicsApi graphics_api; /**< The graphics API to use for interop */ + + /** \brief Command queue/submission queue for graphics workloads (optional). + * + * Optional. When provided, the factory may use it for efficient GPU-side synchronization + * with inference streams (performance optimization). When null, the Interop API still + * works; streams use the default context. + * + * For D3D12: ID3D12CommandQueue* + * For Vulkan: pass NULL + */ + void* command_queue; + + /** \brief Additional API-specific options (optional). + * + * Can be used for future extensibility without changing the struct layout. + * For example, D3D12 fence sharing flags or provider-specific options like + * onnxruntime::nv::provider_option_names::kExternalComputeQueueDataParamNV_data + * for Vulkan interop for the NvTensorRTRTX provider. + */ + const OrtKeyValuePairs* additional_options; +} OrtGraphicsInteropConfig; + +/** \brief Descriptor for creating a tensor from imported external memory. + * + * \note The version field must be set to ORT_API_VERSION. + * This ensures forward compatibility as fields may be added in future versions. + * + * \since Version 1.24. + */ +typedef struct OrtExternalTensorDescriptor { + uint32_t version; /**< Must be ORT_API_VERSION */ + ONNXTensorElementDataType element_type; /**< Data type of tensor elements */ + const int64_t* shape; /**< Array of dimension sizes */ + size_t rank; /**< Number of dimensions */ + size_t offset_bytes; /**< Additional offset within imported memory (default 0). + Applied relative to OrtExternalMemoryDescriptor::offset_bytes. + Enables multiple tensors from the same imported memory handle. */ +} OrtExternalTensorDescriptor; + +/* + * Public enum for compiled model compatibility across EPs. + */ +typedef enum OrtCompiledModelCompatibility { + OrtCompiledModelCompatibility_EP_NOT_APPLICABLE = 0, + OrtCompiledModelCompatibility_EP_SUPPORTED_OPTIMAL, + OrtCompiledModelCompatibility_EP_SUPPORTED_PREFER_RECOMPILATION, + OrtCompiledModelCompatibility_EP_UNSUPPORTED, +} OrtCompiledModelCompatibility; + +/** \brief Configuration options for creating an OrtEnv. + * + * \note The version field must be set to ORT_API_VERSION. + * This ensures forward compatibility as fields may be added in future versions. + * + * \since Version 1.24. + */ +typedef struct OrtEnvCreationOptions { + uint32_t version; ///< Must be set to ORT_API_VERSION + + /** \brief The logging severity level for the environment. Must be set to a value from OrtLoggingLevel. + * + * \note Logging messages which are less severe than the `logging_severity_level` are not emitted. + * + * \note Serves as the default logging severity level for session creation and runs. + * Use OrtApi::SetSessionLogSeverityLevel to set a logging severity level for the creation of specific session. + * Use OrtApi::RunOptionsSetRunLogSeverityLevel to set a logging severity level for a specific session run. + * + * \since Version 1.24. + */ + int32_t logging_severity_level; + + /** \brief The log identifier. Must be set to a valid UTF-8 null-terminated string. + * + * \note This string identifier is copied by ORT. + * + * \since Version 1.24. + */ + const char* log_id; + + /** \brief Optional custom logging function. May be set to NULL. + * + * \note The OrtEnvCreationOptions::custom_logging_param is provided as the first argument to this logging function. + * This allows passing custom state into the logging function. + * + * \note This function is only called when a message's severity meets or exceeds the set logging severity level. + * + * \since Version 1.24. + */ + OrtLoggingFunction custom_logging_function; + + /** \brief Optional state to pass as the first argument to OrtEnvCreationOptions::custom_logger_function. + * May be set to NULL. + * + * \since Version 1.24. + */ + void* custom_logging_param; + + /** \brief Optional threading options for creating an environment with global thread pools shared across sessions. + * May be set to NULL. + * + * \note The OrtThreadingOptions instance is copied by ORT. + * + * \note Use OrtApi::CreateThreadingOptions() to create an instance of OrtThreadingOptions. + * + * \note Use this in conjunction with OrtApi::DisablePerSessionThreads or else the session will use its own + * thread pools. + * + * \since Version 1.24. + */ + const OrtThreadingOptions* threading_options; + + /** \brief Optional environment configuration entries represented as string key-value pairs. May be set to NULL. + * + * \note The OrtKeyValuePairs instance is copied by ORT. + * + * \note Refer to onnxruntime_env_config_keys.h for common config entry keys and their supported values. + * + * \note An application provides environment-level configuration options for execution provider libraries by + * using keys with the prefix 'ep_factory.\\.'. Ex: the key 'ep_factory.my_ep.some_ep_key' represents + * a key named 'some_ep_key' that is meant to be consumed by an execution provider named 'my_ep'. Refer to + * the specific execution provider's documentation for valid keys and values. + * + * \note An application may separately set session-level configuration options for execution providers via other APIs + * such as SessionOptionsAppendExecutionProvider_V2, which store configuration entries within OrtSessionOptions. + * If an environment-level configuration conflicts with a session-level configuration, then + * precedence is determined by the execution provider library itself. + * + * \since Version 1.24. + */ + const OrtKeyValuePairs* config_entries; + + // + // End of fields available in ORT 1.24 + // + +} OrtEnvCreationOptions; + +/** \brief The C API + * + * All C API functions are defined inside this structure as pointers to functions. + * Call OrtApiBase::GetApi to get a pointer to it + * + * \nosubgrouping + */ +struct OrtApi { + /// \name OrtStatus + /// @{ + + /** + * \brief Create an OrtStatus from a null terminated string + * + * \param[in] code + * \param[in] msg A null-terminated string. Its contents will be copied. + * \return A new OrtStatus object, must be destroyed with OrtApi::ReleaseStatus + */ + OrtStatus*(ORT_API_CALL* CreateStatus)(OrtErrorCode code, _In_ const char* msg)NO_EXCEPTION ORT_ALL_ARGS_NONNULL; + + /** \brief Get OrtErrorCode from OrtStatus + * + * \param[in] status + * \return OrtErrorCode that \p status was created with + */ + OrtErrorCode(ORT_API_CALL* GetErrorCode)(_In_ const OrtStatus* status) NO_EXCEPTION ORT_ALL_ARGS_NONNULL; + + /** \brief Get error string from OrtStatus + * + * \param[in] status + * \return The error message inside the `status`. Do not free the returned value. + */ + const char*(ORT_API_CALL* GetErrorMessage)(_In_ const OrtStatus* status)NO_EXCEPTION ORT_ALL_ARGS_NONNULL; + + /// @} + /// \name OrtEnv + /// @{ + + /** \brief Create an OrtEnv + * + * \note Invoking this function will return the same instance of the environment as that returned by a previous call + * to another env creation function; all arguments to this function will be ignored. + * \param[in] log_severity_level The log severity level. + * \param[in] logid The log identifier. + * \param[out] out Returned newly created OrtEnv. Must be freed with OrtApi::ReleaseEnv + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(CreateEnv, OrtLoggingLevel log_severity_level, _In_ const char* logid, _Outptr_ OrtEnv** out); + + /** \brief Create an OrtEnv + * + * \note Invoking this function will return the same instance of the environment as that returned by a previous call + * to another env creation function; all arguments to this function will be ignored. If you want to provide your + * own logging function, consider setting it using the SetUserLoggingFunction API instead. + * \param[in] logging_function A pointer to a logging function. + * \param[in] logger_param A pointer to arbitrary data passed as the ::OrtLoggingFunction `param` parameter to + * `logging_function`. This parameter is optional. + * \param[in] log_severity_level The log severity level. + * \param[in] logid The log identifier. + * \param[out] out Returned newly created OrtEnv. Must be freed with OrtApi::ReleaseEnv + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(CreateEnvWithCustomLogger, _In_ OrtLoggingFunction logging_function, _In_opt_ void* logger_param, + _In_ OrtLoggingLevel log_severity_level, _In_ const char* logid, _Outptr_ OrtEnv** out); + + /** \brief Enable Telemetry + * + * \note Telemetry events are on by default since they are lightweight + * \param[in] env + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(EnableTelemetryEvents, _In_ const OrtEnv* env); + /** \brief Disable Telemetry + * + * \see OrtApi::EnableTelemetryEvents + * \param[in] env + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(DisableTelemetryEvents, _In_ const OrtEnv* env); + + /// @} + /// \name OrtSession + /// @{ + + /** \brief Create an OrtSession from a model file + * + * \param[in] env + * \param[in] model_path + * \param[in] options + * \param[out] out Returned newly created OrtSession. Must be freed with OrtApi::ReleaseSession + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + // TODO: document the path separator convention? '/' vs '\' + // TODO: should specify the access characteristics of model_path. Is this read only during the + // execution of CreateSession, or does the OrtSession retain a handle to the file/directory + // and continue to access throughout the OrtSession lifetime? + // What sort of access is needed to model_path : read or read/write? + ORT_API2_STATUS(CreateSession, _In_ const OrtEnv* env, _In_ const ORTCHAR_T* model_path, + _In_ const OrtSessionOptions* options, _Outptr_ OrtSession** out); + + /** \brief Create an OrtSession from memory + * + * \param[in] env + * \param[in] model_data + * \param[in] model_data_length + * \param[in] options + * \param[out] out Returned newly created OrtSession. Must be freed with OrtApi::ReleaseSession + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(CreateSessionFromArray, _In_ const OrtEnv* env, + _In_ const void* model_data, size_t model_data_length, + _In_ const OrtSessionOptions* options, _Outptr_ OrtSession** out); + + /** \brief Run the model in an ::OrtSession + * + * Will not return until the model run has completed. Multiple threads might be used to run the model based on + * the options in the ::OrtSession and settings used when creating the ::OrtEnv + * + * \param[in] session + * \param[in] run_options If nullptr, will use a default ::OrtRunOptions + * \param[in] input_names Array of null terminated UTF8 encoded strings of the input names + * \param[in] inputs Array of ::OrtValue%s of the input values + * \param[in] input_len Number of elements in the input_names and inputs arrays + * \param[in] output_names Array of null terminated UTF8 encoded strings of the output names + * \param[in] output_names_len Number of elements in the output_names and outputs array + * \param[out] outputs Array of ::OrtValue%s that the outputs are stored in. This can also be + * an array of nullptr values, in this case ::OrtValue objects will be allocated and pointers + * to them will be set into the `outputs` array. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(Run, _Inout_ OrtSession* session, _In_opt_ const OrtRunOptions* run_options, + _In_reads_(input_len) const char* const* input_names, + _In_reads_(input_len) const OrtValue* const* inputs, size_t input_len, + _In_reads_(output_names_len) const char* const* output_names, size_t output_names_len, + _Inout_updates_all_(output_names_len) OrtValue** outputs); + + /// @} + /// \name OrtSessionOptions + /// @{ + + /** \brief Create an ::OrtSessionOptions object + * + * To use additional providers, you must build ORT with the extra providers enabled. Then call one of these + * functions to enable them in the session:
+ * OrtSessionOptionsAppendExecutionProvider_CPU
+ * OrtSessionOptionsAppendExecutionProvider_CUDA
+ * OrtSessionOptionsAppendExecutionProvider_(remaining providers...)
+ * The order they are called indicates the preference order as well. In other words call this method + * on your most preferred execution provider first followed by the less preferred ones. + * If none are called Ort will use its internal CPU execution provider. + * + * \param[out] options The newly created OrtSessionOptions. Must be freed with OrtApi::ReleaseSessionOptions + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(CreateSessionOptions, _Outptr_ OrtSessionOptions** options); + + /** \brief Set filepath to save optimized model after graph level transformations + * + * \param[in] options + * \param[in] optimized_model_filepath + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(SetOptimizedModelFilePath, _Inout_ OrtSessionOptions* options, + _In_ const ORTCHAR_T* optimized_model_filepath); + + /** \brief Create a copy of an existing ::OrtSessionOptions + * + * \param[in] in_options OrtSessionOptions to copy + * \param[out] out_options Returned newly created ::OrtSessionOptions. Must be freed with OrtApi::ReleaseSessionOptions + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(CloneSessionOptions, _In_ const OrtSessionOptions* in_options, + _Outptr_ OrtSessionOptions** out_options); + + /** \brief Set execution mode + * + * Controls whether you want to execute operators in your graph sequentially or in parallel. Usually when the model + * has many branches, setting this option to ExecutionMode.ORT_PARALLEL will give you better performance. + * See [docs/ONNX_Runtime_Perf_Tuning.md] for more details. + * + * \param[in] options + * \param[in] execution_mode + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(SetSessionExecutionMode, _Inout_ OrtSessionOptions* options, ExecutionMode execution_mode); + + /** \brief Enable profiling for a session + * + * \param[in] options + * \param[in] profile_file_prefix + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(EnableProfiling, _Inout_ OrtSessionOptions* options, _In_ const ORTCHAR_T* profile_file_prefix); + + /** \brief Disable profiling for a session + * + * \param[in] options + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(DisableProfiling, _Inout_ OrtSessionOptions* options); + + /** \brief Enable the memory pattern optimization + * + * The idea is if the input shapes are the same, we could trace the internal memory allocation + * and generate a memory pattern for future request. So next time we could just do one allocation + * with a big chunk for all the internal memory allocation. + * \note Memory pattern optimization is only available when Sequential Execution mode is enabled (see OrtApi::SetSessionExecutionMode) + * + * \see OrtApi::DisableMemPattern + * + * \param[in] options + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(EnableMemPattern, _Inout_ OrtSessionOptions* options); + + /** \brief Disable the memory pattern optimization + * + * \see OrtApi::EnableMemPattern + * + * \param[in] options + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(DisableMemPattern, _Inout_ OrtSessionOptions* options); + + /** \brief Enable the memory arena on CPU + * + * Arena may pre-allocate memory for future usage. + * + * \param[in] options + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(EnableCpuMemArena, _Inout_ OrtSessionOptions* options); + + /** \brief Disable the memory arena on CPU + * + * \param[in] options + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(DisableCpuMemArena, _Inout_ OrtSessionOptions* options); + + /** \brief Set session log id + * + * \param[in] options + * \param[in] logid The log identifier. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(SetSessionLogId, _Inout_ OrtSessionOptions* options, const char* logid); + + /** \brief Set session log verbosity level + * + * Applies to session load, initialization, etc + * + * \param[in] options + * \param[in] session_log_verbosity_level \snippet{doc} snippets.dox Log Verbosity Level + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(SetSessionLogVerbosityLevel, _Inout_ OrtSessionOptions* options, int session_log_verbosity_level); + + /** \brief Set session log severity level + * + * \param[in] options + * \param[in] session_log_severity_level The log severity level (refer to ::OrtLoggingLevel for possible values). + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(SetSessionLogSeverityLevel, _Inout_ OrtSessionOptions* options, int session_log_severity_level); + + /** \brief Set the optimization level to apply when loading a graph + * + * Please see https://onnxruntime.ai/docs/performance/model-optimizations/graph-optimizations.html for an in-depth explanation + * \param[in,out] options The session options object + * \param[in] graph_optimization_level The optimization level + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(SetSessionGraphOptimizationLevel, _Inout_ OrtSessionOptions* options, + GraphOptimizationLevel graph_optimization_level); + + /** \brief Sets the number of threads used to parallelize the execution within nodes + * + * When running a single node operation, ex. add, this sets the maximum number of threads to use. + * + * \note If built with OpenMP, this has no effect on the number of threads used. In this case + * use the OpenMP env variables to configure the number of intra op num threads. + * + * \param[in] options + * \param[in] intra_op_num_threads Number of threads to use
+ * A value of 0 will use the default number of threads
+ * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(SetIntraOpNumThreads, _Inout_ OrtSessionOptions* options, int intra_op_num_threads); + + /** \brief Sets the number of threads used to parallelize the execution of the graph + * + * If nodes can be run in parallel, this sets the maximum number of threads to use to run them in parallel. + * + * \note If sequential execution is enabled this value is ignored, it acts as if it was set to 1. + * + * \param[in] options + * \param[in] inter_op_num_threads Number of threads to use
+ * A value of 0 will use the default number of threads
+ * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(SetInterOpNumThreads, _Inout_ OrtSessionOptions* options, int inter_op_num_threads); + + /// @} + /// \name OrtCustomOpDomain + /// @{ + + /** \brief Create a custom op domain + * + * \param[in] domain + * \param[out] out Newly created domain. Must be freed with OrtApi::ReleaseCustomOpDomain + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(CreateCustomOpDomain, _In_ const char* domain, _Outptr_ OrtCustomOpDomain** out); + + /** \brief Add a custom op to a custom op domain + * + * \note The OrtCustomOp* pointer must remain valid until the ::OrtCustomOpDomain using it is released + * + * \param[in] custom_op_domain + * \param[in] op + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(CustomOpDomain_Add, _Inout_ OrtCustomOpDomain* custom_op_domain, _In_ const OrtCustomOp* op); + + /// @} + /// \name OrtSessionOptions + /// @{ + + /** \brief Add custom op domain to a session options + * + * \note The OrtCustomOpDomain* must not be deleted until all sessions using it are released + * + * \param[in] options + * \param[in] custom_op_domain + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(AddCustomOpDomain, _Inout_ OrtSessionOptions* options, _In_ OrtCustomOpDomain* custom_op_domain); + + /** \deprecated Use OrtApi::RegisterCustomOpsLibrary_V2. + * + * Registers custom ops from a shared library. + * + * Loads a shared library (dll on windows, so on linux, etc) named 'library_path' and looks for this entry point: + * OrtStatus* RegisterCustomOps(OrtSessionOptions * options, const OrtApiBase* api); + * It then passes in the provided session options to this function along with the api base. + * The handle to the loaded library is returned in library_handle. It can be freed by the caller after all sessions using the passed in + * session options are destroyed, or if an error occurs and it is non null. + * + * \param[in] options + * \param[in] library_path + * \param[out] library_handle OS specific handle to the loaded library (Use FreeLibrary on Windows, dlclose on Linux, etc.. to unload) + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(RegisterCustomOpsLibrary, _Inout_ OrtSessionOptions* options, _In_ const char* library_path, _Outptr_ void** library_handle); + + /// @} + /// \name OrtSession + /// @{ + + /** \brief Get input count for a session + * + * This number must also match the number of inputs passed to OrtApi::Run + * + * \see OrtApi::SessionGetInputTypeInfo, OrtApi::SessionGetInputName, OrtApi::Session + * + * \param[in] session + * \param[out] out Number of inputs + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(SessionGetInputCount, _In_ const OrtSession* session, _Out_ size_t* out); + + /** \brief Get output count for a session + * + * This number must also match the number of outputs returned by OrtApi::Run + * + * \see OrtApi::SessionGetOutputTypeInfo, OrtApi::SessionGetOutputName, OrtApi::Session + * + * \param[in] session + * \param[out] out Number of outputs + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(SessionGetOutputCount, _In_ const OrtSession* session, _Out_ size_t* out); + + /** \brief Get overridable initializer count + * + * \see OrtApi::SessionGetOverridableInitializerTypeInfo, OrtApi::SessionGetOverridableInitializerName + * + * \param[in] session + * \param[in] out + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(SessionGetOverridableInitializerCount, _In_ const OrtSession* session, _Out_ size_t* out); + + /** \brief Get input type information + * + * \param[in] session + * \param[in] index Must be between 0 (inclusive) and what OrtApi::SessionGetInputCount returns (exclusive) + * \param[out] type_info Must be freed with OrtApi::ReleaseTypeInfo + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(SessionGetInputTypeInfo, _In_ const OrtSession* session, size_t index, _Outptr_ OrtTypeInfo** type_info); + + /** \brief Get output type information + * + * \param[in] session + * \param[in] index Must be between 0 (inclusive) and what OrtApi::SessionGetOutputCount returns (exclusive) + * \param[out] type_info Must be freed with OrtApi::ReleaseTypeInfo + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(SessionGetOutputTypeInfo, _In_ const OrtSession* session, size_t index, _Outptr_ OrtTypeInfo** type_info); + + /** \brief Get overridable initializer type information + * + * \param[in] session + * \param[in] index Must be between 0 (inclusive) and what OrtApi::SessionGetOverridableInitializerCount returns (exclusive) + * \param[out] type_info Must be freed with OrtApi::ReleaseTypeInfo + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(SessionGetOverridableInitializerTypeInfo, _In_ const OrtSession* session, size_t index, _Outptr_ OrtTypeInfo** type_info); + + /** \brief Get input name + * + * \param[in] session + * \param[in] index Must be between 0 (inclusive) and what OrtApi::SessionGetInputCount returns (exclusive) + * \param[in] allocator + * \param[out] value Set to a null terminated UTF-8 encoded string allocated using `allocator`. Must be freed using `allocator`. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(SessionGetInputName, _In_ const OrtSession* session, size_t index, _Inout_ OrtAllocator* allocator, _Outptr_ char** value); + + /** \brief Get output name + * + * \param[in] session + * \param[in] index Must be between 0 (inclusive) and what OrtApi::SessionGetOutputCount returns (exclusive) + * \param[in] allocator + * \param[out] value Set to a null terminated UTF-8 encoded string allocated using `allocator`. Must be freed using `allocator`. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(SessionGetOutputName, _In_ const OrtSession* session, size_t index, _Inout_ OrtAllocator* allocator, _Outptr_ char** value); + + /** \brief Get overridable initializer name + * + * \param[in] session + * \param[in] index Must be between 0 (inclusive) and what OrtApi::SessionGetOverridableInitializerCount returns (exclusive) + * \param[in] allocator + * \param[out] value Set to a null terminated UTF-8 encoded string allocated using `allocator`. Must be freed using `allocator`. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(SessionGetOverridableInitializerName, _In_ const OrtSession* session, size_t index, + _Inout_ OrtAllocator* allocator, _Outptr_ char** value); + + /// @} + /// \name OrtRunOptions + /// @{ + + /** \brief Create an OrtRunOptions + * + * \param[out] out Returned newly created ::OrtRunOptions. Must be freed with OrtApi::ReleaseRunOptions + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(CreateRunOptions, _Outptr_ OrtRunOptions** out); + + /** \brief Set per-run log verbosity level + * + * \see OrtApi::RunOptionsGetRunLogVerbosityLevel + * + * \param[in] options + * \param[in] log_verbosity_level \snippet{doc} snippets.dox Log Verbosity Level + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(RunOptionsSetRunLogVerbosityLevel, _Inout_ OrtRunOptions* options, int log_verbosity_level); + + /** \brief Set per-run log severity level + * + * \see OrtApi::RunOptionsGetRunLogSeverityLevel + * + * \param[in] options + * \param[in] log_severity_level The log severity level (refer to ::OrtLoggingLevel for possible values). + */ + ORT_API2_STATUS(RunOptionsSetRunLogSeverityLevel, _Inout_ OrtRunOptions* options, int log_severity_level); + + /** \brief Set per-run tag + * + * This is used in a per-run log identifier. + * + * \see OrtApi::RunOptionsGetRunTag + * + * \param[in] options + * \param[in] run_tag The run tag. + */ + ORT_API2_STATUS(RunOptionsSetRunTag, _Inout_ OrtRunOptions* options, _In_ const char* run_tag); + + /** \brief Get per-run log verbosity level + * + * \see OrtApi::RunOptionsSetRunLogVerbosityLevel + * + * \param[in] options + * \param[out] log_verbosity_level \snippet{doc} snippets.dox Log Verbosity Level + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(RunOptionsGetRunLogVerbosityLevel, _In_ const OrtRunOptions* options, + _Out_ int* log_verbosity_level); + + /** \brief Get per-run log severity level + * + * \see OrtApi::RunOptionsSetRunLogSeverityLevel + * + * \param[in] options + * \param[out] log_severity_level The log severity level (refer to ::OrtLoggingLevel for possible values). + */ + ORT_API2_STATUS(RunOptionsGetRunLogSeverityLevel, _In_ const OrtRunOptions* options, _Out_ int* log_severity_level); + + /** \brief Get per-run tag + * + * This is used in a per-run log identifier. + * + * \see OrtApi::RunOptionsSetRunTag + * + * \param[in] options + * \param[out] run_tag The run tag. + * Do not free this value, it is owned by `options`. It will be invalidated if the run tag + * changes (i.e., with OrtApi::RunOptionsSetRunTag) or `options` is freed. + */ + ORT_API2_STATUS(RunOptionsGetRunTag, _In_ const OrtRunOptions* options, _Out_ const char** run_tag); + + /** \brief Set terminate flag + * + * If a currently executing session needs to be force terminated, this can be called from another thread to force it to fail with an error. + * + * \param[in] options + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(RunOptionsSetTerminate, _Inout_ OrtRunOptions* options); + + /** \brief Clears the terminate flag + * + * Used so the OrtRunOptions instance can be used in a new OrtApi::Run call without it instantly terminating + * + * \param[in] options + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(RunOptionsUnsetTerminate, _Inout_ OrtRunOptions* options); + + /// @} + /// \name OrtValue + /// @{ + + /** \brief Create a tensor + * + * Create a tensor using a supplied ::OrtAllocator + * + * \param[in] allocator + * \param[in] shape Pointer to the tensor shape dimensions. + * \param[in] shape_len The number of tensor shape dimensions. + * \param[in] type + * \param[out] out Returns newly created ::OrtValue. Must be freed with OrtApi::ReleaseValue + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(CreateTensorAsOrtValue, _Inout_ OrtAllocator* allocator, _In_ const int64_t* shape, size_t shape_len, + ONNXTensorElementDataType type, _Outptr_ OrtValue** out); + + /** \brief Create a tensor backed by a user supplied buffer + * + * Create a tensor with user's buffer. You can fill the buffer either before calling this function or after. + * p_data is owned by caller. ReleaseValue won't release p_data. + * + * If you wish to transfer ownership of p_data to ORT use CreateTensorWithDataAndDeleterAsOrtValue. + * + * \param[in] info Memory description of where the p_data buffer resides (CPU vs GPU etc). + * \param[in] p_data Pointer to the data buffer. + * \param[in] p_data_len The number of bytes in the data buffer. + * \param[in] shape Pointer to the tensor shape dimensions. + * \param[in] shape_len The number of tensor shape dimensions. + * \param[in] type The data type. + * \param[out] out Returns newly created ::OrtValue. Must be freed with OrtApi::ReleaseValue + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(CreateTensorWithDataAsOrtValue, _In_ const OrtMemoryInfo* info, _Inout_ void* p_data, + size_t p_data_len, _In_ const int64_t* shape, size_t shape_len, ONNXTensorElementDataType type, + _Outptr_ OrtValue** out); + + /** \brief Return if an ::OrtValue is a tensor type + * + * \param[in] value A tensor type (string tensors are not supported) + * \param[out] out Set to 1 iff ::OrtValue is a tensor, 0 otherwise + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(IsTensor, _In_ const OrtValue* value, _Out_ int* out); + + /** \brief Get a pointer to the raw data inside a tensor + * + * Used to read/write/modify the internal tensor data directly. + * \note The returned pointer is valid until the \p value is destroyed. + * + * \param[in] value A tensor type (string tensors are not supported) + * \param[out] out Filled in with a pointer to the internal storage + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(GetTensorMutableData, _In_ OrtValue* value, _Outptr_ void** out); + + /** \brief Set all strings at once in a string tensor + * + * \param[in,out] value A tensor of type ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING + * \param[in] s An array of strings. Each string in this array must be null terminated. + * \param[in] s_len Count of strings in s (Must match the size of \p value's tensor shape) + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(FillStringTensor, _Inout_ OrtValue* value, _In_ const char* const* s, size_t s_len); + + /** \brief Get total byte length for all strings in a string tensor + * + * Typically used with OrtApi::GetStringTensorContent + * + * \param[in] value A tensor of type ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING + * \param[out] len Total byte length of all strings (does not include trailing nulls) + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(GetStringTensorDataLength, _In_ const OrtValue* value, _Out_ size_t* len); + + /** \brief Get all strings from a string tensor + * + * An example of the results:
+ * Given \p value is a string tensor with the strings { "This" "is" "a" "test" }
+ * \p s must have a size of 11 bytes
+ * \p offsets must have 4 elements
+ * After the call, these values will be filled in:
+ * \p s will contain "Thisisatest"
+ * \p offsets will contain { 0, 4, 6, 7 }
+ * The length of the last string is just s_len - offsets[last] + * + * \param[in] value A tensor of type ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING + * \param[in] s Buffer to sequentially write all tensor strings to. Each string is NOT null-terminated. + * \param[in] s_len Number of bytes of buffer pointed to by \p s (Get it from OrtApi::GetStringTensorDataLength) + * \param[out] offsets Array of start offsets into the strings written to \p s + * \param[in] offsets_len Number of elements in offsets + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(GetStringTensorContent, _In_ const OrtValue* value, _Out_writes_bytes_all_(s_len) void* s, + size_t s_len, _Out_writes_all_(offsets_len) size_t* offsets, size_t offsets_len); + + /// @} + /// \name OrtTypeInfo + /// @{ + + /** \brief Get ::OrtTensorTypeAndShapeInfo from an ::OrtTypeInfo + * + * \param[in] type_info + * \param[out] out Do not free this value, it will be valid until type_info is freed. + * If type_info does not represent tensor, this value will be set to nullptr. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(CastTypeInfoToTensorInfo, _In_ const OrtTypeInfo* type_info, + _Outptr_result_maybenull_ const OrtTensorTypeAndShapeInfo** out); + + /** \brief Get ::ONNXType from ::OrtTypeInfo + * + * \param[in] type_info + * \param[out] out + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(GetOnnxTypeFromTypeInfo, _In_ const OrtTypeInfo* type_info, _Out_ enum ONNXType* out); + + /// @} + /// \name OrtTensorTypeAndShapeInfo + /// @{ + + /** \brief Create an ::OrtTensorTypeAndShapeInfo object + * + * \param[out] out Returns newly created ::OrtTensorTypeAndShapeInfo. Must be freed with OrtApi::ReleaseTensorTypeAndShapeInfo + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(CreateTensorTypeAndShapeInfo, _Outptr_ OrtTensorTypeAndShapeInfo** out); + + /** \brief Set element type in ::OrtTensorTypeAndShapeInfo + * + * \param[in] info + * \param[in] type + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(SetTensorElementType, _Inout_ OrtTensorTypeAndShapeInfo* info, enum ONNXTensorElementDataType type); + + /** \brief Set shape information in ::OrtTensorTypeAndShapeInfo + * + * \param[in] info + * \param[in] dim_values Array with `dim_count` elements. Can contain negative values. + * \param[in] dim_count Number of elements in `dim_values` + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(SetDimensions, OrtTensorTypeAndShapeInfo* info, _In_ const int64_t* dim_values, size_t dim_count); + + /** \brief Get element type in ::OrtTensorTypeAndShapeInfo + * + * \see OrtApi::SetTensorElementType + * + * \param[in] info + * \param[out] out + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(GetTensorElementType, _In_ const OrtTensorTypeAndShapeInfo* info, + _Out_ enum ONNXTensorElementDataType* out); + + /** \brief Get dimension count in ::OrtTensorTypeAndShapeInfo + * + * \see OrtApi::GetDimensions + * + * \param[in] info + * \param[out] out + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(GetDimensionsCount, _In_ const OrtTensorTypeAndShapeInfo* info, _Out_ size_t* out); + + /** \brief Get dimensions in ::OrtTensorTypeAndShapeInfo + * + * \param[in] info + * \param[out] dim_values Array with `dim_values_length` elements. On return, filled with the dimensions stored in the ::OrtTensorTypeAndShapeInfo + * \param[in] dim_values_length Number of elements in `dim_values`. Use OrtApi::GetDimensionsCount to get this value + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(GetDimensions, _In_ const OrtTensorTypeAndShapeInfo* info, _Out_ int64_t* dim_values, + size_t dim_values_length); + + /** \brief Get symbolic dimension names in ::OrtTensorTypeAndShapeInfo + * + * \param[in] info + * \param[in] dim_params Array with `dim_params_length` elements. On return filled with pointers to null terminated strings of the dimension names + * \param[in] dim_params_length Number of elements in `dim_params`. Use OrtApi::GetDimensionsCount to get this value + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(GetSymbolicDimensions, _In_ const OrtTensorTypeAndShapeInfo* info, + _Out_writes_all_(dim_params_length) const char* dim_params[], size_t dim_params_length); + + /** \brief Get total number of elements in a tensor shape from an ::OrtTensorTypeAndShapeInfo + * + * Return the number of elements specified by the tensor shape (all dimensions multiplied by each other). + * For 0 dimensions, 1 is returned. If any dimension is less than 0, the result is always -1. + * + * Examples:
+ * [] = 1
+ * [1,3,4] = 12
+ * [2,0,4] = 0
+ * [-1,3,4] = -1
+ * + * \param[in] info + * \param[out] out Number of elements + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(GetTensorShapeElementCount, _In_ const OrtTensorTypeAndShapeInfo* info, _Out_ size_t* out); + + /// @} + /// \name OrtValue + /// @{ + + /** \brief Get type and shape information from a tensor ::OrtValue + * + * \param[in] value Must be a tensor (not a map/sequence/etc) or will return failure + * \param[out] out Newly created ::OrtTensorTypeAndShapeInfo. Must be freed with OrtApi::ReleaseTensorTypeAndShapeInfo + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(GetTensorTypeAndShape, _In_ const OrtValue* value, _Outptr_ OrtTensorTypeAndShapeInfo** out); + + /** \brief Get type information of an OrtValue + * + * \param[in] value + * \param[out] out Newly created ::OrtTypeInfo. Must be freed with OrtApi::ReleaseTypeInfo + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(GetTypeInfo, _In_ const OrtValue* value, _Outptr_result_maybenull_ OrtTypeInfo** out); + + /** \brief Get ONNXType of an ::OrtValue + * + * \param[in] value + * \param[out] out + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(GetValueType, _In_ const OrtValue* value, _Out_ enum ONNXType* out); + + /// @} + /// \name OrtMemoryInfo + /// @{ + + /** \brief Create an ::OrtMemoryInfo + * + * \param[in] name + * \param[in] type + * \param[in] id + * \param[in] mem_type + * \param[out] out Newly created ::OrtMemoryInfo. Must be freed with OrtAPi::ReleaseMemoryInfo + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(CreateMemoryInfo, _In_ const char* name, enum OrtAllocatorType type, int id, + enum OrtMemType mem_type, _Outptr_ OrtMemoryInfo** out); + + /** \brief Create an ::OrtMemoryInfo for CPU memory + * + * Special case version of OrtApi::CreateMemoryInfo for CPU based memory. Same as using OrtApi::CreateMemoryInfo with name = "Cpu" and id = 0. + * + * \param[in] type + * \param[in] mem_type + * \param[out] out + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(CreateCpuMemoryInfo, enum OrtAllocatorType type, enum OrtMemType mem_type, + _Outptr_ OrtMemoryInfo** out); + + /** \brief Compare ::OrtMemoryInfo objects for equality + * + * Compares all settings of each ::OrtMemoryInfo for equality + * + * \param[in] info1 + * \param[in] info2 + * \param[out] out Set to 0 if equal, -1 if not equal + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(CompareMemoryInfo, _In_ const OrtMemoryInfo* info1, _In_ const OrtMemoryInfo* info2, _Out_ int* out); + + /** \brief Get name from ::OrtMemoryInfo + * + * \param[in] ptr + * \param[out] out Writes null terminated string to this pointer. Do NOT free the returned pointer. It is valid for the lifetime of the ::OrtMemoryInfo + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(MemoryInfoGetName, _In_ const OrtMemoryInfo* ptr, _Out_ const char** out); + + /** \brief Get the device id from ::OrtMemoryInfo + */ + ORT_API2_STATUS(MemoryInfoGetId, _In_ const OrtMemoryInfo* ptr, _Out_ int* out); + + /** \brief Get the ::OrtMemType from ::OrtMemoryInfo + */ + ORT_API2_STATUS(MemoryInfoGetMemType, _In_ const OrtMemoryInfo* ptr, _Out_ OrtMemType* out); + + /** \brief Get the ::OrtAllocatorType from ::OrtMemoryInfo + */ + ORT_API2_STATUS(MemoryInfoGetType, _In_ const OrtMemoryInfo* ptr, _Out_ OrtAllocatorType* out); + + /// @} + /// \name OrtAllocator + /// @{ + + /// \brief Calls OrtAllocator::Alloc function + ORT_API2_STATUS(AllocatorAlloc, _Inout_ OrtAllocator* ort_allocator, size_t size, _Outptr_ void** out); + /// \brief Calls OrtAllocator::Free function + ORT_API2_STATUS(AllocatorFree, _Inout_ OrtAllocator* ort_allocator, void* p); + /// \brief Calls OrtAllocator::Info function + ORT_API2_STATUS(AllocatorGetInfo, _In_ const OrtAllocator* ort_allocator, _Outptr_ const struct OrtMemoryInfo** out); + + /** \brief Get the default allocator + * + * The default allocator is a CPU based, non-arena. Always returns the same pointer to the same default allocator. + * + * \param[out] out Returned value should NOT be freed + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(GetAllocatorWithDefaultOptions, _Outptr_ OrtAllocator** out); + + /// @} + /// \name OrtSessionOptions + /// @{ + + /** \brief Override session symbolic dimensions + * + * Override symbolic dimensions (by specific denotation strings) with actual values if known at session initialization time to enable + * optimizations that can take advantage of fixed values (such as memory planning, etc) + * + * \param[in] options + * \param[in] dim_denotation + * \param[in] dim_value + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(AddFreeDimensionOverride, _Inout_ OrtSessionOptions* options, _In_ const char* dim_denotation, + _In_ int64_t dim_value); + + /// @} + /// \name OrtValue + /// @{ + + /* Internal information (not seen in Doxygen) + * + * APIs to support non-tensor types - map and sequence. + * Currently only the following types are supported + * Note: the following types should be kept in sync with data_types.h + * Map types + * ========= + * std::map + * std::map + * std::map + * std::map + * std::map + * std::map + * std::map + * std::map + * + * Sequence types + * ============== + * std::vector + * std::vector + * std::vector + * std::vector + * std::vector> + * std::vector + */ + + /** \brief Get non tensor data from an ::OrtValue + * + * If `value` is of type ONNX_TYPE_MAP, you need to retrieve the keys and values + * separately. Use index=0 to retrieve keys and index=1 to retrieve values. + * If `value` is of type ONNX_TYPE_SEQUENCE, use index to retrieve the index'th element + * of the sequence. + * + * \param[in] value + * \param[in] index See above for usage based on `value` type + * \param[in] allocator Allocator used to allocate ::OrtValue + * \param[out] out Created ::OrtValue that holds the element requested. Must be freed with OrtApi::ReleaseValue + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(GetValue, _In_ const OrtValue* value, int index, _Inout_ OrtAllocator* allocator, + _Outptr_ OrtValue** out); + + /** \brief Get non tensor value count from an ::OrtValue + * + * If `value` is of type ONNX_TYPE_MAP 2 will always be returned. For ONNX_TYPE_SEQUENCE + * the number of elements in the sequence will be returned + * + * \param[in] value + * \param[out] out + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(GetValueCount, _In_ const OrtValue* value, _Out_ size_t* out); + + /** \brief Create a map or sequence ::OrtValue + * + * To construct a map (ONNX_TYPE_MAP), use num_values = 2 and `in` should be an array of 2 ::OrtValue%s + * representing keys and values.
+ * + * To construct a sequence (ONNX_TYPE_SEQUENCE), use num_values = N where N is the number of the elements in the + * sequence. 'in' should be an array of N ::OrtValue%s. + * + * \param[in] in See above for details + * \param[in] num_values + * \param[in] value_type Must be either ONNX_TYPE_MAP or ONNX_TYPE_SEQUENCE + * \param[out] out Newly created ::OrtValue. Must be freed with OrtApi::ReleaseValue + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(CreateValue, _In_reads_(num_values) const OrtValue* const* in, size_t num_values, + enum ONNXType value_type, _Outptr_ OrtValue** out); + + /** \brief Create an opaque (custom user defined type) ::OrtValue + * + * Constructs an ::OrtValue that contains a value of non-standard type created for + * experiments or while awaiting standardization. ::OrtValue in this case would contain + * an internal representation of the Opaque type. Opaque types are distinguished from + * each other by two strings 1) domain and 2) type name. The combination of the two + * must be unique, so the type representation is properly identified internally. The combination + * must be properly registered from within ORT at both compile/run time or by another API. + * + * To construct the ::OrtValue pass domain and type names, also a pointer to a data container + * the type of which must be known to both ORT and the client program. That data container may or may + * not match the internal representation of the Opaque type. The sizeof(data_container) is passed for + * verification purposes. + * + * \param[in] domain_name Null terminated string of the domain name + * \param[in] type_name Null terminated string of the type name + * \param[in] data_container User pointer Data to populate ::OrtValue + * \param[in] data_container_size Size in bytes of what `data_container` points to + * \param[out] out Newly created ::OrtValue. Must be freed with OrtApi::ReleaseValue + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(CreateOpaqueValue, _In_z_ const char* domain_name, _In_z_ const char* type_name, + _In_ const void* data_container, size_t data_container_size, _Outptr_ OrtValue** out); + + /** \brief Get internal data from an opaque (custom user defined type) ::OrtValue + * + * Copies internal data from an opaque value into a user provided buffer + * + * \see OrtApi::CreateOpaqueValue + * + * \param[in] domain_name Null terminated string of the domain name + * \param[in] type_name Null terminated string of the type name + * \param[in] in The opaque ::OrtValue + * \param[out] data_container Buffer to copy data into + * \param[out] data_container_size Size in bytes of the buffer pointed to by data_container. Must match the size of the internal buffer. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(GetOpaqueValue, _In_ const char* domain_name, _In_ const char* type_name, _In_ const OrtValue* in, + _Out_ void* data_container, size_t data_container_size); + + /// @} + /// \name OrtKernelInfo + /// Custom operator APIs. + /// @{ + + /** \brief Get a float stored as an attribute in the graph node + * + * \param[in] info ::OrtKernelInfo instance + * \param[in] name Null terminated string of the name of the attribute + * \param[out] out Pointer to memory where the attribute will be stored + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(KernelInfoGetAttribute_float, _In_ const OrtKernelInfo* info, _In_ const char* name, + _Out_ float* out); + + /** \brief Fetch a 64-bit int stored as an attribute in the graph node + * + * \param[in] info ::OrtKernelInfo instance + * \param[in] name Null terminated string of the name of the attribute + * \param[out] out Pointer to memory where the attribute will be stored + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(KernelInfoGetAttribute_int64, _In_ const OrtKernelInfo* info, _In_ const char* name, + _Out_ int64_t* out); + + /** \brief Fetch a string stored as an attribute in the graph node + * + * If `out` is nullptr, the value of `size` is set to the true size of the string + * attribute, and a success status is returned. + * + * If the `size` parameter is greater than or equal to the actual string attribute's size, + * the value of `size` is set to the true size of the string attribute, the provided memory + * is filled with the attribute's contents, and a success status is returned. + * + * If the `size` parameter is less than the actual string attribute's size and `out` + * is not nullptr, the value of `size` is set to the true size of the string attribute + * and a failure status is returned.) + * + * \param[in] info ::OrtKernelInfo instance + * \param[in] name Null terminated string of the name of the attribute + * \param[out] out Pointer to memory where the attribute will be stored + * \param[in,out] size See above comments for details + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(KernelInfoGetAttribute_string, _In_ const OrtKernelInfo* info, _In_ const char* name, _Out_ char* out, + _Inout_ size_t* size); + + /// @} + /// \name OrtKernelContext + /// Custom operator APIs. + /// @{ + + /** \brief Used for custom operators, get the input count of a kernel + * + * \see ::OrtCustomOp + */ + ORT_API2_STATUS(KernelContext_GetInputCount, _In_ const OrtKernelContext* context, _Out_ size_t* out); + + /** \brief Used for custom operators, get the output count of a kernel + * + * \see ::OrtCustomOp + */ + ORT_API2_STATUS(KernelContext_GetOutputCount, _In_ const OrtKernelContext* context, _Out_ size_t* out); + + /** \brief Used for custom operators, get an input of a kernel + * + * The function attempts fetches the input of the kernel. If the input is optional + * and not present, the function returns success and out is set to nullptr. + * + * \param[in] context ::OrtKernelContext instance + * \param[in] index See KernelContext_GetInputCount for boundaries check. + * \param[out] out OrtValue if the input is present otherwise is set nullptr + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(KernelContext_GetInput, _In_ const OrtKernelContext* context, _In_ size_t index, + _Out_ const OrtValue** out); + + /** \brief Used for custom operators, get an output of a kernel + * + * The function attempts fetches the output of the kernel. If the output is optional + * and not present, the function returns success and out is set to nullptr. + * + * \param[in] context ::OrtKernelContext instance + * \param[in] index See KernelContext_GetOutputCount for boundaries check. + * \param[in] dim_values output dimensions + * \param[in] dim_count number of dimensions + * \param[out] out a ptr to OrtValue to output otherwise set to nullptr + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(KernelContext_GetOutput, _Inout_ OrtKernelContext* context, _In_ size_t index, + _In_ const int64_t* dim_values, size_t dim_count, _Outptr_ OrtValue** out); + + /// @} + /// \name OrtEnv + /// @{ + ORT_CLASS_RELEASE(Env); + /// @} + /// \name OrtStatus + /// @{ + ORT_CLASS_RELEASE(Status); + /// @} + /// \name OrtMemoryInfo + /// @{ + ORT_CLASS_RELEASE(MemoryInfo); + /// @} + /// \name OrtSession + /// @{ + ORT_CLASS_RELEASE(Session); // Don't call ReleaseSession from Dllmain (because session owns a thread pool) + /// @} + /// \name OrtValue + /// @{ + ORT_CLASS_RELEASE(Value); + /// @} + /// \name OrtRunOptions + /// @{ + ORT_CLASS_RELEASE(RunOptions); + /// @} + /// \name OrtTypeInfo + /// @{ + ORT_CLASS_RELEASE(TypeInfo); + /// @} + /// \name OrtTensorTypeAndShapeInfo + /// @{ + ORT_CLASS_RELEASE(TensorTypeAndShapeInfo); + /// @} + /// \name OrtSessionOptions + /// @{ + ORT_CLASS_RELEASE(SessionOptions); + /// @} + /// \name OrtCustomOpDomain + /// @{ + ORT_CLASS_RELEASE(CustomOpDomain); + + /// @} + /// \name OrtTypeInfo + /// @{ + + /** \brief Get denotation from type information + * + * Augments ::OrtTypeInfo to return denotations on the type. + * + * This is used by WinML to determine if an input/output is intended to be an Image or a Tensor. + * + * \param[in] type_info + * \param[out] denotation Pointer to the null terminated denotation string is written to this pointer. This pointer is valid until the object is destroyed or the name is changed, do not free. + * \param[out] len Length in bytes of the string returned in `denotation` + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(GetDenotationFromTypeInfo, _In_ const OrtTypeInfo* type_info, _Out_ const char** const denotation, + _Out_ size_t* len); + + /** \brief Get detailed map information from an ::OrtTypeInfo + * + * This augments ::OrtTypeInfo to return an ::OrtMapTypeInfo when the type is a map. + * The OrtMapTypeInfo has additional information about the map's key type and value type. + * + * This is used by WinML to support model reflection APIs. + * + * \param[out] type_info + * \param[out] out A pointer to the ::OrtMapTypeInfo. Do not free this value. If type_info + * does not contain a map, this value will be set to nullptr. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(CastTypeInfoToMapTypeInfo, _In_ const OrtTypeInfo* type_info, + _Outptr_result_maybenull_ const OrtMapTypeInfo** out); + + /** \brief Cast ::OrtTypeInfo to an ::OrtSequenceTypeInfo + * + * This api augments ::OrtTypeInfo to return an ::OrtSequenceTypeInfo when the type is a sequence. + * The ::OrtSequenceTypeInfo has additional information about the sequence's element type. + * + * This is used by WinML to support model reflection APIs. + * + * \param[in] type_info + * \param[out] out A pointer to the OrtSequenceTypeInfo. Do not free this value. If type_info + * doesn not contain a sequence, this value will be set to nullptr. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(CastTypeInfoToSequenceTypeInfo, _In_ const OrtTypeInfo* type_info, + _Outptr_result_maybenull_ const OrtSequenceTypeInfo** out); + + /// @} + /// \name OrtMapTypeInfo + /// @{ + + /** \brief Get key type from an ::OrtMapTypeInfo + * + * Key types are restricted to being scalar types. + * + * This is used by WinML to support model reflection APIs. + * + * \param[in] map_type_info + * \param[out] out + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(GetMapKeyType, _In_ const OrtMapTypeInfo* map_type_info, _Out_ enum ONNXTensorElementDataType* out); + + /** \brief Get the value type from an ::OrtMapTypeInfo + * + * \param[in] map_type_info + * \param[out] type_info A copy of the OrtTypeInfo for the map value type. + * The user must free this value with ReleaseTypeInfo. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(GetMapValueType, _In_ const OrtMapTypeInfo* map_type_info, _Outptr_ OrtTypeInfo** type_info); + + /// @} + /// \name OrtSequenceTypeInfo + /// @{ + + /** \brief Get element type from an ::OrtSequenceTypeInfo + * + * This is used by WinML to support model reflection APIs. + * + * \param[in] sequence_type_info + * \param[out] type_info A copy of the OrtTypeInfo for the sequence element type. + * The user must free this value with ReleaseTypeInfo. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(GetSequenceElementType, _In_ const OrtSequenceTypeInfo* sequence_type_info, + _Outptr_ OrtTypeInfo** type_info); + + /// @} + /// \name OrtMapTypeInfo + /// @{ + ORT_CLASS_RELEASE(MapTypeInfo); + /// @} + /// \name OrtSequenceTypeInfo + /// @{ + ORT_CLASS_RELEASE(SequenceTypeInfo); + + /// @} + /// \name OrtSession + /// @{ + + /** \brief End profiling and return filename of the profile data + * + * Profiling is turned on through OrtApi::EnableProfiling + * + * \param[in] session + * \param[in] allocator + * \param[out] out Null terminated string of the filename, allocated using `allocator`. Must be freed using `allocator` + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(SessionEndProfiling, _In_ OrtSession* session, _Inout_ OrtAllocator* allocator, _Outptr_ char** out); + + /** \brief Get ::OrtModelMetadata from an ::OrtSession + * + * \param[in] session + * \param[out] out Newly created ::OrtModelMetadata. Must be freed using OrtApi::ReleaseModelMetadata + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(SessionGetModelMetadata, _In_ const OrtSession* session, _Outptr_ OrtModelMetadata** out); + + /// @} + /// \name OrtModelMetadata + /// @{ + + /** \brief Get `producer name` from an ::OrtModelMetadata + * + * \param[in] model_metadata + * \param[in] allocator + * \param[out] value Set to a null terminated string allocated using `allocator`. Must be freed using `allocator` + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(ModelMetadataGetProducerName, _In_ const OrtModelMetadata* model_metadata, + _Inout_ OrtAllocator* allocator, _Outptr_ char** value); + + /** \brief Get `graph name` from an ::OrtModelMetadata + * + * \param[in] model_metadata + * \param[in] allocator + * \param[out] value Set to a null terminated string allocated using `allocator`. Must be freed using `allocator` + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(ModelMetadataGetGraphName, _In_ const OrtModelMetadata* model_metadata, + _Inout_ OrtAllocator* allocator, _Outptr_ char** value); + + /** \brief Get `domain` from an ::OrtModelMetadata + * + * \param[in] model_metadata + * \param[in] allocator + * \param[out] value Set to a null terminated string allocated using `allocator`. Must be freed using `allocator` + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(ModelMetadataGetDomain, _In_ const OrtModelMetadata* model_metadata, _Inout_ OrtAllocator* allocator, + _Outptr_ char** value); + + /** \brief Get `description` from an ::OrtModelMetadata + * + * \param[in] model_metadata + * \param[in] allocator + * \param[out] value Set to a null terminated string allocated using `allocator`. Must be freed using `allocator` + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(ModelMetadataGetDescription, _In_ const OrtModelMetadata* model_metadata, + _Inout_ OrtAllocator* allocator, _Outptr_ char** value); + + /** \brief Return data for a key in the custom metadata map in an ::OrtModelMetadata + * + * \param[in] model_metadata + * \param[in] allocator + * \param[in] key Null terminated string + * \param[out] value Set to a null terminated string allocated using `allocator`. Must be freed using `allocator` + * `value` will be set to nullptr if the given key is not found in the custom metadata map. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(ModelMetadataLookupCustomMetadataMap, _In_ const OrtModelMetadata* model_metadata, + _Inout_ OrtAllocator* allocator, _In_ const char* key, _Outptr_result_maybenull_ char** value); + + /** \brief Get version number from an ::OrtModelMetadata + * + * \param[in] model_metadata + * \param[out] value Set to the version number + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(ModelMetadataGetVersion, _In_ const OrtModelMetadata* model_metadata, _Out_ int64_t* value); + + ORT_CLASS_RELEASE(ModelMetadata); + + /// @} + /// \name OrtEnv + /// @{ + + /** \brief Create an OrtEnv + * + * Create an environment with global threadpools that will be shared across sessions. + * Use this in conjunction with OrtApi::DisablePerSessionThreads or else the session will use + * its own thread pools. + * + * \param[in] log_severity_level The log severity level. + * \param[in] logid The log identifier. + * \param[in] tp_options + * \param[out] out Returned newly created OrtEnv. Must be freed with OrtApi::ReleaseEnv + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(CreateEnvWithGlobalThreadPools, OrtLoggingLevel log_severity_level, _In_ const char* logid, + _In_ const OrtThreadingOptions* tp_options, _Outptr_ OrtEnv** out); + + /// @} + /// \name OrtSessionOptions + /// @{ + + /** \brief Use global thread pool on a session + * + * Disable using per session thread pool and use the shared global threadpool. + * This should be used in conjunction with OrtApi::CreateEnvWithGlobalThreadPools. + * + * \param[in] options + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(DisablePerSessionThreads, _Inout_ OrtSessionOptions* options); + + /// @} + /// \name OrtThreadingOptions + /// @{ + + /** \brief Create an ::OrtThreadingOptions + * + * \param[out] out Newly created ::OrtThreadingOptions. Must be freed with OrtApi::ReleaseThreadingOptions + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(CreateThreadingOptions, _Outptr_ OrtThreadingOptions** out); + + ORT_CLASS_RELEASE(ThreadingOptions); + + /// @} + /// \name OrtModelMetadata + /// @{ + + /** + * + * \param[in] model_metadata + * \param[in] allocator + * \param[out] keys Array of null terminated strings (array count = num_keys) allocated using `allocator`. + * The strings and the pointer array must be freed using `allocator` + * `keys` will be set to nullptr if the custom metadata map is empty. + * \param[out] num_keys Set to the number of elements in the `keys` array + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(ModelMetadataGetCustomMetadataMapKeys, _In_ const OrtModelMetadata* model_metadata, + _Inout_ OrtAllocator* allocator, _Outptr_result_buffer_maybenull_(*num_keys) char*** keys, _Out_ int64_t* num_keys); + + /// @} + /// \name OrtSessionOptions + /// @{ + + /** + * + * Override symbolic dimensions (by specific name strings) with actual values + * if known at session initialization time to enable optimizations that can + * take advantage of fixed values (such as memory planning, etc) + * + */ + ORT_API2_STATUS(AddFreeDimensionOverrideByName, + _Inout_ OrtSessionOptions* options, _In_ const char* dim_name, + _In_ int64_t dim_value); + + /// @} + /// \name Misc + /// @{ + + /** \brief Get the names of all available providers + * + * \note The providers in the list are not guaranteed to be usable. They may fail to load due to missing system dependencies. + * For example, if the CUDA/cuDNN libraries are not installed, the CUDA provider will report an error when it is added to the session options. + * + * \param[out] out_ptr Set to a pointer to an array of null terminated strings of the available providers. The entries and the + * array itself must be freed using OrtApi::ReleaseAvailableProviders + * \param[out] provider_length Set to the number of entries in the `out_ptr` array + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(GetAvailableProviders, _Outptr_ char*** out_ptr, _Out_ int* provider_length); + + /** \brief Release data from OrtApi::GetAvailableProviders. This API will never fail + * so you can rely on it in a noexcept code. + * + * \param[in] ptr The `out_ptr` result from OrtApi::GetAvailableProviders. + * \param[in] providers_length The `provider_length` result from OrtApi::GetAvailableProviders + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(ReleaseAvailableProviders, _In_ char** ptr, + _In_ int providers_length); + + /// @} + /// \name OrtValue + /// @{ + + /** \brief Get the length of a single string in a string tensor + * + * \param[in] value A string tensor + * \param[in] index Index of the string in the tensor + * \param[out] out Set to number of bytes of the string element + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(GetStringTensorElementLength, _In_ const OrtValue* value, size_t index, _Out_ size_t* out); + + /** \brief Get a single string from a string tensor + * + * \param[in] value A string tensor + * \param[in] s_len Number of bytes in the `s` buffer. Must match the value returned by OrtApi::GetStringTensorElementLength. + * \param[in] index Index of the string in the tensor + * \param[out] s The string element contents in UTF-8 encoding. The string is NOT null-terminated. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(GetStringTensorElement, _In_ const OrtValue* value, size_t s_len, size_t index, _Out_writes_bytes_all_(s_len) void* s); + + /** \brief Set a single string in a string tensor + * + * \param[in] value A string tensor + * \param[in] s A null terminated UTF-8 encoded string + * \param[in] index Index of the string in the tensor to set + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(FillStringTensorElement, _Inout_ OrtValue* value, _In_ const char* s, size_t index); + + /// @} + /// \name OrtSessionOptions + /// @{ + + /** \brief Set a session configuration entry as a pair of strings + * + * If a configuration with same key exists, this will overwrite the configuration with the given config_value. + * + * The config_key and the format of config_value are defined in onnxruntime_session_options_config_keys.h + * + * \param[in] options + * \param[in] config_key A null terminated string representation of the config key + * \param[in] config_value A null terminated string representation of the config value + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(AddSessionConfigEntry, _Inout_ OrtSessionOptions* options, + _In_z_ const char* config_key, _In_z_ const char* config_value); + + /// @} + /// \name OrtAllocator + /// @{ + + /** \brief Create an allocator for an ::OrtSession following an ::OrtMemoryInfo + * + * The allocator wraps the internal allocator from the OrtSession and becomes invalid when the session does. + * + * \param[in] session + * \param[in] mem_info valid ::OrtMemoryInfo instance + * \param[out] out Newly created ::OrtAllocator. Must be freed with OrtApi::ReleaseAllocator + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(CreateAllocator, _In_ const OrtSession* session, _In_ const OrtMemoryInfo* mem_info, + _Outptr_ OrtAllocator** out); + + /** \brief Release an ::OrtAllocator obtained from OrtApi::CreateAllocator + */ + ORT_CLASS_RELEASE(Allocator); + + /// @} + /// \name OrtSession + /// @{ + + /** \brief Run a model using Io Bindings for the inputs & outputs + * + * \see OrtApi::Run + * + * \param[in] session + * \param[in] run_options + * \param[in] binding_ptr + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(RunWithBinding, _Inout_ OrtSession* session, _In_ const OrtRunOptions* run_options, _In_ const OrtIoBinding* binding_ptr); + + /** \brief Create an ::OrtIoBinding instance + * + * An IoBinding object allows one to bind pre-allocated ::OrtValue%s to input names. + * Thus if you want to use a raw on device buffer as input or output you can avoid + * extra copy during runtime. + * + * \param[in] session + * \param[out] out Newly created ::OrtIoBinding. Must be freed with OrtApi::ReleaseIoBinding + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(CreateIoBinding, _Inout_ OrtSession* session, _Outptr_ OrtIoBinding** out); + + /// @} + /// \name OrtIoBinding + /// @{ + + /** \brief Release an ::OrtIoBinding obtained from OrtApi::CreateIoBinding + */ + ORT_CLASS_RELEASE(IoBinding); + + /** \brief Bind an ::OrtValue to an ::OrtIoBinding input + * + * When using OrtApi::RunWithBinding this value is used for the named input + * + * \param[in] binding_ptr + * \param[in] name Name for the model input + * \param[in] val_ptr ::OrtValue of Tensor type. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(BindInput, _Inout_ OrtIoBinding* binding_ptr, _In_ const char* name, _In_ const OrtValue* val_ptr); + + /** \brief Bind an ::OrtValue to an ::OrtIoBinding output + * + * When using OrtApi::RunWithBinding this value is used for the named output + * + * \param[in] binding_ptr + * \param[in] name Null terminated string of the model output name + * \param[in] val_ptr ::OrtValue of Tensor type. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(BindOutput, _Inout_ OrtIoBinding* binding_ptr, _In_ const char* name, _In_ const OrtValue* val_ptr); + + /** \brief Bind an ::OrtIoBinding output to a device + * + * Binds the ::OrtValue to a device which is specified by ::OrtMemoryInfo. + * You can either create an instance of ::OrtMemoryInfo with a device id or obtain one from the allocator that you have created/are using + * This is useful when one or more outputs have dynamic shapes and, it is hard to pre-allocate and bind a chunk of + * memory within ::OrtValue ahead of time. + * + * \see OrtApi::RunWithBinding + * + * \param[in] binding_ptr + * \param[in] name Null terminated string of the device name + * \param[in] mem_info_ptr + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(BindOutputToDevice, _Inout_ OrtIoBinding* binding_ptr, _In_ const char* name, _In_ const OrtMemoryInfo* mem_info_ptr); + + /** \brief Get the names of an ::OrtIoBinding's outputs + * + * Returns the names of the outputs in the order they were bound. This is useful after running the model + * with bound outputs because the returned names are in order in which output ::OrtValue are returned. This is useful if + * the order of outputs and their names is not known. + * + * \param[in] binding_ptr + * \param[in] allocator Allocator used to allocate continuous buffers for output strings and lengths. + * \param[out] buffer Returns an array of non-null terminated UTF-8 strings. The number of strings stored is returned in the count parameter. + * This buffer is allocated using `allocator` and must be freed using it. + * \param[out] lengths Returns an array of `count` lengths of the strings returned in `buffer` + * This buffer is allocated using `allocator` and must be freed using it. + * \param[out] count Number of strings returned. If `binding_ptr` has no bound outputs, zero is returned, + * no memory allocation is performed and buffer and lengths are set to nullptr. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(GetBoundOutputNames, _In_ const OrtIoBinding* binding_ptr, _In_ OrtAllocator* allocator, + _Out_ char** buffer, _Out_writes_all_(count) size_t** lengths, _Out_ size_t* count); + + /** \brief Get the output ::OrtValue objects from an ::OrtIoBinding + * + * Returns an array of pointers to individually allocated ::OrtValue%s that contain results of a model execution with OrtApi::RunWithBinding + * The array contains the same number of ::OrtValue%s and they are in the same order as they were bound with OrtApi::BindOutput + * or OrtApi::BindOutputToDevice. + * + * The returned ::OrtValue%s must be released using OrtApi::ReleaseValue after they are no longer needed. + * The array is allocated using the specified instance of the allocator and must be freed using the same allocator after + * all the ::OrtValue%s contained therein are individually released. + * + * \param[in] binding_ptr + * \param[in] allocator Allocator used to allocate output array + * \param[out] output Set to the allocated array of allocated ::OrtValue outputs. Set to nullptr if there are 0 outputs. + * \param[out] output_count Set to number of ::OrtValue%s returned + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(GetBoundOutputValues, _In_ const OrtIoBinding* binding_ptr, _In_ OrtAllocator* allocator, + _Out_writes_all_(output_count) OrtValue*** output, _Out_ size_t* output_count); + + /** \brief Clears any previously set Inputs for an ::OrtIoBinding + */ + void(ORT_API_CALL* ClearBoundInputs)(_Inout_ OrtIoBinding* binding_ptr) NO_EXCEPTION ORT_ALL_ARGS_NONNULL; + + /** \brief Clears any previously set Outputs for an ::OrtIoBinding + */ + void(ORT_API_CALL* ClearBoundOutputs)(_Inout_ OrtIoBinding* binding_ptr) NO_EXCEPTION ORT_ALL_ARGS_NONNULL; + + /// @} + /// \name OrtValue + /// @{ + + /** \brief Direct memory access to a specified tensor element + * + * For example, given a tensor with shape of [3,224,224], a pointer to the element at location [2,150,128] can be retrieved + * + * This function only works for numeric type tensors (No strings, etc). + * This function does not support sub-byte packed types (e.g., int4). + * This is a no-copy method whose returned pointer is valid until the passed in ::OrtValue is free'd. + * + * \param[in] value + * \param[in] location_values Pointer to an array of index values that specify an element's location relative to its shape + * \param[in] location_values_count Number of elements in location_values. Must match the number of elements in the tensor's shape. + * \param[out] out Set to a pointer to the element specified + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(TensorAt, _Inout_ OrtValue* value, const int64_t* location_values, size_t location_values_count, _Outptr_ void** out); + + /// @} + /// \name OrtEnv + /// @{ + + /** \brief Create an allocator and register it with the ::OrtEnv + * + * Enables sharing the allocator between multiple sessions that use the same env instance. + * Lifetime of the created allocator will be valid for the duration of the environment. + * Returns an error if an allocator with the same ::OrtMemoryInfo is already registered. + * + * See https://onnxruntime.ai/docs/get-started/with-c.html for details. + * + * \param[in] env ::OrtEnv instance + * \param[in] mem_info + * \param[in] arena_cfg Pass nullptr for defaults + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(CreateAndRegisterAllocator, _Inout_ OrtEnv* env, _In_ const OrtMemoryInfo* mem_info, + _In_ const OrtArenaCfg* arena_cfg); + + /** \brief Set language projection + * + * Set the language projection for collecting telemetry data when Env is created. + * + * The default is ORT_PROJECTION_C, which means it will classify the language not in the list to C also. + * + * \param[in] ort_env + * \param[in] projection + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(SetLanguageProjection, _In_ const OrtEnv* ort_env, _In_ OrtLanguageProjection projection); + + /// @} + /// \name OrtSession + /// @{ + + /** \brief Return the time that profiling was started + * + * \note The timer precision varies per platform. On Windows and MacOS, the precision will be ~100ns + * + * \param[in] session + * \param[out] out nanoseconds of profiling's start time + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(SessionGetProfilingStartTimeNs, _In_ const OrtSession* session, _Outptr_ uint64_t* out); + + /// @} + /// \name OrtThreadingOptions + /// @{ + + /** \brief Set global intra-op thread count + * + * This configures the global thread pool options to be used in the call to OrtApi::CreateEnvWithGlobalThreadPools + * + * \param[in] tp_options + * \param[in] intra_op_num_threads Number of threads, special values:
+ * 0 = Use default thread count
+ * 1 = The invoking thread will be used; no threads will be created in the thread pool. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(SetGlobalIntraOpNumThreads, _Inout_ OrtThreadingOptions* tp_options, int intra_op_num_threads); + + /** \brief Set global inter-op thread count + * + * This configures the global thread pool options to be used in the call to OrtApi::CreateEnvWithGlobalThreadPools + * + * \param[in] tp_options + * \param[in] inter_op_num_threads Number of threads, special values:
+ * 0 = Use default thread count
+ * 1 = The invoking thread will be used; no threads will be created in the thread pool. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(SetGlobalInterOpNumThreads, _Inout_ OrtThreadingOptions* tp_options, int inter_op_num_threads); + + /** \brief Set global spin control options + * + * This will configure the global thread pool options to be used in the call to OrtApi::CreateEnvWithGlobalThreadPools. + * Allow spinning of thread pools when their queues are empty. This will set the value for both + * inter_op and intra_op threadpools. + * + * \param[in] tp_options + * \param[in] allow_spinning Valid values are 0 or 1.
+ * 0 = It won't spin (recommended if CPU usage is high)
+ * 1 = Threadpool will spin to wait for queue to become non-empty + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(SetGlobalSpinControl, _Inout_ OrtThreadingOptions* tp_options, int allow_spinning); + + /// @} + /// \name OrtSessionOptions + /// @{ + + /** \brief Add a pre-allocated initializer to a session + * + * If a model contains an initializer with a name that is same as the name passed to this call, + * ORT will use this initializer instance instead of deserializing one from the model file. This + * is useful when you want to share the same initializer across sessions. + * + * \param[in] options + * \param[in] name Null terminated string of the initializer name + * \param[in] val ::OrtValue containing the initializer. Its lifetime and the underlying initializer buffer must be + * managed by the user (created using the OrtApi::CreateTensorWithDataAsOrtValue) and it must outlive the session object + * to which it is added. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(AddInitializer, _Inout_ OrtSessionOptions* options, _In_z_ const char* name, + _In_ const OrtValue* val); + + /// @} + /// \name OrtEnv + /// @{ + + /** + * Create a custom environment with global threadpools and logger that will be shared across sessions. + * Use this in conjunction with OrtApi::DisablePerSessionThreads or else the session will use + * its own thread pools. + * + * \param[in] logging_function A pointer to a logging function. + * \param[in] logger_param A pointer to arbitrary data passed as the ::OrtLoggingFunction `param` parameter to + * `logging_function`. + * \param[in] log_severity_level The log severity level. + * \param[in] logid The log identifier. + * \param[in] tp_options + * \param[out] out Newly created OrtEnv. Must be freed with OrtApi::ReleaseEnv + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(CreateEnvWithCustomLoggerAndGlobalThreadPools, OrtLoggingFunction logging_function, _In_opt_ void* logger_param, OrtLoggingLevel log_severity_level, + _In_ const char* logid, _In_ const struct OrtThreadingOptions* tp_options, _Outptr_ OrtEnv** out); + + /// @} + /// \name OrtSessionOptions + /// @{ + + /** \brief Append CUDA provider to session options + * + * If CUDA is not available (due to a non CUDA enabled build, or if CUDA is not installed on the system), this function will return failure. + * + * \param[in] options + * \param[in] cuda_options + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(SessionOptionsAppendExecutionProvider_CUDA, + _In_ OrtSessionOptions* options, _In_ const OrtCUDAProviderOptions* cuda_options); + + /** \brief Append ROCM execution provider to the session options + * + * If ROCM is not available (due to a non ROCM enabled build, or if ROCM is not installed on the system), this function will return failure. + * + * \param[in] options + * \param[in] rocm_options + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(SessionOptionsAppendExecutionProvider_ROCM, + _In_ OrtSessionOptions* options, _In_ const OrtROCMProviderOptions* rocm_options); + + /** \brief Append OpenVINO execution provider to the session options + * + * If OpenVINO is not available (due to a non OpenVINO enabled build, or if OpenVINO is not installed on the system), this function will fail. + * + * \param[in] options + * \param[in] provider_options + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(SessionOptionsAppendExecutionProvider_OpenVINO, + _In_ OrtSessionOptions* options, _In_ const OrtOpenVINOProviderOptions* provider_options); + + /// @} + /// \name OrtThreadingOptions + /// @{ + + /** \brief Set threading flush-to-zero and denormal-as-zero + * + * Sets global thread pool options to be used in the call to OrtApi::CreateEnvWithGlobalThreadPools. + * Flush-to-zero and denormal-as-zero are applied to threads in both intra and inter global thread pool. + * \note This option is not needed if the models used have no denormals. Having no denormals is recommended as this option may hurt model accuracy. + * + * \param[in] tp_options + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(SetGlobalDenormalAsZero, _Inout_ OrtThreadingOptions* tp_options); + + /// @} + /// \name OrtArenaCfg + /// @{ + + /** \deprecated Use OrtApi::CreateArenaCfgV2 + * + * This will create the configuration of an arena that can eventually be used to define an arena based allocator's behavior + * + * \param[in] max_mem Use 0 to allow ORT to choose the default + * \param[in] arena_extend_strategy Use -1 to allow ORT to choose the default, 0 = kNextPowerOfTwo, 1 = kSameAsRequested + * \param[in] initial_chunk_size_bytes Use -1 to allow ORT to choose the default + * \param[in] max_dead_bytes_per_chunk Use -1 to allow ORT to choose the default + * \param[in] out A pointer to an OrtArenaCfg instance + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(CreateArenaCfg, _In_ size_t max_mem, int arena_extend_strategy, int initial_chunk_size_bytes, + int max_dead_bytes_per_chunk, _Outptr_ OrtArenaCfg** out); + + ORT_CLASS_RELEASE(ArenaCfg); + + /// @} + /// \name OrtModelMetadata + /// @{ + + /** + * Use this to obtain the description of the graph present in the model + * (doc_string field of the GraphProto message within the ModelProto message). + * If it doesn't exist, an empty string will be returned. + * + * \param[in] model_metadata An instance of ::OrtModelMetadata + * \param[in] allocator Allocator used to allocate the string that will be returned back + * \param[out] value Set to a null terminated string allocated using `allocator`. The caller is responsible for freeing it using `allocator` + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(ModelMetadataGetGraphDescription, _In_ const OrtModelMetadata* model_metadata, + _Inout_ OrtAllocator* allocator, _Outptr_ char** value); + + /// @} + /// \name OrtSessionOptions + /// @{ + + /** \brief Append TensorRT provider to session options + * + * If TensorRT is not available (due to a non TensorRT enabled build, or if TensorRT is not installed on the system), this function will return failure. + * + * \param[in] options + * \param[in] tensorrt_options + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(SessionOptionsAppendExecutionProvider_TensorRT, + _In_ OrtSessionOptions* options, _In_ const OrtTensorRTProviderOptions* tensorrt_options); + + /// @} + /// \name Misc + /// @{ + + /** \brief Set current GPU device ID + * + * Set the current device id of the GPU execution provider (CUDA/tensorrt/rocm). The device id should be less + * than the total number of devices available. This is only useful when multiple-GPUs are installed and it is + * required to restrict execution to a single GPU. + * + * \param[in] device_id + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(SetCurrentGpuDeviceId, _In_ int device_id); + + /** \brief Get current GPU device ID + * + * Get the current device id of the GPU execution provider (CUDA/tensorrt/rocm). + * + * \see OrtApi::SetCurrentGpuDeviceId + * + * \param[out] device_id + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(GetCurrentGpuDeviceId, _In_ int* device_id); + + /// @} + /// \name OrtKernelInfo + /// Custom operator APIs. + /// @{ + + /** \brief Fetch an array of int64_t values stored as an attribute in the graph node + * + * + * If `out` is nullptr, the value of `size` is set to the true size of the attribute + * array's size, and a success status is returned. + * + * If the `size` parameter is greater than or equal to the actual attribute array's size, + * the value of `size` is set to the true size of the attribute array's size, + * the provided memory is filled with the attribute's contents, + * and a success status is returned. + * + * If the `size` parameter is less than the actual attribute array's size and `out` + * is not nullptr, the value of `size` is set to the true size of the attribute array's size + * and a failure status is returned.) + * + * \param[in] info instance + * \param[in] name name of the attribute to be parsed + * \param[out] out pointer to memory where the attribute's contents are to be stored + * \param[in, out] size actual size of attribute array + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(KernelInfoGetAttributeArray_float, _In_ const OrtKernelInfo* info, _In_ const char* name, + _Out_ float* out, _Inout_ size_t* size); + + /** \brief Fetch an array of int64_t values stored as an attribute in the graph node + * + * If `out` is nullptr, the value of `size` is set to the true size of the attribute + * array's size, and a success status is returned. + * + * If the `size` parameter is greater than or equal to the actual attribute array's size, + * the value of `size` is set to the true size of the attribute array's size, + * the provided memory is filled with the attribute's contents, + * and a success status is returned. + * + * If the `size` parameter is less than the actual attribute array's size and `out` + * is not nullptr, the value of `size` is set to the true size of the attribute array's size + * and a failure status is returned.) + * + * \param[in] info instance + * \param[in] name name of the attribute to be parsed + * \param[out] out pointer to memory where the attribute's contents are to be stored + * \param[in, out] size actual size of attribute array + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(KernelInfoGetAttributeArray_int64, _In_ const OrtKernelInfo* info, _In_ const char* name, + _Out_ int64_t* out, _Inout_ size_t* size); + + /// @} + /// \name OrtArenaCfg + /// @{ + + /** \brief Create an ::OrtArenaCfg + * + * Create the configuration of an arena that can eventually be used to define an arena based allocator's behavior. + * + * Supported keys are (See https://onnxruntime.ai/docs/get-started/with-c.html for details on what the + * following parameters mean and how to choose these values.): + * "max_mem": Maximum memory that can be allocated by the arena based allocator. + * Use 0 for ORT to pick the best value. Default is 0. + * "arena_extend_strategy": 0 = kNextPowerOfTwo, 1 = kSameAsRequested. + * Use -1 to allow ORT to choose the default. + * "initial_chunk_size_bytes": (Possible) Size of the first allocation in the arena. + * Only relevant if arena strategy is `kNextPowerOfTwo`. Use -1 to allow ORT to choose the default. + * Ultimately, the first allocation size is determined by the allocation memory request. + * "max_dead_bytes_per_chunk": Threshold of unused memory in an allocated chunk of arena memory after + * crossing which the current chunk is chunked into 2. + * "initial_growth_chunk_size_bytes": (Possible) Size of the second allocation in the arena. + * Only relevant if arena strategy is `kNextPowerOfTwo`. Use -1 to allow ORT to choose the default. + * "max_power_of_two_extend_bytes": The maximum extend size if arena strategy is `kNextPowerOfTwo`. + * It is not an allocation limit, it is only a limit for extension when requested byte is less than the limit. + * When requested bytes is more than the limit, allocator will still return as requested. + * Use -1 to allow ORT to choose the default 1GB for max_power_of_two_extend_bytes. + * Ultimately, the allocation size is determined by the allocation memory request. + * Further allocation sizes are governed by the arena extend strategy. + * + * \param[in] arena_config_keys Keys to configure the arena + * \param[in] arena_config_values Values to configure the arena + * \param[in] num_keys Number of keys in `arena_config_keys` and `arena_config_values` + * \param[out] out Newly created ::OrtArenaCfg. Must be freed with OrtApi::ReleaseArenaCfg + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(CreateArenaCfgV2, _In_reads_(num_keys) const char* const* arena_config_keys, + _In_reads_(num_keys) const size_t* arena_config_values, _In_ size_t num_keys, + _Outptr_ OrtArenaCfg** out); + + /// @} + /// \name OrtRunOptions + /// @{ + + /** \brief Set a single run configuration entry as a pair of strings + * + * If a configuration with same key exists, this will overwrite the configuration with the given config_value + * + * The config_key and the format of config_value are defined in onnxruntime_run_options_config_keys.h + * + * \param[in] options + * \param[in] config_key A null terminated string representation of the config key + * \param[in] config_value A null terminated string representation of the config value + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(AddRunConfigEntry, _Inout_ OrtRunOptions* options, + _In_z_ const char* config_key, _In_z_ const char* config_value); + + /// @} + /// \name OrtPrepackedWeightsContainer + /// @{ + + /** \brief Create an ::OrtPrepackedWeightsContainer + * + * This container will hold pre-packed buffers of shared initializers for sharing between sessions + * (i.e.) if there are shared initializers that can be shared between sessions, the pre-packed buffers + * of these (if any) may possibly be shared to provide memory footprint savings. Pass this container + * to sessions that you would like to share pre-packed buffers of shared initializers at session + * creation time. + * + * \param[out] out Newly created ::OrtPrepackedWeightsContainer. Must be freed with OrtApi::ReleasePrepackedWeightsContainer + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(CreatePrepackedWeightsContainer, _Outptr_ OrtPrepackedWeightsContainer** out); + + /** \brief Release OrtPrepackedWeightsContainer instance + * + * \note instance must not be released until the sessions using it are released + */ + ORT_CLASS_RELEASE(PrepackedWeightsContainer); + + /// @} + /// \name OrtSession + /// @{ + + /** \brief Create session with prepacked weights container + * + * Same functionality offered by OrtApi::CreateSession except that a container that contains + * pre-packed weights' buffers is written into/read from by the created session. + * This is useful when used in conjunction with OrtApi::AddInitializer which injects + * shared initializer info into sessions. Wherever possible, the pre-packed versions of these + * shared initializers are cached in this container so that multiple sessions can just re-use + * these instead of duplicating these in memory. + * + * \param[in] env OrtEnv instance instance + * \param[in] model_path Null terminated string of the path (wchar on Windows, char otherwise) + * \param[in] options + * \param[in] prepacked_weights_container + * \param[out] out Newly created ::OrtSession. Must be freed with OrtApi::ReleaseSession + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(CreateSessionWithPrepackedWeightsContainer, _In_ const OrtEnv* env, _In_ const ORTCHAR_T* model_path, + _In_ const OrtSessionOptions* options, + _Inout_ OrtPrepackedWeightsContainer* prepacked_weights_container, + _Outptr_ OrtSession** out); + + /** \brief Create session from memory with prepacked weights container + * + * Same functionality offered by OrtApi::CreateSessionFromArray except that a container that contains + * pre-packed weights' buffers is written into/read from by the created session. + * This is useful when used in conjunction with OrtApi::AddInitializer which injects + * shared initializer info into sessions. Wherever possible, the pre-packed versions of these + * shared initializers are cached in this container so that multiple sessions can just re-use + * these instead of duplicating these in memory. + * + * \param[in] env + * \param[in] model_data Array of bytes holding the model + * \param[in] model_data_length Number of bytes in `model_data_model` + * \param[in] options + * \param[in] prepacked_weights_container + * \param[out] out Newly created ::OrtSession. Must be freed with OrtApi::ReleaseSession + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(CreateSessionFromArrayWithPrepackedWeightsContainer, _In_ const OrtEnv* env, + _In_ const void* model_data, size_t model_data_length, + _In_ const OrtSessionOptions* options, + _Inout_ OrtPrepackedWeightsContainer* prepacked_weights_container, + _Outptr_ OrtSession** out); + + /// @} + /// \name OrtSessionOptions + /// @{ + + /** \brief Append TensorRT execution provider to the session options + * + * If TensorRT is not available (due to a non TensorRT enabled build), this function will return failure. + * + * This is slightly different from OrtApi::SessionOptionsAppendExecutionProvider_TensorRT, it takes an + * ::OrtTensorRTProviderOptions which is publicly defined. This takes an opaque ::OrtTensorRTProviderOptionsV2 + * which must be created with OrtApi::CreateTensorRTProviderOptions. + * + * For OrtApi::SessionOptionsAppendExecutionProvider_TensorRT, the user needs to instantiate ::OrtTensorRTProviderOptions + * as well as allocate/release buffers for some members of ::OrtTensorRTProviderOptions. + * Here, OrtApi::CreateTensorRTProviderOptions and Ortapi::ReleaseTensorRTProviderOptions will do the memory management for you. + * + * \param[in] options + * \param[in] tensorrt_options + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(SessionOptionsAppendExecutionProvider_TensorRT_V2, + _In_ OrtSessionOptions* options, _In_ const OrtTensorRTProviderOptionsV2* tensorrt_options); + + /// @} + /// \name OrtTensorRTProviderOptionsV2 + /// @{ + + /** \brief Create an OrtTensorRTProviderOptionsV2 + * + * \param[out] out Newly created ::OrtTensorRTProviderOptionsV2. Must be released with OrtApi::ReleaseTensorRTProviderOptions + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(CreateTensorRTProviderOptions, _Outptr_ OrtTensorRTProviderOptionsV2** out); + + /** \brief Set options in a TensorRT Execution Provider. + * + * Please refer to https://onnxruntime.ai/docs/execution-providers/TensorRT-ExecutionProvider.html#cc + * to know the available keys and values. Key should be in null terminated string format of the member of ::OrtTensorRTProviderOptionsV2 + * and value should be its related range. Recreates the options and only sets the supplied values. + * + * For example, key="trt_max_workspace_size" and value="2147483648" + * + * \param[in] tensorrt_options + * \param[in] provider_options_keys Array of UTF-8 null-terminated string for provider options keys + * \param[in] provider_options_values Array of UTF-8 null-terminated string for provider options values + * \param[in] num_keys Number of elements in the `provider_option_keys` and `provider_options_values` arrays + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(UpdateTensorRTProviderOptions, _Inout_ OrtTensorRTProviderOptionsV2* tensorrt_options, + _In_reads_(num_keys) const char* const* provider_options_keys, + _In_reads_(num_keys) const char* const* provider_options_values, + _In_ size_t num_keys); + + /** \brief Get serialized TensorRT provider options string. + * + * For example, "trt_max_workspace_size=2147483648;trt_max_partition_iterations=10;trt_int8_enable=1;......" + * + * \param tensorrt_options - OrtTensorRTProviderOptionsV2 instance + * \param allocator - a ptr to an instance of OrtAllocator obtained with OrtApi::CreateAllocator or OrtApi::GetAllocatorWithDefaultOptions + * the specified allocator will be used to allocate continuous buffers for output strings and lengths. + * \param ptr - is a UTF-8 null terminated string allocated using 'allocator'. The caller is responsible for using the same allocator to free it. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(GetTensorRTProviderOptionsAsString, _In_ const OrtTensorRTProviderOptionsV2* tensorrt_options, _Inout_ OrtAllocator* allocator, _Outptr_ char** ptr); + + /** \brief Release an ::OrtTensorRTProviderOptionsV2 + * + * \note This is an exception in the naming convention of other Release* functions, as the name of the method does not have the V2 suffix, but the type does + */ + void(ORT_API_CALL* ReleaseTensorRTProviderOptions)(_Frees_ptr_opt_ OrtTensorRTProviderOptionsV2* input); + + /// @} + /// \name OrtSessionOptions + /// @{ + + /** \brief Enable custom operators + * + * See onnxruntime-extensions: https://github.com/microsoft/onnxruntime-extensions.git + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(EnableOrtCustomOps, _Inout_ OrtSessionOptions* options); + + /// @} + /// \name OrtAllocator + /// @{ + + /** \brief Register a custom allocator + * + * Enables sharing between multiple sessions that use the same env instance. + * Returns an error if an allocator with the same ::OrtMemoryInfo is already registered. + * + * The behavior of this is exactly the same as OrtApi::CreateAndRegisterAllocator except + * instead of ORT creating an allocator based on provided info, in this case + * ORT uses the user-provided custom allocator. + * See https://onnxruntime.ai/docs/get-started/with-c.html for details. + * + * \param[in] env + * \param[in] allocator User provided allocator + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(RegisterAllocator, _Inout_ OrtEnv* env, _In_ OrtAllocator* allocator); + + /** \brief Unregister a custom allocator + * + * It is an error if you provide an ::OrtMemoryInfo not corresponding to any + * registered allocators for sharing. + * + * \param[in] env + * \param[in] mem_info + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(UnregisterAllocator, _Inout_ OrtEnv* env, + _In_ const OrtMemoryInfo* mem_info); + + /// @} + /// \name OrtValue + /// @{ + + /** \brief Sets *out to 1 iff an ::OrtValue is a SparseTensor, and 0 otherwise + * + * \param[in] value existing ::OrtValue + * \param[out] out unless an error occurs, contains 1 iff the value contains an instance + * of sparse tensor or 0 otherwise. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(IsSparseTensor, _In_ const OrtValue* value, _Out_ int* out); + + /** \brief Create an ::OrtValue with a sparse tensor that is empty. + * + * Use FillSparseTensor() functions to populate sparse tensor with non-zero values and + * format specific indices data. + * Use ReleaseValue to destroy the sparse tensor, this will also release the buffer inside the output value + * if any was allocated. + * \param[in,out] allocator allocator to use when performing an allocation. Allocation will be performed + * by FillSparseTensor() APIs. The lifespan of the allocator instance must eclipse the lifespan + * this sparse tensor instance as the same allocator will be used to free memory. + * \param[in] dense_shape shape of the original dense tensor + * \param[in] dense_shape_len number of shape dimensions being passed + * \param[in] type must be one of TENSOR_ELEMENT_DATA_TYPE_xxxx + * \param[out] out Should be freed by calling ReleaseValue + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(CreateSparseTensorAsOrtValue, _Inout_ OrtAllocator* allocator, _In_ const int64_t* dense_shape, + size_t dense_shape_len, ONNXTensorElementDataType type, _Outptr_ OrtValue** out); + + /** + * This fills populates an empty tensor that was created using OrtApi::CreateSparseTensorAsOrtValue. + * This will allocate required memory and copy the supplied NNZ values and COO indices into that memory allocation. + * Memory allocation is performed using the allocator that was specified with OrtApi::CreateSparseTensorAsOrtValue. + * + * \param[in,out] ort_value ::OrtValue to populate with data + * \param[in] data_mem_info serves to identify the location of the data to be copied. If the allocator specified + * at the creation time has memory info that is not the same as mem_info argument to this function a X-device copy will be performed. + * String data is assumed to be on CPU and will only be copied into a CPU allocated buffer. + * \param[in] values_shape pointer to values shape array + * \param[in] values_shape_len length of the values_shape + * \param[in] values pointer to an array of values. For strings, pass const char**. + * \param[in] indices_data pointer to a location of COO indices + * \param[in] indices_num number of COO indices + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(FillSparseTensorCoo, _Inout_ OrtValue* ort_value, _In_ const OrtMemoryInfo* data_mem_info, + _In_ const int64_t* values_shape, size_t values_shape_len, _In_ const void* values, + _In_ const int64_t* indices_data, size_t indices_num); + + /** + * This fills populates an empty tensor that was created using OrtApi::CreateSparseTensorAsOrtValue. + * This will allocate required memory and copy the supplied NNZ values and CSR indices into that memory allocation. + * Memory allocation is performed using the allocator that was specified with OrtApi::CreateSparseTensorAsOrtValue. + * + * \param[in,out] ort_value ::OrtValue to populate with data + * \param[in] data_mem_info serves to identify the location of the data to be copied. If the allocator specified + * at the creation time has memory info that is not the same as mem_info argument to this function a X-device copy will be performed. + * String data is assumed to be on CPU and will only be copied into a CPU allocated buffer. + * \param[in] values_shape pointer to values shape array + * \param[in] values_shape_len length of the values_shape + * \param[in] values - pointer to an array of values. For strings, pass const char**. + * \param[in] inner_indices_data pointer to a location of CSR inner indices + * \param[in] inner_indices_num number of CSR inner indices + * \param[in] outer_indices_data pointer to a location of CSR outer indices + * \param[in] outer_indices_num number of CSR outer indices + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(FillSparseTensorCsr, _Inout_ OrtValue* ort_value, _In_ const OrtMemoryInfo* data_mem_info, + _In_ const int64_t* values_shape, size_t values_shape_len, _In_ const void* values, + _In_ const int64_t* inner_indices_data, size_t inner_indices_num, + _In_ const int64_t* outer_indices_data, size_t outer_indices_num); + + /** + * This fills populates an empty tensor that was created using OrtApi::CreateSparseTensorAsOrtValue. + * This will allocate required memory and copy the supplied NNZ values and BlockSparse indices into that memory allocation. + * Memory allocation is performed using the allocator that was specified with OrtApi::CreateSparseTensorAsOrtValue. + * + * \param[in,out] ort_value ::OrtValue to populate with data + * \param[in] data_mem_info serves to identify the location of the data to be copied. If the allocator specified + * at the creation time has memory info that is not the same as mem_info argument to this function a X-device copy will be performed. + * String data is assumed to be on CPU and will only be copied into a CPU allocated buffer. + * \param[in] values_shape + * \param[in] values_shape_len + * \param[in] values structure with values information + * \param[in] indices_shape_data pointer to a location of indices shape + * \param[in] indices_shape_len length of the block sparse indices shape + * \param[in] indices_data pointer to a location of indices data. Shape will determine the length of the indices data. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(FillSparseTensorBlockSparse, _Inout_ OrtValue* ort_value, _In_ const OrtMemoryInfo* data_mem_info, + _In_ const int64_t* values_shape, size_t values_shape_len, _In_ const void* values, + _In_ const int64_t* indices_shape_data, size_t indices_shape_len, + _In_ const int32_t* indices_data); + + /** + * Create an ::OrtValue with a sparse tensor. This is the first step. + * Next, use UseIndices() functions to supply sparse tensor with + * format specific indices data and set its sparse format to a specific enum value. + * This will not perform memory allocations. It will + * use supplied user buffer which should outlive the created sparse tensor. + * Use OrtApi::ReleaseValue to destroy the sparse tensor. It would not release the supplied values buffer. + * This function can not be used to map strings from the user allocated memory. Strings must always be copied + * and have UTF-8 encoding. Therefore, use OrtApi::CreateSparseTensorAsOrtValue above and then fill it with data + * using appropriate Make*() function. + * + * \param[in] info memory info where sparse values reside. + * \param[in,out] p_data pointer to a user allocated buffer with values. To create a full sparse tensor with no non-zero + * values, pass nullptr + * \param[in] dense_shape shape of the original dense tensor + * \param[in] dense_shape_len number of shape dimensions being passed + * \param[in] values_shape shape of the values data. To create a fully sparse tensor with no non-zero values, + * pass {0} shape. + * \param[in] values_shape_len number of values shape dimensions + * \param[in] type must be one of TENSOR_ELEMENT_DATA_TYPE_xxxx + * \param[out] out Should be freed by calling ReleaseValue + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(CreateSparseTensorWithValuesAsOrtValue, _In_ const OrtMemoryInfo* info, _Inout_ void* p_data, + _In_ const int64_t* dense_shape, size_t dense_shape_len, + _In_ const int64_t* values_shape, size_t values_shape_len, + ONNXTensorElementDataType type, _Outptr_ OrtValue** out); + + /** + * This assigns Coo format indices to the SparseTensor that was created by + * OrtApi::CreateSparseTensorWithValuesAsOrtValue above. It also sets OrtSparseFormat to + * ORT_SPARSE_COO. This will not allocate any additional memory for data. The life span of + * indices_data buffer should eclipse the life span of this ::OrtValue. + * + * \param[in,out] ort_value ::OrtValue instance constructed with OrtApi::CreateSparseTensorWithValuesAsOrtValue + * \param[in,out] indices_data pointer to a user pre-allocated buffer or nullptr for fully sparse tensors. + * \param[in] indices_num number of COO indices. Should either be 0 for fully sparse tensors, be equal + * to the number of nnz values specified to OrtApi::CreateSparseTensorWithValuesAsOrtValue for 1-D {nnz} indices or + * be twice as number of nnz values for a 2-D indices {nnz, 2} + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(UseCooIndices, _Inout_ OrtValue* ort_value, _Inout_ int64_t* indices_data, size_t indices_num); + + /** + * The assigns CSR format indices to the SparseTensor that was created by + * OrtApi::CreateSparseTensorWithValuesAsOrtValue above. It also sets OrtSparseFormat to + * ORT_SPARSE_CSRC. This will not allocate any additional memory for data. The life spans of + * inner_data and outer_data buffers should eclipse the life span of this ::OrtValue. + * + * \param[in,out] ort_value ::OrtValue instance constructed with OrtApi::CreateSparseTensorWithValuesAsOrtValue + * \param[in,out] inner_data pointer to a user pre-allocated buffer or nullptr for fully sparse tensors. + * \param[in] inner_num number of inner CSR indices. Should either be 0 for fully sparse tensors or be equal + * to the number of nnz values specified to OrtApi::CreateSparseTensorWithValuesAsOrtValue. + * \param[in,out] outer_data pointer to user pre-allocated buffer or nullptr for fully sparse tensors. + * \param[in] outer_num number of CSR outer indices. Should either be 0 for fully sparse tensors or + * equal to rows + 1 of the dense shape. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(UseCsrIndices, _Inout_ OrtValue* ort_value, _Inout_ int64_t* inner_data, size_t inner_num, + _Inout_ int64_t* outer_data, size_t outer_num); + + /** + * The assigns BlockSparse format indices to the SparseTensor that was created by + * OrtApi::CreateSparseTensorWithValuesAsOrtValue above. It also sets OrtSparseFormat to + * ORT_SPARSE_BLOCK_SPARSE. This will not allocate any additional memory for data. The life span of + * indices_data buffer must eclipse the lifespan of this ::OrtValue. + * + * \param[in,out] ort_value OrtValue instance constructed with OrtApi::CreateSparseTensorWithValuesAsOrtValue + * \param[in] indices_shape pointer to indices shape. Use {0} for fully sparse tensors + * \param[in] indices_shape_len length of the indices shape + * \param[in,out] indices_data pointer to user pre-allocated buffer or nullptr for fully sparse tensors. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(UseBlockSparseIndices, _Inout_ OrtValue* ort_value, const int64_t* indices_shape, size_t indices_shape_len, _Inout_ int32_t* indices_data); + + /** \brief Returns sparse tensor format enum iff a given ort value contains an instance of sparse tensor. + * + * \param[in] ort_value ::OrtValue that contains an instance of sparse tensor + * \param[out] out pointer to out parameter + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(GetSparseTensorFormat, _In_ const OrtValue* ort_value, _Out_ enum OrtSparseFormat* out); + + /** \brief Returns data type and shape of sparse tensor values (nnz) iff ::OrtValue contains a SparseTensor. + * + * \param[in] ort_value An ::OrtValue that contains a fully constructed sparse tensor + * \param[out] out Must be freed by OrtApi::ReleaseTensorTypeAndShapeInfo + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(GetSparseTensorValuesTypeAndShape, _In_ const OrtValue* ort_value, _Outptr_ OrtTensorTypeAndShapeInfo** out); + + /** \brief Returns numeric data for sparse tensor values (nnz). For string values use GetStringTensor*(). + * + * \param[in] ort_value an instance of ::OrtValue containing sparse tensor + * \param[out] out returns a pointer to values data. Do not attempt to free this ptr. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(GetSparseTensorValues, _In_ const OrtValue* ort_value, _Outptr_ const void** out); + + /** \brief Returns data type, shape for the type of indices specified by indices_format. + * + * \param[in] ort_value ::OrtValue containing sparse tensor. + * \param[in] indices_format One of the indices formats. It is an error to request a format that the sparse + * tensor does not contain. + * \param[out] out an instance of ::OrtTensorTypeAndShapeInfo. Must be freed by OrtApi::ReleaseTensorTypeAndShapeInfo + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(GetSparseTensorIndicesTypeShape, _In_ const OrtValue* ort_value, enum OrtSparseIndicesFormat indices_format, _Outptr_ OrtTensorTypeAndShapeInfo** out); + + /** \brief Returns indices data for the type of the indices specified by indices_format + * + * \param[in] ort_value ::OrtValue containing sparse tensor. + * \param[in] indices_format One of the indices formats. It is an error to request a format that the sparse tensor does not contain. + * \param[out] num_indices Pointer to where the number of indices entries is returned + * \param[out] indices Returned pointer to the indices data. Do not free the returned pointer as it refers to internal data owned by the ::OrtValue + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(GetSparseTensorIndices, _In_ const OrtValue* ort_value, enum OrtSparseIndicesFormat indices_format, _Out_ size_t* num_indices, _Outptr_ const void** indices); + /// @} + /// \name OrtSessionOptions + /// @{ + + /** + * \brief Sets out to 1 iff an optional type OrtValue has an element, 0 otherwise (OrtValue is None) + * Use this API to find if the optional type OrtValue is None or not. + * If the optional type OrtValue is not None, use the OrtValue just like any other OrtValue. + * For example, if you get an OrtValue that corresponds to Optional(tensor) and + * if HasValue() returns true, use it as tensor and so on. + + * \param[in] value Input OrtValue. + * \param[out] out indicating if the input OrtValue contains data (1) or if it is a None (0) + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(HasValue, _In_ const OrtValue* value, _Out_ int* out); + + /// @} + /// \name OrtKernelContext + /// Custom operator APIs. + /// @{ + + /** \brief Used for custom operators, gets the GPU compute stream to use to launch the custom a GPU kernel + * \see ::OrtCustomOp + * \param[in] context OrtKernelContext instance + * \param[out] out Returns pointer to a GPU compute stream that can be used to launch the custom GPU kernel. + * If retrieving the GPU compute stream is not relevant (GPU not enabled in the build, kernel partitioned to + * some other EP), then a nullptr is returned as the output param. + * Do not free or mutate the returned pointer as it refers to internal data owned by the underlying session. + * Only use it for custom kernel launching. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(KernelContext_GetGPUComputeStream, _In_ const OrtKernelContext* context, _Outptr_ void** out); + + /// @} + /// \name GetTensorMemoryInfo + /// @{ + /** \brief Returns a pointer to the ::OrtMemoryInfo of a Tensor + * \param[in] value ::OrtValue containing tensor. + * \param[out] mem_info ::OrtMemoryInfo of the tensor. Do NOT free the returned pointer. It is valid for the lifetime of the ::OrtValue + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(GetTensorMemoryInfo, _In_ const OrtValue* value, _Out_ const OrtMemoryInfo** mem_info); + + /// @} + /// \name GetExecutionProviderApi + /// @{ + /** \brief Get a pointer to the requested version of the Execution Provider specific + * API extensions to the OrtApi + * \param[in] provider_name The name of the execution provider name. Currently only the following + * values are supported: "DML". + * \param[in] version Must be ::ORT_API_VERSION. + * \param[out] provider_api A void pointer containing a reference to the execution provider versioned api structure. + * For example, the provider_api pointer can be cast to the OrtDmlApi* when the provider_name is "DML". + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(GetExecutionProviderApi, _In_ const char* provider_name, _In_ uint32_t version, _Outptr_ const void** provider_api); + + /// @} + + /// \name SessionOptions + /// @{ + /** \brief Set custom thread creation function + * + * \param[in] options Session options + * \param[in] ort_custom_create_thread_fn Custom thread creation function + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(SessionOptionsSetCustomCreateThreadFn, _Inout_ OrtSessionOptions* options, _In_ OrtCustomCreateThreadFn ort_custom_create_thread_fn); + + /** \brief Set creation options for custom thread + * + * \param[in] options Session options + * \param[in] ort_custom_thread_creation_options Custom thread creation options (can be nullptr) + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(SessionOptionsSetCustomThreadCreationOptions, _Inout_ OrtSessionOptions* options, _In_ void* ort_custom_thread_creation_options); + + /** \brief Set custom thread join function + * + * \param[in] options Session options + * \param[in] ort_custom_join_thread_fn Custom join thread function, must not be nullptr when ort_custom_create_thread_fn is set + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(SessionOptionsSetCustomJoinThreadFn, _Inout_ OrtSessionOptions* options, _In_ OrtCustomJoinThreadFn ort_custom_join_thread_fn); + /// @} + + /// \name OrtThreadingOptions + /// @{ + /** \brief Set custom thread creation function for global thread pools + * + * \param[inout] tp_options + * \param[in] ort_custom_create_thread_fn Custom thread creation function + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(SetGlobalCustomCreateThreadFn, _Inout_ OrtThreadingOptions* tp_options, _In_ OrtCustomCreateThreadFn ort_custom_create_thread_fn); + + /** \brief Set custom thread creation options for global thread pools + * + * \param[inout] tp_options + * \param[in] ort_custom_thread_creation_options Custom thread creation options (can be nullptr) + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(SetGlobalCustomThreadCreationOptions, _Inout_ OrtThreadingOptions* tp_options, _In_ void* ort_custom_thread_creation_options); + + /** \brief Set custom thread join function for global thread pools + * + * \param[inout] tp_options + * \param[in] ort_custom_join_thread_fn Custom thread join function, must not be nullptr when global ort_custom_create_thread_fn is set + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(SetGlobalCustomJoinThreadFn, _Inout_ OrtThreadingOptions* tp_options, _In_ OrtCustomJoinThreadFn ort_custom_join_thread_fn); + /// @} + + /** \brief Synchronize bound inputs. The call may be necessary for some providers, such as cuda, + * in case the system that allocated bound memory operated on a different stream. However, the + * operation is provider specific and could be a no-op. + * + * \param[inout] binding_ptr + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(SynchronizeBoundInputs, _Inout_ OrtIoBinding* binding_ptr); + + /** \brief Synchronize bound outputs. The call may be necessary for some providers, such as cuda, + * in case the system that allocated bound memory operated on a different stream. However, the + * operation is provider specific and could be a no-op. + * + * \param[inout] binding_ptr + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(SynchronizeBoundOutputs, _Inout_ OrtIoBinding* binding_ptr); + + /// \name OrtSessionOptions + /// @{ + + /** \brief Append CUDA execution provider to the session options + * + * If CUDA is not available (due to a non CUDA enabled build), this function will return failure. + * + * This is slightly different from OrtApi::SessionOptionsAppendExecutionProvider_CUDA, it takes an + * ::OrtCUDAProviderOptions which is publicly defined. This takes an opaque ::OrtCUDAProviderOptionsV2 + * which must be created with OrtApi::CreateCUDAProviderOptions. + * + * For OrtApi::SessionOptionsAppendExecutionProvider_CUDA, the user needs to instantiate ::OrtCUDAProviderOptions + * as well as allocate/release buffers for some members of ::OrtCUDAProviderOptions. + * Here, OrtApi::CreateCUDAProviderOptions and Ortapi::ReleaseCUDAProviderOptions will do the memory management for you. + * + * \param[in] options + * \param[in] cuda_options + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.11. + */ + ORT_API2_STATUS(SessionOptionsAppendExecutionProvider_CUDA_V2, + _In_ OrtSessionOptions* options, _In_ const OrtCUDAProviderOptionsV2* cuda_options); + + /// @} + /// \name OrtCUDAProviderOptionsV2 + /// @{ + + /** \brief Create an OrtCUDAProviderOptionsV2 + * + * \param[out] out Newly created ::OrtCUDAProviderOptionsV2. Must be released with OrtApi::ReleaseCudaProviderOptions + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.11. + */ + ORT_API2_STATUS(CreateCUDAProviderOptions, _Outptr_ OrtCUDAProviderOptionsV2** out); + + /** \brief Set options in a CUDA Execution Provider. + * + * Please refer to https://onnxruntime.ai/docs/execution-providers/CUDA-ExecutionProvider.html#configuration-options + * to know the available keys and values. Key should be in null terminated string format of the member of ::OrtCUDAProviderOptionsV2 + * and value should be its related range. Recreates the options and only sets the supplied values. + * + * For example, key="device_id" and value="0" + * + * \param[in] cuda_options + * \param[in] provider_options_keys Array of UTF-8 null-terminated string for provider options keys + * \param[in] provider_options_values Array of UTF-8 null-terminated string for provider options values + * \param[in] num_keys Number of elements in the `provider_option_keys` and `provider_options_values` arrays + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.11. + */ + ORT_API2_STATUS(UpdateCUDAProviderOptions, _Inout_ OrtCUDAProviderOptionsV2* cuda_options, + _In_reads_(num_keys) const char* const* provider_options_keys, + _In_reads_(num_keys) const char* const* provider_options_values, + _In_ size_t num_keys); + + /** + * Get serialized CUDA provider options string. + * + * For example, "device_id=0;arena_extend_strategy=0;......" + * + * \param cuda_options - OrtCUDAProviderOptionsV2 instance + * \param allocator - a ptr to an instance of OrtAllocator obtained with CreateAllocator() or GetAllocatorWithDefaultOptions() + * the specified allocator will be used to allocate continuous buffers for output strings and lengths. + * \param ptr - is a UTF-8 null terminated string allocated using 'allocator'. The caller is responsible for using the same allocator to free it. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.11. + */ + ORT_API2_STATUS(GetCUDAProviderOptionsAsString, _In_ const OrtCUDAProviderOptionsV2* cuda_options, _Inout_ OrtAllocator* allocator, _Outptr_ char** ptr); + + /** \brief Release an ::OrtCUDAProviderOptionsV2 + * + * \note This is an exception in the naming convention of other Release* functions, as the name of the method does not have the V2 suffix, but the type does + * + * \since Version 1.11. + */ + void(ORT_API_CALL* ReleaseCUDAProviderOptions)(_Frees_ptr_opt_ OrtCUDAProviderOptionsV2* input); + + /// @} + + /** \brief Append MIGraphX provider to session options + * + * If MIGraphX is not available (due to a non MIGraphX enabled build, or if MIGraphX is not installed on the system), this function will return failure. + * + * \param[in] options + * \param[in] migraphx_options + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.11. + */ + ORT_API2_STATUS(SessionOptionsAppendExecutionProvider_MIGraphX, + _In_ OrtSessionOptions* options, _In_ const OrtMIGraphXProviderOptions* migraphx_options); + + /** \brief Replace initialized Tensors with external data with the data provided in initializers. + * + * The function will find the initialized TensorProtos with external data in the graph with the provided names and + * replace them with the provided tensors. The API verifies that the TensorProto being replaced + * has an external data reference and has the same name, dimensions and data type as its replacement. The replacement + * will occur before any of the optimizations take place. The data will be copied into the graph + * since TensorProto can't refer to the user provided buffers. + * + * Once the model has been loaded, the OrtValue(s) added to SessionOptions instance will be removed + * from the internal SessionOptions copy to save memory, the user provided buffers can then be deallocated + * and the SessionOptions instance that refers to them can be destroyed. + * + * \param[in] options + * \param[in] initializer_names Array of null terminated UTF-8 encoded strings of the initializers names. + * \param[in] initializers Array of ::OrtValue type + * \param[in] num_initializers Number of elements in the initializer_names and initializers + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.12. + */ + ORT_API2_STATUS(AddExternalInitializers, _In_ OrtSessionOptions* options, + _In_reads_(num_initializers) const char* const* initializer_names, + _In_reads_(num_initializers) const OrtValue* const* initializers, size_t num_initializers); + + /** \brief: Create attribute of onnxruntime operator + * + * \param[in] name Name of the attribute + * \param[in] data Data content of the attribute + * \param[in] len Number of bytes stored in data for ORT_OP_ATTR_STRING. + Number of elements if data represents an array (e.g., ORT_OP_ATTR_INTS). Otherwise, set to 1. + * \param[in] type Data type + * \param[out] op_attr Attribute that has been created, which must be released by OrtApi::ReleaseOpAttr + * + * \since Version 1.12. + */ + ORT_API2_STATUS(CreateOpAttr, + _In_ const char* name, + _In_ const void* data, + _In_ int len, + _In_ OrtOpAttrType type, + _Outptr_ OrtOpAttr** op_attr); + + /* \brief: Release op attribute + * + * \param[in] opAttr Attribute created by OrtApi::CreateOpAttr + * + * \since Version 1.12. + */ + ORT_CLASS_RELEASE(OpAttr); + + /** \brief: Create onnxruntime native operator + * + * \param[in] info Kernel info + * \param[in] op_name Operator name + * \param[in] domain Operator domain + * \param[in] version Operator opset version + * \param[in] type_constraint_names Name of the type constraints, such as "T" or "T1" + * \param[in] type_constraint_values Type of each constraints + * \param[in] type_constraint_count Number of constraints + * \param[in] attr_values Attributes used to initialize the operator + * \param[in] attr_count Number of the attributes + * \param[in] input_count Number of inputs + * \param[in] output_count Number of outputs + * \param[out] ort_op Operator that has been created + * + * \since Version 1.12. + */ + ORT_API2_STATUS(CreateOp, + _In_ const OrtKernelInfo* info, + _In_z_ const char* op_name, + _In_z_ const char* domain, + int version, + _In_reads_(type_constraint_count) const char** type_constraint_names, + _In_reads_(type_constraint_count) const ONNXTensorElementDataType* type_constraint_values, + int type_constraint_count, + _In_reads_(attr_count) const OrtOpAttr* const* attr_values, + int attr_count, + int input_count, + int output_count, + _Outptr_ OrtOp** ort_op); + + /** \brief: Invoke the operator created by OrtApi::CreateOp + * The inputs must follow the order as specified in onnx specification + * + * \param[in] context Kernel context + * \param[in] ort_op Operator that has been created + * \param[in] input_values Array of inputs + * \param[in] input_count Number of inputs + * \param[in] output_values Array of outputs + * \param[in] output_count Number of outputs + * + * \since Version 1.12. + */ + ORT_API2_STATUS(InvokeOp, + _In_ const OrtKernelContext* context, + _In_ const OrtOp* ort_op, + _In_ const OrtValue* const* input_values, + _In_ int input_count, + _Inout_ OrtValue* const* output_values, + _In_ int output_count); + + /* \brief: Release an onnxruntime operator + * + * \param[in] Op Operator created by OrtApi::CreateOp + * + * \since Version 1.12. + */ + ORT_CLASS_RELEASE(Op); + + /** \brief: Append execution provider to the session options. + * \param[in] options + * \param[in] provider_name - provider to add. + * \param[in] provider_options_keys - keys to configure the provider options + * \param[in] provider_options_values - values to configure the provider options + * \param[in] num_keys - number of keys passed in + * + * Currently supported provider names: + * QNNExecutionProvider (or QNN) + * OpenVINOExecutionProvider (or OpenVINO) + * XnnpackExecutionProvider (or XNNPACK) + * WebNNExecutionProvider (or WEBNN) + * WebGpuExecutionProvider (or WebGPU) + * AzureExecutionProvider (or AZURE) + * JsExecutionProvider (or JS) + * VitisAIExecutionProvider (or VitisAI) + * CoreMLExecutionProvider (or CoreML) + * + * Note: If an execution provider has a dedicated SessionOptionsAppendExecutionProvider_ function + * that should be used to add it. + * + * QNN supported keys: + * "backend_type": Type of QNN backend. Specifies a backend path that is the associated QNN backend library file + * name. E.g., given backend type "htp", on Windows, the backend path would be "QnnHtp.dll", and on other + * platforms, it would be "libQnnHtp.so". Mutually exclusive with "backend_path". + * Available options: + * -# "cpu" + * -# "gpu" + * -# "htp": Default. + * -# "saver" + * -# "ir" + * "backend_path": File path to QNN backend library. Mutually exclusive with "backend_type". + * "profiling_level": QNN profiling level. + * Available options: + * -# "off": Default. + * -# "basic" + * -# "detailed" + * "profiling_file_path": QNN profiling file path if ETW not enabled. + * "rpc_control_latency": QNN RPC control latency. + * "vtcm_mb": QNN VTCM size in MB. default to 0(not set). + * "htp_performance_mode": QNN performance mode. + * Available options: + * -# "burst" + * -# "balanced" + * -# "default": Default. + * -# "high_performance" + * -# "high_power_saver" + * -# "low_balanced" + * -# "extreme_power_saver" + * -# "low_power_saver" + * -# "power_saver" + * -# "sustained_high_performance" + * "dump_qnn_ir_dlc": Use the QnnIr backend library to write .dlc files for each subgraph dispatched to QNN. When + * enabled, inference results will be incorrect. Use only for debugging. + * -# "0": Default: disabled + * -# "1": enabled + * "dump_qnn_ir_dlc_dir": Set the directory into which QnnIr will be configured to write QNN graphs as .dlc files. + * Default is current working directory. + * "qnn_ir_backend_path": File path to the QnnIr backend library. If "dump_qnn_ir_dlc" is enabled, use this path + * instead of looking for the Ir backend in the standard location. + * "qnn_saver_path": File path to the QNN Saver backend library. If specified, QNN Saver will be enabled and will + * dump QNN API calls to disk for replay/debugging. QNN Saver produces incorrect model inference results and + * may alter model/EP partitioning. Use only for debugging. + * "qnn_context_priority": QNN context priority. + * Available options: + * -# "low" + * -# "normal": Default. + * -# "normal_high" + * -# "high" + * "htp_graph_finalization_optimization_mode": Set the optimization mode for graph finalization on the HTP backend. + * Available options: + * -# "0": Default. + * -# "1": Faster preparation time, less optimal graph. + * -# "2": Longer preparation time, more optimal graph. + * -# "3": Longest preparation time, most likely even more optimal graph. See QNN SDK documentation for specific + * details. + * "soc_model": The SoC model number. Refer to the QNN SDK documentation for valid values. + * Defaults to "0" (unknown). + * "htp_arch": The minimum HTP architecture the driver will use to select compatible QNN operators. + * Available options: + * -# "0": Default (none). + * -# "68" + * -# "69" + * -# "73" + * -# "75" + * -# "81" + * "device_id": The ID of the device to use when setting 'htp_arch'. Defaults to "0" (for single device). + * "enable_htp_fp16_precision": Used for float32 model for HTP backend. + * Enable the float32 model to be inferenced with fp16 precision. Otherwise, it will be fp32 precision. + * -# "0": With fp32 precision. + * -# "1": Default. With fp16 precision. + * "offload_graph_io_quantization": Offload graph input quantization and graph output dequantization to another + * execution provider (typically CPU EP). + * -# "0": Disabled. QNN EP will handle quantization and dequantization of graph I/O. + * -# "1": Enabled. This is the default value. + * "enable_htp_spill_fill_buffer": Enable HTP spill fill buffer setting. The flag is used while generating context + * binary. + * -# "0": Default. Disabled. + * -# "1": Enabled. + * "enable_htp_shared_memory_allocator": Enable the QNN HTP shared memory allocator. Requires libcdsprpc.so/dll to + * be available. + * -# "0": Default. Disabled. + * -# "1": Enabled. + * "dump_json_qnn_graph": Set to "1" to dump QNN graphs generated by QNN EP as JSON files. Each graph partition + * assigned to QNN EP is dumped to a separate file. + * "json_qnn_graph_dir": Directory in which to dump QNN JSON graphs. If not specified, QNN graphs are dumped in the + * program's current working directory. Ignored if "dump_json_qnn_graph" is not set. + * "op_packages": QNN UDO op_package for QNN EP, allowed format: + *   "::[:],::[:]", + *   where op_type is the name of the operation, op_package_path is the path to the op package shared library, + * interface is the symbol name to register the op life cycle functions, and target is the backend type. For more + * details, refer to: https://docs.qualcomm.com/bundle/publicresource/topics/80-63442-50/op_packages.html + * [Advanced] "skip_qnn_version_check": Set to "1" to allow a different version of QNN to be used than what was compiled + * into ONNX Runtime. Differences in operator support, accuracy, performance, and QNN's ABI may lead to crashes, inaccurate + * results, and poor performance. Use with caution and test thoroughly. + * + * XNNPACK supported keys: + * "intra_op_num_threads": number of thread-pool size to use for XNNPACK execution provider. + * default value is 0, which means to use the session thread-pool size. + * + * \since Version 1.12. + */ + ORT_API2_STATUS(SessionOptionsAppendExecutionProvider, _In_ OrtSessionOptions* options, + _In_ const char* provider_name, + _In_reads_(num_keys) const char* const* provider_options_keys, + _In_reads_(num_keys) const char* const* provider_options_values, + _In_ size_t num_keys); + + /* \brief: Get a copy of kernel info + * + * \param[in] info Kernel info + * \param[out] info_copy Copy of kernel info + * + * \since Version 1.12. + */ + ORT_API2_STATUS(CopyKernelInfo, + _In_ const OrtKernelInfo* info, + _Outptr_ OrtKernelInfo** info_copy); + + /* \brief: Release kernel info + * + * \param[in] KernelInfo A copy of kernel info returned by CopyKernelInfo + * + * \since Version 1.12. + */ + ORT_CLASS_RELEASE(KernelInfo); + + /// \name Ort Training + /// @{ + /** \brief Gets the Training C Api struct + * + * Call this function to access the ::OrtTrainingApi structure that holds pointers to functions that enable + * training with onnxruntime. + * \note A NULL pointer will be returned and no error message will be printed if the training api + * is not supported with this build. A NULL pointer will be returned and an error message will be + * printed if the provided version is unsupported, for example when using a runtime older than the + * version created with this header file. + * + * \param[in] version Must be ::ORT_API_VERSION + * \return The ::OrtTrainingApi struct for the version requested. + * + * \since Version 1.13 + */ + const OrtTrainingApi*(ORT_API_CALL* GetTrainingApi)(uint32_t version)NO_EXCEPTION; + + /// @} + + /** \brief Append CANN provider to session options + * + * If CANN is not available (due to a non CANN enabled build, or if CANN is not installed on the system), this function will return failure. + * + * \param[in] options + * \param[in] cann_options + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.13. + */ + ORT_API2_STATUS(SessionOptionsAppendExecutionProvider_CANN, + _In_ OrtSessionOptions* options, _In_ const OrtCANNProviderOptions* cann_options); + + /** \brief Create an OrtCANNProviderOptions + * + * \param[out] out created ::OrtCANNProviderOptions. Must be released with OrtApi::ReleaseCANNProviderOptions + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.13. + */ + ORT_API2_STATUS(CreateCANNProviderOptions, _Outptr_ OrtCANNProviderOptions** out); + + /** \brief Set options in a CANN Execution Provider. + * + * \param[in] cann_options + * \param[in] provider_options_keys Array of UTF-8 null-terminated string for provider options keys + * \param[in] provider_options_values Array of UTF-8 null-terminated string for provider options values + * \param[in] num_keys Number of elements in the `provider_option_keys` and `provider_options_values` arrays + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.13. + */ + ORT_API2_STATUS(UpdateCANNProviderOptions, _Inout_ OrtCANNProviderOptions* cann_options, + _In_reads_(num_keys) const char* const* provider_options_keys, + _In_reads_(num_keys) const char* const* provider_options_values, + _In_ size_t num_keys); + + /** \brief Get serialized CANN provider options string. + * + * \param[in] cann_options OrtCANNProviderOptions instance + * \param[in] allocator a ptr to an instance of OrtAllocator obtained with CreateAllocator() + * or GetAllocatorWithDefaultOptions(), the specified allocator will be used to allocate + * continuous buffers for output strings and lengths. + * \param[out] ptr is a UTF-8 null terminated string allocated using 'allocator'. + * The caller is responsible for using the same allocator to free it. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.13. + */ + ORT_API2_STATUS(GetCANNProviderOptionsAsString, _In_ const OrtCANNProviderOptions* cann_options, + _Inout_ OrtAllocator* allocator, _Outptr_ char** ptr); + + /** \brief Release an OrtCANNProviderOptions + * + * \param[in] input The pointer of OrtCANNProviderOptions which will been deleted + * + * \since Version 1.13. + */ + void(ORT_API_CALL* ReleaseCANNProviderOptions)(_Frees_ptr_opt_ OrtCANNProviderOptions* input); + + /* \brief Get OrtDevice type from MemoryInfo + * + * \since Version 1.14 + */ + void(ORT_API_CALL* MemoryInfoGetDeviceType)(_In_ const OrtMemoryInfo* ptr, _Out_ OrtMemoryInfoDeviceType* out); + + /* \brief Update the OrtEnv instance with custom log severity level + * + * \param[in] ort_env The OrtEnv instance being used + * \param[in] log_severity_level The log severity level. + * + * \since Version 1.14. + */ + ORT_API2_STATUS(UpdateEnvWithCustomLogLevel, _In_ OrtEnv* ort_env, OrtLoggingLevel log_severity_level); + + /* \brief Set affinities for intra op threads + * + * Affinity string follows format: + * logical_processor_id,logical_processor_id;logical_processor_id,logical_processor_id + * Semicolon isolates configurations among threads, while comma split processors where ith thread expected to attach to. + * e.g. 1,2,3;4,5 + * specifies affinities for two threads, with the 1st thread attach to the 1st, 2nd, and 3rd processor, and 2nd thread to the 4th and 5th. + * To ease the configuration, an "interval" is also allowed: + * e.g. 1-8;8-16;17-24 + * orders that the 1st thread runs on first eight processors, 2nd thread runs on next eight processors, and so forth. + * Note: + * 1. Once set, the number of thread affinities must equal to intra_op_num_threads - 1, + * ort does not set affinity on the main thread which is started and managed by the calling app; + * 2. For windows, ort will infer the group id from a logical processor id, for example, assuming there are two groups with each has 64 logical processors, + * an id of 64 will be inferred as the last processor of the 1st group, while 65 will be interpreted as the 1st processor of the second group. + * Hence 64-65 is an invalid configuration, because a windows thread cannot be attached to processors across group boundary. + * + * \since Version 1.14 + */ + ORT_API2_STATUS(SetGlobalIntraOpThreadAffinity, _Inout_ OrtThreadingOptions* tp_options, const char* affinity_string); + + /** \brief Register custom ops from a shared library. + * + * Loads a shared library (.dll on windows, .so on linux, etc) named 'library_name' and looks for this entry point: + * OrtStatus* RegisterCustomOps(OrtSessionOptions * options, const OrtApiBase* api); + * It then passes in the provided session options to this function along with the api base. + * + * The handle to the loaded library is automatically released by ORT when the last OrtSession that references the + * library handle is released. If no OrtSession is created, then the library handle is released when the provided + * OrtSessionOptions is released. + * + * \param[in] options The session options. + * \param[in] library_name The name of the shared library to load and register. Refer to OS-specific dynamic library + * loading utilities (e.g., LoadLibraryEx on Windows or dlopen on Linux/MacOS) for information + * on the format of library names and search paths. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * \since Version 1.14 + */ + ORT_API2_STATUS(RegisterCustomOpsLibrary_V2, _Inout_ OrtSessionOptions* options, _In_ const ORTCHAR_T* library_name); + + /** \brief Register custom ops by calling a RegisterCustomOpsFn function. + * + * Searches for registration_func_name and if found calls it. + * + * The library containing the function must either be linked against or previously loaded by the executable. + * + * If you want ONNX Runtime to load the library and manage its lifetime, use RegisterCustomOpsLibrary_V2. + * + * RegisterCustomOpsUsingFunction can be used in scenarios where it may not be possible for ONNX Runtime to load + * the library from a path. e.g. mobile platforms where the library must be linked into the app. + * + * The registration function must have the signature of RegisterCustomOpsFn: + * OrtStatus* (*fn)(OrtSessionOptions* options, const OrtApiBase* api); + * + * See https://onnxruntime.ai/docs/reference/operators/add-custom-op.html for details on how the registration + * function should be implemented. + * + * \param[in] options OrtSessionOptions that is passed through as the first argument in the call to the + * registration function. + * \param[in] registration_func_name Name of registration function to use. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * \since Version 1.14 + */ + ORT_API2_STATUS(RegisterCustomOpsUsingFunction, _Inout_ OrtSessionOptions* options, + _In_ const char* registration_func_name); + + /// \name OrtKernelInfo + /// Custom operator APIs. + /// @{ + + /** \brief Get the number of inputs from ::OrtKernelInfo. + * + * Used in the CreateKernel callback of an OrtCustomOp to query the number of inputs + * during kernel/session creation. + * + * \param[in] info Instance of ::OrtKernelInfo. + * \param[out] out Pointer to variable assigned with the result on success. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * \since Version 1.14 + */ + ORT_API2_STATUS(KernelInfo_GetInputCount, _In_ const OrtKernelInfo* info, _Out_ size_t* out); + + /** \brief Get the number of outputs from ::OrtKernelInfo. + * + * Used in the CreateKernel callback of an OrtCustomOp to query the number of outputs + * during kernel/session creation. + * + * \param[in] info Instance of ::OrtKernelInfo. + * \param[out] out Pointer to variable assigned with the result on success. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * \since Version 1.14 + */ + ORT_API2_STATUS(KernelInfo_GetOutputCount, _In_ const OrtKernelInfo* info, _Out_ size_t* out); + + /** \brief Get the name of a ::OrtKernelInfo's input. + * + * Used in the CreateKernel callback of an OrtCustomOp to query an input's name + * during kernel/session creation. + * + * If `out` is nullptr, the value of `size` is set to the size of the name + * string (including null-terminator), and a success status is returned. + * + * If the `size` parameter is greater than or equal to the name string's size and `out` is not nullptr, + * the value of `size` is set to the true size of the string (including null-terminator), + * the provided memory is filled with the string's contents, and a success status is returned. + * + * If the `size` parameter is less than the actual string's size and `out` + * is not nullptr, the value of `size` is set to the true size of the string + * and a failure status is returned. + * + * \param[in] info An instance of ::OrtKernelInfo. + * \param[in] index The index of the input name to get. Returns a failure status if out-of-bounds. + * \param[out] out Memory location into which to write the UTF-8 null-terminated string representing the input's name. + * \param[in,out] size Pointer to the size of the `out` buffer. See above comments for details. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * \since Version 1.14 + */ + ORT_API2_STATUS(KernelInfo_GetInputName, _In_ const OrtKernelInfo* info, size_t index, _Out_opt_ char* out, + _Inout_ size_t* size); + + /** \brief Get the name of a ::OrtKernelInfo's output. + * + * Used in the CreateKernel callback of an OrtCustomOp to query an output's name + * during kernel/session creation. + * + * If `out` is nullptr, the value of `size` is set to the size of the name + * string (including null-terminator), and a success status is returned. + * + * If the `size` parameter is greater than or equal to the name string's size and `out` is not nullptr, + * the value of `size` is set to the true size of the string (including null-terminator), + * the provided memory is filled with the string's contents, and a success status is returned. + * + * If the `size` parameter is less than the actual string's size and `out` + * is not nullptr, the value of `size` is set to the true size of the string + * and a failure status is returned. + * + * \param[in] info An instance of ::OrtKernelInfo. + * \param[in] index The index of the output name to get. Returns a failure status if out-of-bounds. + * \param[out] out Memory location into which to write the UTF-8 null-terminated string representing the output's + * name. + * \param[in,out] size Pointer to the size of the `out` buffer. See above comments for details. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * \since Version 1.14 + */ + ORT_API2_STATUS(KernelInfo_GetOutputName, _In_ const OrtKernelInfo* info, size_t index, _Out_opt_ char* out, + _Inout_ size_t* size); + + /** \brief Get the type information for a ::OrtKernelInfo's input. + * + * Used in the CreateKernel callback of an OrtCustomOp to query the shape and type information + * of an input during kernel/session creation. + * + * \param[in] info An instance of ::OrtKernelInfo. + * \param[in] index Which input to get the type information for + * \param[out] type_info Pointer set to the resulting ::OrtTypeInfo. Must be freed with OrtApi::ReleaseTypeInfo. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * \since Version 1.14 + */ + ORT_API2_STATUS(KernelInfo_GetInputTypeInfo, _In_ const OrtKernelInfo* info, size_t index, + _Outptr_ OrtTypeInfo** type_info); + + /** \brief Get the type information for a ::OrtKernelInfo's output. + * + * Used in the CreateKernel callback of an OrtCustomOp to query the shape and type information + * of an output during kernel/session creation. + * + * \param[in] info An instance of ::OrtKernelInfo. + * \param[in] index Which input to get the type information for + * \param[out] type_info Pointer set to the resulting ::OrtTypeInfo. Must be freed with OrtApi::ReleaseTypeInfo. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * \since Version 1.14 + */ + ORT_API2_STATUS(KernelInfo_GetOutputTypeInfo, _In_ const OrtKernelInfo* info, size_t index, + _Outptr_ OrtTypeInfo** type_info); + + /** \brief Get a ::OrtValue tensor stored as an attribute in the graph node. + * + * Used in the CreateKernel callback of an OrtCustomOp to get a tensor attribute. + * + * \param[in] info ::OrtKernelInfo instance. + * \param[in] name UTF-8 null-terminated string representing the attribute's name. + * \param[in] allocator Allocator used to allocate the internal tensor state. + * \param[out] out Returns newly created ::OrtValue. Must be freed with OrtApi::ReleaseValue, + * which will also free internal tensor state allocated with the provided allocator. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(KernelInfoGetAttribute_tensor, _In_ const OrtKernelInfo* info, _In_z_ const char* name, + _Inout_ OrtAllocator* allocator, _Outptr_ OrtValue** out); + + /// @} + /// \name OrtSessionOptions + /// Custom operator APIs + /// @{ + + /** \brief Checks if the given session configuration entry exists. + * + * The config_key formats are defined in onnxruntime_session_options_config_keys.h + * + * Can be used in a custom operator library to check for session configuration entries + * that target one or more custom operators in the library. Example: The config entry + * custom_op.myop.some_key targets a custom op named "myop". + * + * \param[in] options The ::OrtSessionOptions instance. + * \param[in] config_key A null-terminated UTF-8 string representation of the configuration key. + * \param[out] out Pointer set to 1 if the entry exists and 0 otherwise. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * \since Version 1.14 + */ + ORT_API2_STATUS(HasSessionConfigEntry, _In_ const OrtSessionOptions* options, + _In_z_ const char* config_key, _Out_ int* out); + + /** \brief Get a session configuration value. + * + * Returns a failure status if the configuration key does not exist. + * The config_key and the format of config_value are defined in onnxruntime_session_options_config_keys.h + * + * If `config_value` is nullptr, the value of `size` is set to the true size of the string + * value (including null-terminator), and a success status is returned. + * + * If the `size` parameter is greater than or equal to the actual string value's size, + * the value of `size` is set to the true size of the string value, the provided memory + * is filled with the value's contents, and a success status is returned. + * + * If the `size` parameter is less than the actual string value's size and `config_value` + * is not nullptr, the value of `size` is set to the true size of the string value + * and a failure status is returned. + * + * Can be used in a custom operator library to get session configuration entries + * that target one or more custom operators in the library. Example: The config entry + * custom_op.myop.some_key targets a custom op named "myop". + * + * \param[in] options The session options. + * \param[in] config_key A null-terminated UTF-8 string representation of the config key. + * \param[in] config_value Pointer to memory where the null-terminated UTF-8 string value will be stored. + * \param[in,out] size Pointer to the size of the `config_value` buffer. See above comments for details. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * \since Version 1.14 + */ + ORT_API2_STATUS(GetSessionConfigEntry, _In_ const OrtSessionOptions* options, + _In_z_ const char* config_key, _Out_ char* config_value, _Inout_ size_t* size); + + /// @} + + /** \brief Append dnnl provider to session options + * + * If oneDNN is not available, this function will return failure. + * + * \param[in] options + * \param[in] dnnl_options + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.15. + */ + ORT_API2_STATUS(SessionOptionsAppendExecutionProvider_Dnnl, + _In_ OrtSessionOptions* options, _In_ const OrtDnnlProviderOptions* dnnl_options); + + /** \brief Create an OrtDnnlProviderOptions + * + * \param[out] out Newly created ::OrtDnnlProviderOptions. Must be released with OrtApi::ReleaseDnnlProviderOptions + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.15. + */ + ORT_API2_STATUS(CreateDnnlProviderOptions, _Outptr_ OrtDnnlProviderOptions** out); + + /** \brief Set options in a oneDNN Execution Provider. + * + * Key should be in null terminated string format of the member of ::OrtDnnlProviderOptions + * and value should be its related range. + * + * For example, key="use_arena" and value="1" + * + * \param[in] dnnl_options + * \param[in] provider_options_keys Array of UTF-8 null-terminated string for provider options keys + * \param[in] provider_options_values Array of UTF-8 null-terminated string for provider options values + * \param[in] num_keys Number of elements in the `provider_option_keys` and `provider_options_values` arrays + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.15. + */ + ORT_API2_STATUS(UpdateDnnlProviderOptions, _Inout_ OrtDnnlProviderOptions* dnnl_options, + _In_reads_(num_keys) const char* const* provider_options_keys, + _In_reads_(num_keys) const char* const* provider_options_values, + _In_ size_t num_keys); + + /** + * Get serialized oneDNN provider options string. + * + * For example, "use_arena=1;......" + * + * \param dnnl_options - OrtDnnlProviderOptions instance + * \param allocator - a ptr to an instance of OrtAllocator obtained with CreateAllocator() or GetAllocatorWithDefaultOptions() + * the specified allocator will be used to allocate continuous buffers for output strings and lengths. + * \param ptr - is a UTF-8 null terminated string allocated using 'allocator'. The caller is responsible for using the same allocator to free it. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.15. + */ + ORT_API2_STATUS(GetDnnlProviderOptionsAsString, _In_ const OrtDnnlProviderOptions* dnnl_options, _Inout_ OrtAllocator* allocator, _Outptr_ char** ptr); + + /** \brief Release an ::OrtDnnlProviderOptions + * + * \since Version 1.15. + */ + void(ORT_API_CALL* ReleaseDnnlProviderOptions)(_Frees_ptr_opt_ OrtDnnlProviderOptions* input); + + /// \name OrtKernelInfo + /// Custom operator APIs. + /// @{ + + /** \brief Get the graph node name from ::OrtKernelInfo. + * + * If `out` is nullptr, the value of `size` is set to the size of the name + * string (including null-terminator), and a success status is returned. + * + * If the `size` parameter is greater than or equal to the name string's size and `out` is not nullptr, + * the value of `size` is set to the true size of the string (including null-terminator), + * the provided memory is filled with the string's contents, and a success status is returned. + * + * If the `size` parameter is less than the actual string's size and `out` + * is not nullptr, the value of `size` is set to the true size of the string + * and a failure status is returned. + * + * Can be used in a custom operator's CreateKernel callback to get the name of the operator's node name in the graph. + * + * \param[in] info An instance of ::OrtKernelInfo. + * \param[out] out Memory location into which to write the UTF-8 null-terminated string representing the name. + * \param[in,out] size Pointer to the size of the `out` buffer. See above comments for details. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * \since Version 1.15 + */ + ORT_API2_STATUS(KernelInfo_GetNodeName, _In_ const OrtKernelInfo* info, _Out_opt_ char* out, _Inout_ size_t* size); + + /** \brief Get the session logger from ::OrtKernelInfo. + * + * Used in the CreateKernel callback of an OrtCustomOp to get a logger that can be used to log + * messages. + * + * \param[in] info An instance of ::OrtKernelInfo. + * \param[out] logger Pointer set to the session's ::OrtLogger. Owned by ONNX Runtime, so do not free. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * \since Version 1.15 + */ + ORT_API2_STATUS(KernelInfo_GetLogger, _In_ const OrtKernelInfo* info, _Outptr_ const OrtLogger** logger); + + /// @} + /// \name OrtKernelContext + /// Custom operator APIs. + /// @{ + + /** \brief Get the runtime logger from ::OrtKernelContext. + * + * Used in the KernelCompute callback of an OrtCustomOp to get a logger that can be used to log + * messages during inference. + * + * \param[in] context An instance of ::OrtKernelContext. + * \param[out] logger Pointer set to the kernel context's ::OrtLogger. Owned by ONNX Runtime, so do not free. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * \since Version 1.15 + */ + ORT_API2_STATUS(KernelContext_GetLogger, _In_ const OrtKernelContext* context, _Outptr_ const OrtLogger** logger); + + /// @} + /// \name OrtLogger + /// Custom operator APIs. + /// @{ + + /** \brief Logs a message at the given severity level using the provided ::OrtLogger. + * + * Only messages with a severity level equal or greater than the ::OrtLogger's logging severity level + * are logged. Use OrtApi::Logger_GetLoggingSeverityLevel to get the ::OrtLogger's logging severity + * level. + * + * Can be used in custom operators to log messages with the logger retrieved via OrtApi::KernelInfo_GetLogger. + * + * \param[in] logger The ::OrtLogger instance. + * \param[in] log_severity_level The message's severity level. + * \param[in] message The message to log. + * \param[in] file_path The filepath of the file in which the message is logged. Usually the value of ORT_FILE. + * \param[in] line_number The file line number in which the message is logged. Usually the value of __LINE__. + * \param[in] func_name The name of the function in which the message is logged. Usually the value of __FUNCTION__. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * \since Version 1.15 + */ + ORT_API2_STATUS(Logger_LogMessage, _In_ const OrtLogger* logger, OrtLoggingLevel log_severity_level, + _In_z_ const char* message, _In_z_ const ORTCHAR_T* file_path, int line_number, + _In_z_ const char* func_name); + + /** \brief Get the logging severity level of the ::OrtLogger. + * + * Can be used in a custom operator to get the logging severity level of the ::OrtLogger associated with + * the ::OrtKernelInfo. + * + * \param[in] logger The ::OrtLogger instance. + * \param[out] out Pointer to variable assigned with the logging severity level on success. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * \since Version 1.15 + */ + ORT_API2_STATUS(Logger_GetLoggingSeverityLevel, _In_ const OrtLogger* logger, _Out_ OrtLoggingLevel* out); + + /// @} + + /** \brief Get a ::OrtValue tensor stored as a constant initializer in the graph node. + * + * Used in the CreateKernel callback of an OrtCustomOp to get a tensor value. + * + * \param[in] info ::OrtKernelInfo instance. + * \param[in] index The node index. + * \param[out] is_constant Is it a constant node input or not. + * \param[out] out The OrtValue tensor value. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.15. + */ + ORT_API2_STATUS(KernelInfoGetConstantInput_tensor, _In_ const OrtKernelInfo* info, size_t index, _Out_ int* is_constant, _Outptr_ const OrtValue** out); + + /** \brief Get Optional Type information from an ::OrtTypeInfo + * + * This augments ::OrtTypeInfo to return an ::OrtOptionalTypeInfo when the type is optional. + * The OrtOptionalTypeInfo also has a nested ::OrtTypeInfo that describes the type of the optional value. + * ::OrtOptionalTypeInfo type can only appear within model metadata to describe inputs/outputs. + * The actual OrtValues that are supplied in place of optional type inputs should contain + * specific type that is described by ::OrtOptionalTypeInfo. + * + * So the picture: ::OrtTypeInfo -> ::OrtOptionalTypeInfo -> ::OrtTypeInfo (describes the type that can be supplied + * in place of the optional type when creating the actual ::OrtValue). + * + * \param[in] type_info + * \param[out] out A pointer to the ::OrtOptionalTypeInfo. Do not free this value, + * it is owned by OrtTypeInfo instance. When the type_info does not represent + * optional type, nullptr is returned in out. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.15. + */ + ORT_API2_STATUS(CastTypeInfoToOptionalTypeInfo, _In_ const OrtTypeInfo* type_info, + _Outptr_result_maybenull_ const OrtOptionalTypeInfo** out); + + /** \brief Get OrtTypeInfo for the allowed contained type from an ::OrtOptionalTypeInfo. + * + * This augments ::OrtOptionalTypeInfo to return an ::OrtTypeInfo for the contained type. + * The OrtOptionalTypeInfo has a nested ::OrtTypeInfo that describes the type of the optional value. + * ::OrtOptionalTypeInfo type can only appear within model metadata to describe inputs/outputs. + * The actual OrtValues that are supplied in place of optional type inputs should contain + * specific type that is described by the returned ::OrtTypeInfo. + * + * \param[in] optional_type_info + * \param[out] out A copy of ::OrtTypeInfo for what the optional value could be. + * The user must free this value with ReleaseTypeInfo. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.15. + */ + ORT_API2_STATUS(GetOptionalContainedTypeInfo, _In_ const OrtOptionalTypeInfo* optional_type_info, + _Outptr_ OrtTypeInfo** out); + + /** \brief Set a single string in a string tensor + * Do not zero terminate the string data. + * + * \param[in] value A string tensor + * \param[in] index - flat index of the element + * \param[in] length_in_bytes length of the buffer in utf-8 bytes (without the null terminator) + * \param[inout] buffer - address of return value + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(GetResizedStringTensorElementBuffer, _Inout_ OrtValue* value, _In_ size_t index, _In_ size_t length_in_bytes, _Inout_ char** buffer); + + /** \brief Get Allocator from KernelContext for a specific memoryInfo. Please use C API ReleaseAllocator to release out object + * + * \param[in] context OrtKernelContext instance + * \param[in] mem_info OrtMemoryInfo instance + * \param[out] out A pointer to OrtAllocator. Must be released with OrtApi::ReleaseAllocator. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.15. + */ + ORT_API2_STATUS(KernelContext_GetAllocator, _In_ const OrtKernelContext* context, _In_ const OrtMemoryInfo* mem_info, _Outptr_ OrtAllocator** out); + + /** \brief Returns a null terminated string of the build info including git info and cxx flags + * + * \return UTF-8 encoded version string. Do not deallocate the returned buffer. + * + * \since Version 1.15. + */ + const char*(ORT_API_CALL* GetBuildInfoString)(void); + + /// \name OrtROCMProviderOptions + /// @{ + + /** \brief Create an OrtROCMProviderOptions + * + * \param[out] out Newly created ::OrtROCMProviderOptions. Must be released with OrtApi::ReleaseROCMProviderOptions + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.16. + */ + ORT_API2_STATUS(CreateROCMProviderOptions, _Outptr_ OrtROCMProviderOptions** out); + + /** \brief Set options in a ROCm Execution Provider. + * + * Please refer to https://onnxruntime.ai/docs/execution-providers/ROCm-ExecutionProvider.html + * to know the available keys and values. Key should be in null terminated string format of the member of + * ::OrtROCMProviderOptions and value should be its related range. + * + * For example, key="device_id" and value="0" + * + * \param[in] rocm_options + * \param[in] provider_options_keys Array of UTF-8 null-terminated string for provider options keys + * \param[in] provider_options_values Array of UTF-8 null-terminated string for provider options values + * \param[in] num_keys Number of elements in the `provider_option_keys` and `provider_options_values` arrays + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.16. + */ + ORT_API2_STATUS(UpdateROCMProviderOptions, _Inout_ OrtROCMProviderOptions* rocm_options, + _In_reads_(num_keys) const char* const* provider_options_keys, + _In_reads_(num_keys) const char* const* provider_options_values, + _In_ size_t num_keys); + + /** + * Get serialized ROCm provider options string. + * + * For example, "device_id=0;arena_extend_strategy=0;......" + * + * \param rocm_options - OrtROCMProviderOptions instance + * \param allocator - a ptr to an instance of OrtAllocator obtained with CreateAllocator() or GetAllocatorWithDefaultOptions() + * the specified allocator will be used to allocate continuous buffers for output strings and lengths. + * \param ptr - is a UTF-8 null terminated string allocated using 'allocator'. The caller is responsible for using the same allocator to free it. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.16. + */ + ORT_API2_STATUS(GetROCMProviderOptionsAsString, _In_ const OrtROCMProviderOptions* rocm_options, _Inout_ OrtAllocator* allocator, _Outptr_ char** ptr); + + /** \brief Release an ::OrtROCMProviderOptions + * + * \note This is an exception in the naming convention of other Release* functions, as the name of the method does not have the V2 suffix, but the type does + * + * \since Version 1.16. + */ + void(ORT_API_CALL* ReleaseROCMProviderOptions)(_Frees_ptr_opt_ OrtROCMProviderOptions* input); + + /** \brief Create an allocator with specific type and register it with the ::OrtEnv + * This API enhance CreateAndRegisterAllocator that it can create an allocator with specific type, not just CPU allocator + * Enables sharing the allocator between multiple sessions that use the same env instance. + * Lifetime of the created allocator will be valid for the duration of the environment. + * Returns an error if an allocator with the same ::OrtMemoryInfo is already registered. + * \param[in] env OrtEnv instance + * \param[in] provider_type ExecutionProvider type + * \param[in] mem_info OrtMemoryInfo instance + * \param[in] arena_cfg Arena configuration + * \param[in] provider_options_keys key of the provider options map + * \param[in] provider_options_values value of the provider options map + * \param[in] num_keys Length of the provider options map + */ + ORT_API2_STATUS(CreateAndRegisterAllocatorV2, _Inout_ OrtEnv* env, _In_ const char* provider_type, + _In_ const OrtMemoryInfo* mem_info, _In_ const OrtArenaCfg* arena_cfg, + _In_reads_(num_keys) const char* const* provider_options_keys, _In_reads_(num_keys) const char* const* provider_options_values, _In_ size_t num_keys); + + /** \brief Run the model asynchronously in a thread owned by intra op thread pool + * + * \param[in] session + * \param[in] run_options If nullptr, will use a default ::OrtRunOptions + * \param[in] input_names Array of null terminated UTF8 encoded strings of the input names + * \param[in] input Array of ::OrtValue%s of the input values + * \param[in] input_len Number of elements in the input_names and inputs arrays + * \param[in] output_names Array of null terminated UTF8 encoded strings of the output names + * \param[in] output_names_len Number of elements in the output_names and outputs array + * \param[out] output OrtValue* array of size output_names_len. + * On calling RunAsync, output[i] could either be a null or a pointer to a preallocated OrtValue. + * Later, the output array will be passed to run_async_callback with all null(s) filled with valid + * OrtValue pointer(s) allocated by onnxruntime. + * NOTE: it is customer's duty to finally release the output array and each of its member, + * regardless of whether the member (OrtValue*) is allocated by onnxruntime or preallocated by the customer. + * \param[in] run_async_callback Callback function on model run completion + * \param[in] user_data User data that pass back to run_async_callback + */ + ORT_API2_STATUS(RunAsync, _Inout_ OrtSession* session, _In_opt_ const OrtRunOptions* run_options, + _In_reads_(input_len) const char* const* input_names, + _In_reads_(input_len) const OrtValue* const* input, size_t input_len, + _In_reads_(output_names_len) const char* const* output_names, size_t output_names_len, + _Inout_updates_all_(output_names_len) OrtValue** output, + _In_ RunAsyncCallbackFn run_async_callback, _In_opt_ void* user_data); + + /** + * Update TensorRT EP provider option where its data type is pointer, for example 'user_compute_stream'. + * If the data type of the provider option can be represented by string please use UpdateTensorRTProviderOptions. + * + * Note: It's caller's responsibility to properly manage the lifetime of the instance pointed by this pointer. + * + * \param tensorrt_options - OrtTensorRTProviderOptionsV2 instance + * \param key - Name of the provider option + * \param value - A pointer to the instance that will be assigned to this provider option + * + * \since Version 1.16. + */ + ORT_API2_STATUS(UpdateTensorRTProviderOptionsWithValue, _Inout_ OrtTensorRTProviderOptionsV2* tensorrt_options, _In_ const char* key, _In_ void* value); + + /** + * Get TensorRT EP provider option where its data type is pointer. + * If the data type of the provider option can be represented by string please use GetTensorRTProviderOptionsAsString. + * + * \param tensorrt_options - OrtTensorRTProviderOptionsV2 instance + * \param key - Name of the provider option + * \param ptr - A pointer to the instance that is kept by the provider option + * + * \since Version 1.16. + */ + ORT_API2_STATUS(GetTensorRTProviderOptionsByName, _In_ const OrtTensorRTProviderOptionsV2* tensorrt_options, _In_ const char* key, _Outptr_ void** ptr); + + /** + * Update CUDA EP provider option where its data type is pointer, for example 'user_compute_stream'. + * If the data type of the provider option can be represented by string please use UpdateCUDAProviderOptions. + * + * Note: It's caller's responsibility to properly manage the lifetime of the instance pointed by this pointer. + * + * \param cuda_options - OrtCUDAProviderOptionsV2 instance + * \param key - Name of the provider option + * \param value - A pointer to the instance that will be assigned to this provider option + * + * \since Version 1.16. + */ + ORT_API2_STATUS(UpdateCUDAProviderOptionsWithValue, _Inout_ OrtCUDAProviderOptionsV2* cuda_options, _In_ const char* key, _In_ void* value); + + /** + * Get CUDA EP provider option where its data type is pointer. + * If the data type of the provider option can be represented by string please use GetCUDAProviderOptionsAsString. + * + * \param cuda_options - OrtCUDAProviderOptionsV2 instance + * \param key - Name of the provider option + * \param ptr - A pointer to the instance that is kept by the provider option + * + * \since Version 1.16. + */ + ORT_API2_STATUS(GetCUDAProviderOptionsByName, _In_ const OrtCUDAProviderOptionsV2* cuda_options, _In_ const char* key, _Outptr_ void** ptr); + + /** + * Get a EP resource. + * E.g. a cuda stream or a cublas handle + * + * \param context - Kernel context + * \param resource_version - Version of the resource + * \param resource_id - Type of resource + * \param resource - A pointer to returned resource + * + * \since Version 1.16. + */ + ORT_API2_STATUS(KernelContext_GetResource, _In_ const OrtKernelContext* context, _In_ int resource_version, + _In_ int resource_id, _Outptr_ void** resource); + + /** \brief Set user logging function + * + * By default the logger created by the CreateEnv* functions is used to create the session logger as well. + * This function allows a user to override this default session logger with a logger of their own choosing. This way + * the user doesn't have to create a separate environment with a custom logger. This addresses the problem when + * the user already created an env but now wants to use a different logger for a specific session (for debugging or + * other reasons). + * + * \param[in] options + * \param[in] user_logging_function A pointer to a logging function. + * \param[in] user_logging_param A pointer to arbitrary data passed as the ::OrtLoggingFunction `param` parameter to + * `user_logging_function`. This parameter is optional. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.17. + */ + ORT_API2_STATUS(SetUserLoggingFunction, _Inout_ OrtSessionOptions* options, + _In_ OrtLoggingFunction user_logging_function, _In_opt_ void* user_logging_param); + + /** + * Get number of input from OrtShapeInferContext + * + * \param[in] context + * \param[out] out The number of inputs + * + * \since Version 1.17. + */ + ORT_API2_STATUS(ShapeInferContext_GetInputCount, _In_ const OrtShapeInferContext* context, _Out_ size_t* out); + + /** + * Get type and shape info of an input + * + * \param[in] context + * \param[in] index The index of the input + * \param[out] info Type shape info of the input + * + * \since Version 1.17. + */ + ORT_API2_STATUS(ShapeInferContext_GetInputTypeShape, _In_ const OrtShapeInferContext* context, _In_ size_t index, _Outptr_ OrtTensorTypeAndShapeInfo** info); + + /** + * Get attribute from OrtShapeInferContext. Note that OrtShapeInferContext is a per-node context, one could only read attribute from current node. + * + * \param[in] context + * \param[in] attr_name Name of the attribute + * \param[out] attr Handle of the attribute fetched + * + * \since Version 1.17. + */ + ORT_API2_STATUS(ShapeInferContext_GetAttribute, _In_ const OrtShapeInferContext* context, _In_ const char* attr_name, _Outptr_ const OrtOpAttr** attr); + + /** + * Set type and shape info of an output + * + * \param[in] context + * \param[in] index The index of the output + * \param[out] info Type shape info of the output + * + * \since Version 1.17. + */ + ORT_API2_STATUS(ShapeInferContext_SetOutputTypeShape, _In_ const OrtShapeInferContext* context, _In_ size_t index, _In_ const OrtTensorTypeAndShapeInfo* info); + + /** + * Set symbolic shape to type shape info + * + * \param[in] info Type shape info + * \param[in] dim_params Symbolic strings + * \param[in] dim_params_length Number of strings + * + * \since Version 1.17. + */ + ORT_API2_STATUS(SetSymbolicDimensions, _In_ OrtTensorTypeAndShapeInfo* info, _In_ const char* dim_params[], _In_ size_t dim_params_length); + + /** + * Read contents of an attribute to data + * + * \param[in] op_attr + * \param[in] type Attribute type + * \param[out] data Memory address to save raw content of the attribute + * \param[in] len Number of bytes allowed to store in data + * \param[out] out Number of bytes required to save the data when the call failed, or the real number of bytes saved to data on success + * + * \note Does not support reading graph attributes. Refer to Node_GetSubgraphs. + * + * \since Version 1.17. + */ + ORT_API2_STATUS(ReadOpAttr, _In_ const OrtOpAttr* op_attr, _In_ OrtOpAttrType type, _Inout_ void* data, _In_ size_t len, _Out_ size_t* out); + + /** \brief Set whether to use deterministic compute. + * + * Default is false. If set to true, this will enable deterministic compute for GPU kernels where possible. + * Note that this most likely will have a performance cost. + * + * \param[in] options + * \param[in] value + * + * \since Version 1.17. + */ + ORT_API2_STATUS(SetDeterministicCompute, _Inout_ OrtSessionOptions* options, bool value); + + /** + * Run fn in parallel + * + * \param[in] context + * \param[in] fn Function accepting usr_data and an integer as iterator + * \param[in] total The number of times fn is to be invoked + * \param[in] num_batch Number of batches by which the "total" is to be divided in maximum. When zero, there is no limit + * \param[in] usr_data User data to be passed back to fn + * + * \since Version 1.17. + */ + ORT_API2_STATUS(KernelContext_ParallelFor, _In_ const OrtKernelContext* context, _In_ void (*fn)(void*, size_t), _In_ size_t total, _In_ size_t num_batch, _In_ void* usr_data); + + /** \brief Append OpenVINO execution provider to the session options + * + * If OpenVINO is not available (due to a non OpenVINO enabled build, or if OpenVINO is not installed on the system), this function will fail. + * + * \param[in] options + * \param[in] provider_options_keys + * \param[in] provider_options_values + * \param[in] num_keys + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.17. + */ + ORT_API2_STATUS(SessionOptionsAppendExecutionProvider_OpenVINO_V2, + _In_ OrtSessionOptions* options, + _In_reads_(num_keys) const char* const* provider_options_keys, + _In_reads_(num_keys) const char* const* provider_options_values, + _In_ size_t num_keys); + + /** \brief Append VitisAI provider to session options + * + * If VitisAI is not available (due to a non VitisAI enabled build, or if VitisAI is not installed on the system), this function will return failure. + * + * \param[in] options + * \param[in] provider_options_keys + * \param[in] provider_options_values + * \param[in] num_keys + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.18. + */ + ORT_API2_STATUS(SessionOptionsAppendExecutionProvider_VitisAI, + _In_ OrtSessionOptions* options, + _In_reads_(num_keys) const char* const* provider_options_keys, + _In_reads_(num_keys) const char* const* provider_options_values, + _In_ size_t num_keys); + + /** \brief Get scratch buffer from the corresponding allocator under the specific OrtMemoryInfo object. + * NOTE: callers are responsible to release this scratch buffer from the corresponding allocator + * \param[in] context OrtKernelContext instance + * \param[in] mem_info OrtMemoryInfo instance + * \param[in] count_or_bytes How many bytes is this scratch buffer + * \param[out] out A pointer to the scratch buffer + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.18. + */ + ORT_API2_STATUS(KernelContext_GetScratchBuffer, _In_ const OrtKernelContext* context, _In_ const OrtMemoryInfo* mem_info, _In_ size_t count_or_bytes, _Outptr_ void** out); + + /** \brief Get allocator from KernelInfo for a specific memory type. Please use C API ReleaseAllocator to release out object + * + * \param[in] info OrtKernelInfo instance + * \param[in] mem_type OrtMemType object + * \param[out] out A pointer to OrtAllocator + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.18. + */ + ORT_API2_STATUS(KernelInfoGetAllocator, _In_ const OrtKernelInfo* info, _In_ OrtMemType mem_type, _Outptr_ OrtAllocator** out); + + /** \brief Replace initialized Tensors with external data with the provided files in memory + * + * The function will find the initialized TensorProtos with external data in the graph with the provided + * external file names and the file content in memory. The API gets the external file name, offset, data length + * from TensorProto, and locate the tensor data from the file in memory buffer. + * It creates a Tensor to replace the existing Tensor in graph. The replacement + * will occur before any of the optimizations take place. The data will be copied into the graph + * since TensorProto can't refer to the user provided buffers. + * + * \param[in] options + * \param[in] external_initializer_file_names Array of null terminated UTF-8 encoded strings of the file names + * which holds the external initializers. + * \param[in] external_initializer_file_buffer_array Array of pointers to the buffer of the file content. + * The buffer can be freed after session creation. + * \param[in] external_initializer_file_lengths Array of size_t to indicate the length of file content + * \param[in] num_external_initializer_files Number of external files + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.18. + */ + ORT_API2_STATUS(AddExternalInitializersFromFilesInMemory, _In_ OrtSessionOptions* options, + _In_reads_(num_external_initializer_files) const ORTCHAR_T* const* external_initializer_file_names, + _In_reads_(num_external_initializer_files) char* const* external_initializer_file_buffer_array, + _In_reads_(num_external_initializer_files) const size_t* external_initializer_file_lengths, + size_t num_external_initializer_files); + + /** \brief Create an OrtLoraAdapter + * + * The function attempts to locate file specified by adapter_file_path, read it and create an OrtLoraAdapter + * instance. The adapter_file_path should be a valid path to a file that contains a valid Lora Adapter + * format. The function attempts to validate the format at load time. The file will always be memory mapped, unless + * the platform does not support memory mapping, in which case the file will be read into memory. + * + * \param[in] adapter_file_path adapter file path. + * \param[in] allocator optional pointer to a device allocator. If specified + * data is copied to the device at some point before Run() is invoked. If nullptr, data stays on CPU. + * The data would still be copied to device if required by the model at inference time. + * \param[out] out A pointer to a newly created OrtLoraAdapter instance. Must be released with + * OrtApi::ReleaseLoraAdapter. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.20. + */ + ORT_API2_STATUS(CreateLoraAdapter, const ORTCHAR_T* adapter_file_path, _In_ OrtAllocator* allocator, + _Outptr_ OrtLoraAdapter** out); + + /** \brief Create an OrtLoraAdapter + * + * The function copies the bytes from the array and creates an OrtLoraAdapter instance. + * + * + * \param[in] bytes pointer to a valid Lora Adapter format buffer. + * \param[in] num_bytes length of bytes buffer. + * \param[in] allocator optional pointer to a device allocator. If specified + * data is copied to the device at some point before Run() is invoked. If nullptr, data stays on CPU. + * The data would still be copied to device if required by the model at inference time. + * \param[out] out A pointer to a newly created OrtLoraAdapter instance. Must be released with + * OrtApi::ReleaseLoraAdapter. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.20. + */ + ORT_API2_STATUS(CreateLoraAdapterFromArray, _In_ const void* bytes, size_t num_bytes, _In_ OrtAllocator* allocator, + _Outptr_ OrtLoraAdapter** out); + + /** \brief Release an ::OrtLoraAdapter obtained from OrtApi::CreateLoraAdapter + */ + ORT_CLASS_RELEASE(LoraAdapter); + + /** \brief Add the Lora Adapter to the list of active adapters. + * + * The function adds the Lora Adapter to the list of active adapters. The Lora Adapter must be created with + * OrtApi::CreateLoraAdapter or FromArray. The Lora Adapter will be used by the session to run the model. + * The instance of the OrtRunOptions can then be used to customize the Run() calls. + * More than one OrtLoraAdapter can be active at the same time. Lora Parameters that belong to different + * Lora adapters that will be active at the same time must not overlap. + * This setting does not affect RunWithBinding. + * + * \param[in] options OrtRunOptions instance + * \param[in] adapter OrtLoraAdapter instance + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.20. + */ + ORT_API2_STATUS(RunOptionsAddActiveLoraAdapter, _Inout_ OrtRunOptions* options, _In_ const OrtLoraAdapter* adapter); + + /// @} + /// \name OrtEpDynamicOptions + /// @{ + + /** \brief Set DynamicOptions for EPs (Execution Providers) + * + * Valid options can be found in `include\onnxruntime\core\session\onnxruntime_session_options_config_keys.h` + * Look for `kOrtEpDynamicOptions` + * + * \param[in] sess OrtSession + * \param[in] keys Array of null terminated UTF8 encoded strings of EP dynamic option keys + * \param[in] values Array of null terminated UTF8 encoded string of EP dynamic option values + * \param[in] kv_len Number of elements in the keys and values arrays + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.20. + */ + ORT_API2_STATUS(SetEpDynamicOptions, _Inout_ OrtSession* sess, _In_reads_(kv_len) const char* const* keys, + _In_reads_(kv_len) const char* const* values, _In_ size_t kv_len); + + /// @} + + /** \brief Release an OrtValueInfo instance if it was not added to an OrtGraph. + * \since Version 1.22. + */ + ORT_CLASS_RELEASE(ValueInfo); + + /** \brief Release an OrtNode if it was not added to an OrtGraph. + * \since Version 1.22. + */ + ORT_CLASS_RELEASE(Node); + + /** \brief Release an OrtGraph. + * \since Version 1.22. + */ + ORT_CLASS_RELEASE(Graph); + + /** \brief Release an OrtModel. + * \since Version 1.22. + */ + ORT_CLASS_RELEASE(Model); + + /** \brief Get the value name from an OrtValueInfo instance. + * \param[in] value_info The OrtValueInfo instance. + * \param[out] name The name of the OrtValueInfo + * \snippet{doc} snippets.dox OrtStatus Return Value + * \since Version 1.22. + */ + ORT_API2_STATUS(GetValueInfoName, _In_ const OrtValueInfo* value_info, _Out_ const char** name); + + /** \brief Get the type information from an OrtValueInfo instance. + * \param[in] value_info The OrtValueInfo instance. + * \param[out] type_info The type info of the OrtValueInfo + * \snippet{doc} snippets.dox OrtStatus Return Value + * \since Version 1.22. + */ + ORT_API2_STATUS(GetValueInfoTypeInfo, _In_ const OrtValueInfo* value_info, _Outptr_ const OrtTypeInfo** type_info); + + /** \brief Get the Model Editor API instance + * + * Get the Model Editor API instance to create a new model or augment an existing model. + * + * \return Model Editor API struct + * + * \since Version 1.22. + */ + const OrtModelEditorApi*(ORT_API_CALL* GetModelEditorApi)(void); + + /** \brief Create an OrtValue for a Tensor that uses pre-existing memory. + * + * ORT will take ownership of the memory and free it using the provided deleter when no longer in use. + * + * \param[in] deleter OrtAllocator instance that will be used to free the memory. + * Only the OrtAllocator:Info and OrtAllocator::Release functions are required. + * The OrtMemoryInfo returned by OrtAllocator::Info must match the location of p_data. + * \param[in] p_data Pointer to the memory that will be used by the Tensor. ORT will take ownership of the memory. + * \param[in] p_data_len Length of the memory in bytes. + * \param[in] shape Dimensions of the Tensor. All values should be > 0. + * \param[in] shape_len Number of dimensions in the shape array. + * \param[in] type Data type of the Tensor. + * \param[out] out Newly created ::OrtValue. Must be freed with OrtApi::ReleaseValue + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.22. + */ + ORT_API2_STATUS(CreateTensorWithDataAndDeleterAsOrtValue, _In_ OrtAllocator* deleter, + _In_ void* p_data, size_t p_data_len, + _In_ const int64_t* shape, size_t shape_len, + ONNXTensorElementDataType type, + _Outptr_ OrtValue** out); + + /** \brief sets load cancellation flag to abort session loading process. + * + * \param[in] options instance that was passed to the session at creation time. + * \param[in] cancel setting this to true after model loading process was initiated will + * attempt to cancel the loading process. If cancellation is successful, CreateSession() + * CreateSessionFromArray() or any other session creation API that take session options as an + * argument will return an OrtStatus indicating that session loading was canceled at user request, + * error code ORT_MODEL_LOAD_CANCELED. + * The APIs above would not return any valid Session instance. This is the best case effort and the result + * is not guaranteed. The session may have already been created and initialized + * before the cancellation request was issued. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.22. + */ + ORT_API2_STATUS(SessionOptionsSetLoadCancellationFlag, _Inout_ OrtSessionOptions* options, + _In_ bool cancel); + + /** \brief Get the Compile API instance. + * + * Get the Compile API instance to compile ONNX models. Execution providers that support compilation fuse a subgraph + * into an EPContext node that wraps a provider-specific binary representation of the subgraph. + * For more details about the EPContext design, refer to: + * \htmlonly + * EPContext design document. + * \endhtmlonly + * + * \return Compile API struct instance. + * + * \since Version 1.22. + */ + const OrtCompileApi*(ORT_API_CALL* GetCompileApi)(void); + + // + // OrtKeyValuePairs + // + + /** \brief Create an OrtKeyValuePairs instance. + * + * \param[out] out A pointer to a newly created OrtKeyValuePairs instance. + * + * \note Must be released by calling ReleaseKeyValuePairs. + * + * \since Version 1.22. + */ + void(ORT_API_CALL* CreateKeyValuePairs)(_Outptr_ OrtKeyValuePairs** out); + + /** \brief Add a key-value pair to the OrtKeyValuePairs instance. + * + * If a pair with the same key already exists, it is overwritten. + * + * \param[in] kvps OrtKeyValuePairs instance. + * \param[in] key Key to be added. + * \param[in] value Value to be added. + * + * \note The `key` and `value` are copied internally. + * + * \since Version 1.22. + */ + + void(ORT_API_CALL* AddKeyValuePair)(_In_ OrtKeyValuePairs* kvps, _In_ const char* key, _In_ const char* value); + + /** \brief Get the value associated with a key in the OrtKeyValuePairs instance. + * + * \param[in] kvps OrtKeyValuePairs instance. + * \param[in] key Key to be searched. + * + * \return The value associated with the key, or nullptr if the key does not exist. + * + * \since Version 1.22. + */ + const char*(ORT_API_CALL* GetKeyValue)(_In_ const OrtKeyValuePairs* kvps, _In_ const char* key); + + /** \brief Get all the key-value pairs from the OrtKeyValuePairs instance. + * + * \param[in] kvps OrtKeyValuePairs instance. + * \param[out] keys Array of keys from `kvps`. + * \param[out] values Array of values from `kvps`. + * \param[out] num_entries Number of entries in `keys` and `values`. + * + * \since Version 1.22. + */ + void(ORT_API_CALL* GetKeyValuePairs)(_In_ const OrtKeyValuePairs* kvps, + _Outptr_ const char* const** keys, _Outptr_ const char* const** values, + _Out_ size_t* num_entries); + + /** \brief Remove a key-value pair from the OrtKeyValuePairs instance. + * + * \param[in] kvps OrtKeyValuePairs instance. + * \param[in] key Key to be removed. No error if not found. + * + * \since Version 1.22. + */ + void(ORT_API_CALL* RemoveKeyValuePair)(_In_ OrtKeyValuePairs* kvps, _In_ const char* key); + + /** \brief Release an OrtKeyValuePairs instance. + * + * \param[in] input OrtKeyValuePairs instance to be released. + * + * \since Version 1.22. + */ + ORT_CLASS_RELEASE(KeyValuePairs); + + /** \brief Register an execution provider library with ORT. + * + * The library must export 'CreateEpFactories' and 'ReleaseEpFactory' functions. + * See OrtEpApi for more details. + * + * \param[in] env The OrtEnv instance to register the library in. + * \param[in] registration_name The name to register the execution provider library under. + * \param[in] path The path to the execution provider library. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.22. + */ + ORT_API2_STATUS(RegisterExecutionProviderLibrary, _In_ OrtEnv* env, _In_ const char* registration_name, + _In_ const ORTCHAR_T* path); + + /** \brief Unregister an execution provider library with ORT. + * + * ORT will call ReleaseEpFactory for all factories created by the library, and unload the library. + * + * You MUST ensure there are no Session instances using execution providers created by the library + * before calling this function. + * + * \param[in] env The OrtEnv instance to unregister the library from. + * \param[in] registration_name The name the execution provider library was registered under. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.22. + */ + ORT_API2_STATUS(UnregisterExecutionProviderLibrary, _In_ OrtEnv* env, _In_ const char* registration_name); + + /** \brief Get the list of available OrtEpDevice instances. + * + * Each OrtEpDevice instance contains details of the execution provider and the device it will use. + * + * \param[in] env The OrtEnv instance to query. + * \param[out] ep_devices The OrtEpDevice instances that the execution provider will use. + * \param[out] num_ep_devices The number of OrtEpDevice instances returned. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.22. + */ + ORT_API2_STATUS(GetEpDevices, _In_ const OrtEnv* env, + _Outptr_ const OrtEpDevice* const** ep_devices, _Out_ size_t* num_ep_devices); + + /** \brief Append the execution provider that is responsible for the selected OrtEpDevice instances + * to the session options. + * + * \param[in] session_options Session options to add execution provider to. + * \param[in] env Environment that execution providers were registered with. + * \param[in] ep_devices One or more OrtEpDevice instances to create an execution provider for. + * Obtain from GetEpDevices. All OrtEpDevice instances must be from the same execution + * provider. It is only necessary to provide multiple OrtEpDevices if you want to use the + * same execution provider for multiple devices. + * e.g. the EP is capable of running on GPU and NPU. + * \param[in] num_ep_devices Number of OrtEpDevice instances. + * \param[in] ep_option_keys Optional keys to configure the execution provider. + * \param[in] ep_option_vals Optional values to configure the execution provider. + * \param[in] num_ep_options Number of execution provide options to add. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.22. + */ + ORT_API2_STATUS(SessionOptionsAppendExecutionProvider_V2, _In_ OrtSessionOptions* session_options, + _In_ OrtEnv* env, + _In_reads_(num_ep_devices) const OrtEpDevice* const* ep_devices, _In_ size_t num_ep_devices, + _In_reads_(num_op_options) const char* const* ep_option_keys, + _In_reads_(num_op_options) const char* const* ep_option_vals, + size_t num_ep_options); + + /** \brief Set the execution provider selection policy for the session. + * + * Allows users to specify a device selection policy for automatic execution provider (EP) selection. + * If custom selection is required please use SessionOptionsSetEpSelectionPolicyDelegate instead. + * + * \param[in] session_options The OrtSessionOptions instance. + * \param[in] policy The device selection policy to use (see OrtExecutionProviderDevicePolicy). + * + * \since Version 1.22 + */ + ORT_API2_STATUS(SessionOptionsSetEpSelectionPolicy, _In_ OrtSessionOptions* session_options, + _In_ OrtExecutionProviderDevicePolicy policy); + + /** \brief Set the execution provider selection policy delegate for the session. + * + * Allows users to provide a custom device selection policy for automatic execution provider (EP) selection. + * + * \param[in] session_options The OrtSessionOptions instance. + * \param[in] delegate Delegate callback for custom selection. + * \param[in] delegate_state Optional state that will be passed to the delegate callback. nullptr if not required. + * + * \since Version 1.22 + */ + ORT_API2_STATUS(SessionOptionsSetEpSelectionPolicyDelegate, _In_ OrtSessionOptions* session_options, + _In_ EpSelectionDelegate delegate, + _In_opt_ void* delegate_state); + + /** \brief Get the hardware device type. + * + * \param[in] device The OrtHardwareDevice instance to query. + * \return The hardware device type. + * + * \since Version 1.22. + */ + OrtHardwareDeviceType(ORT_API_CALL* HardwareDevice_Type)(_In_ const OrtHardwareDevice* device); + + /** \brief Get the hardware device's vendor identifier. + * + * \param[in] device The OrtHardwareDevice instance to query. + * \return The hardware device vendor identifier. + * + * \since Version 1.22. + */ + uint32_t(ORT_API_CALL* HardwareDevice_VendorId)(_In_ const OrtHardwareDevice* device); + + /** \brief Get the hardware device's vendor name. + * + * \param[in] device The OrtHardwareDevice instance to query. + * \return The hardware device's vendor name. + * + * \since Version 1.22. + */ + const char*(ORT_API_CALL* HardwareDevice_Vendor)(_In_ const OrtHardwareDevice* device); + + /** \brief Get the hardware device's unique identifier. + * + * \param[in] device The OrtHardwareDevice instance to query. + * \return The device id. + * + * \note This is not a unique identifier. It identifies the hardware type when combined with vendor id. + * \since Version 1.22. + */ + uint32_t(ORT_API_CALL* HardwareDevice_DeviceId)(_In_ const OrtHardwareDevice* device); + + /** \brief Get hardware device metadata. + * + * \param[in] device The OrtHardwareDevice instance to query. + * \return An OrtKeyValuePairs instance containing the metadata for the device. + * Note: ORT owns the instance so the user must not call ReleaseKeyValuePairs with it. + * + * \since Version 1.22. + */ + const OrtKeyValuePairs*(ORT_API_CALL* HardwareDevice_Metadata)(_In_ const OrtHardwareDevice* device); + + /** \brief Get the execution provider name. + * + * \param[in] ep_device The OrtEpDevice instance to query. + * \return The execution provider name. + * + * \since Version 1.22. + */ + const char*(ORT_API_CALL* EpDevice_EpName)(_In_ const OrtEpDevice* ep_device); + + /** \brief Get the execution provider's vendor name. + * + * \param[in] ep_device The OrtEpDevice instance to query. + * \return The execution provider's vendor name. + * + * \since Version 1.22. + */ + const char*(ORT_API_CALL* EpDevice_EpVendor)(_In_ const OrtEpDevice* ep_device); + + /** \brief Get the metadata for the OrtEpDevice. + * + * \param[in] ep_device The OrtEpDevice instance to query. + * \return An OrtKeyValuePairs instance containing the metadata for the device. + * + * \since Version 1.22. + */ + const OrtKeyValuePairs*(ORT_API_CALL* EpDevice_EpMetadata)(_In_ const OrtEpDevice* ep_device); + + /** \brief Get the execution provider options for the OrtEpDevice. + * + * \param[in] ep_device The OrtEpDevice instance to query. + * \return An OrtKeyValuePairs instance containing the execution provider options for the device. + * + * \since Version 1.22. + */ + const OrtKeyValuePairs*(ORT_API_CALL* EpDevice_EpOptions)(_In_ const OrtEpDevice* ep_device); + + /** \brief Get the OrtHardwareDevice instance for the OrtEpDevice. + * + * \param[in] ep_device The OrtEpDevice instance to query. + * \return The OrtHardwareDevice instance for the device. + * + * \since Version 1.22. + */ + const OrtHardwareDevice*(ORT_API_CALL* EpDevice_Device)(_In_ const OrtEpDevice* ep_device); + + /** \brief Get the OrtEpApi instance for implementing an execution provider. + * + * \since Version 1.22. + */ + const OrtEpApi*(ORT_API_CALL* GetEpApi)(void); + + /** \brief Compute total size in bytes of the tensor data contained in an OrtValue. + * + * Returns the total number of bytes used to store the tensor data. For numeric tensors, + * this is sizeof(element_type) * total_element_count. OrtValues that are not tensors or + * that are tensors that contain strings will cause an error to be returned. + * + * \param[in] ort_value OrtValue instance containing a tensor + * \param[out] size The total size of the tensor data in bytes + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23 + */ + ORT_API2_STATUS(GetTensorSizeInBytes, _In_ const OrtValue* ort_value, _Out_ size_t* size); + + /** \brief Calls OrtAllocator::GetStats function + * + * Return a pointer to the OrtKeyValuePairs structure that contains the statistics of the allocator + * and the user should call OrtApi::ReleaseKeyValuePairs. + * + * NOTE: If the allocator does not implement this function, the OrtKeyValuePairs instance will be empty. + * + * \param[in] ort_allocator The allocator to get stats from + * \param[out] out A pointer to the OrtKeyValuePairs instance that contains the stats + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(AllocatorGetStats, _In_ const OrtAllocator* ort_allocator, _Outptr_ OrtKeyValuePairs** out); + + /** \brief Create an ::OrtMemoryInfo + * + * \param[in] name Arbitrary name. + * \param[in] device_type Device type. + * \param[in] vendor_id PCI Vendor ID. Use 0 for a generic allocator (e.g. WebGPU). + * \param[in] device_id Device ID if there are multiple devices of the same type. e.g. 2 GPU devices. + * \param[in] mem_type Memory type. Use OrtDeviceMemoryType_DEFAULT for device memory, and + * OrtDeviceMemoryType_HOST_ACCESSIBLE (if applicable) for memory used to transfer between the + * device and the CPU. Use the device_type and device_id of the GPU/NPU that the memory is also + * accessible to. + * \param[in] alignment Alignment of the memory if required. Pass 0 for default alignment. + * \param[in] allocator_type Allocator type. If OrtAllocatorType::OrtArenaAllocator, the ORT arena will be used. + * Caveat: Support for OrtArenaAllocator is currently limited to usage of internal ORT + * allocators via CreateAllocator/CreateAndRegisterAllocator/CreateAndRegisterAllocatorV2. + * \param[out] out Newly created ::OrtMemoryInfo. Must be freed with OrtApi::ReleaseMemoryInfo + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23 + */ + ORT_API2_STATUS(CreateMemoryInfo_V2, _In_ const char* name, _In_ enum OrtMemoryInfoDeviceType device_type, + _In_ uint32_t vendor_id, _In_ int32_t device_id, _In_ enum OrtDeviceMemoryType mem_type, + _In_ size_t alignment, enum OrtAllocatorType allocator_type, + _Outptr_ OrtMemoryInfo** out); + + /** \brief Get the device memory type from ::OrtMemoryInfo + * + * \param[in] ptr The OrtMemoryInfo instance to query. + * \return The device memory type. + * + * \since Version 1.23 + */ + ORT_API_T(OrtDeviceMemoryType, MemoryInfoGetDeviceMemType, _In_ const OrtMemoryInfo* ptr); + + /** \brief Get the vendor id from ::OrtMemoryInfo + * + * \param[in] ptr The OrtMemoryInfo instance to query. + * \return The vendor id. + * + * \since Version 1.23 + */ + ORT_API_T(uint32_t, MemoryInfoGetVendorId, _In_ const OrtMemoryInfo* ptr); + + /// \name OrtValueInfo + /// @{ + + /** \brief Get the OrtNode that produces the value represented by the given OrtValueInfo. + * Optionally returns the associated output index. + * + * \param[in] value_info The OrtValueInfo instance. + * \param[out] producer_node Output parameter set to the OrtNode that produces the OrtValueInfo. + * \param[out] producer_output_index Optional output parameter set to the OrtNode instance's output index + * that produces the value. Ignored if set to NULL. + * \snippet{doc} snippets.dox OrtStatus Return Value + * \since Version 1.23. + */ + ORT_API2_STATUS(ValueInfo_GetValueProducer, _In_ const OrtValueInfo* value_info, + _Outptr_ const OrtNode** producer_node, _Out_opt_ size_t* producer_output_index); + + /** \brief Get the number of consumers of a value as a node input. + * + * Only nodes are considered "consumers" by this function. To check if an OrtValueInfo is a graph output, + * call ValueInfo_IsGraphOutput(). + * + * A single OrtNode may use a single value for more than one input (e.g., Mul(x, x)), so the returned + * `num_consumers` may be larger than the number of unique OrtNode instances that consume the value. + * + * \param[in] value_info The OrtValueInfo instance. + * \param[out] num_consumers Output parameter set to the number of consumers of the value. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(ValueInfo_GetValueNumConsumers, _In_ const OrtValueInfo* value_info, _Out_ size_t* num_consumers); + + /** \brief Returns information (OrtNode and input index) for all consumer nodes that use the value as an input. + * + * Only nodes are considered "consumers" by this function. + * + * Caller provides 2 pre-allocated arrays that will be filled with the OrtNode and input index values. + * Use ValueInfo_GetValueNumConsumers() to get the number of consumers of the value. + * + * An OrtNode instance may appear multiple times if it uses the given value more than once. + * Example: For a node MulNode(x, x) that consumes the value 'x' twice, the following is returned: + * - nodes: [MulNode, MulNode] + * - input_indices: [0, 1] + * + * \param[in] value_info The OrtValueInfo instance. + * \param[out] nodes Pre-allocated array of size `num_consumers` that is filled with OrtNode instances. + * \param[out] input_indices Pre-allocated array of `num_consumers` elements that is filled + * with input indices. Index is set to -1 for an "implicit" input to a consumer node + * that contains a subgraph (e.g., If, Loop) with nodes that use the value internally. + * \param[in] num_consumers The size of the `consumer_nodes` and `consumer_input_indices` arrays. + * Typical usage sets this to the value of ValueInfo_GetValueNumConsumers(). + * An error status is returned if `num_consumers` is less than the number of actual + * consumers. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(ValueInfo_GetValueConsumers, _In_ const OrtValueInfo* value_info, + _Out_writes_all_(num_consumers) const OrtNode** nodes, + _Out_writes_all_(num_consumers) int64_t* input_indices, + _In_ size_t num_consumers); + + /** \brief Get the underlying initializer value, as an OrtValue, from the given OrtValueInfo. + * + * Sets the output parameter to NULL if the given OrtValueInfo does not represent an initializer. + * Does not return an error status in this case. + * + * Supports initializers defined in an outer scope (i.e., a parent graph). + * + * Supports initializers stored in an external file. For external initializers, ORT memory maps + * the initializer data on the first call to this function. If caller needs custom memory mapping, + * use ValueInfo_GetExternalInitializerInfo to get the location of the initializer data. + * + * \param[in] value_info The OrtValueInfo instance. + * \param[out] initializer_value Output parameter set to the initializer value or NULL. Do not cache the OrtValue + * as it is released when the owning OrtGraph is released. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(ValueInfo_GetInitializerValue, _In_ const OrtValueInfo* value_info, + _Outptr_ const OrtValue** initializer_value); + + /** \brief Get information about an external initializer (e.g., filepath, file offset, byte size). + * + * Sets the output parameter `info` to NULL if the given OrtValueInfo does not represent an initializer + * with external data. In this case, a NULL status (non-error) is returned. + * + * \param[in] value_info The OrtValueInfo instance. + * \param[out] info Output parameter set to an OrtExternalInitializerInfo instance that can be used to query + * file path, file offset, etc. ORT sets this to NULL if the OrtValueInfo does not represent + * an external initializer. + * Must release with ReleaseExternalInitializerInfo. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(ValueInfo_GetExternalInitializerInfo, _In_ const OrtValueInfo* value_info, + _Outptr_result_maybenull_ OrtExternalInitializerInfo** info); + + /** \brief Returns a boolean indicating if the given value is a required graph input. + * + * For ONNX IR version < 4, all graph inputs without a matching initializer are required. + * + * For ONNX IR version >=4, a graph input with a matching initializer is an optional graph input + * with the initializer serving as the default value. + * + * \param[in] value_info The OrtValueInfo instance representing the graph value. + * \param[out] is_required_graph_input Output parameter set to true if the graph value is a required graph input. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(ValueInfo_IsRequiredGraphInput, _In_ const OrtValueInfo* value_info, + _Out_ bool* is_required_graph_input); + + /** \brief Returns a boolean indicating if the given value is an optional graph input. + * + * Optional graph inputs were introduced in ONNX IR version 4. For ONNX IR version >=4, a graph input with a + * matching initializer is an optional graph input with the initializer serving as the default value. + * The matching initializer is also known as a non-constant initializer. + * + * \param[in] value_info The OrtValueInfo instance representing the graph value. + * \param[out] is_optional_graph_input Output parameter set to true if the graph value is an optional graph input. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(ValueInfo_IsOptionalGraphInput, _In_ const OrtValueInfo* value_info, + _Out_ bool* is_optional_graph_input); + + /** \brief Returns a boolean indicating if the given value is a graph output. + * + * \param[in] value_info The OrtValueInfo instance representing the graph value. + * \param[out] is_graph_output Output parameter set to true if the graph value is a graph output. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(ValueInfo_IsGraphOutput, _In_ const OrtValueInfo* value_info, _Out_ bool* is_graph_output); + + /** \brief Returns a boolean indicating if the given value is a constant initializer. + * + * For ONNX IR version < 4, all initializers are constant. + * + * For ONNX IR version >=4, an initializer that serves as the default value for a matching graph input is not a + * constant initializer. + * + * \param[in] value_info The OrtValueInfo instance representing the graph value. + * \param[out] is_constant_initializer Output parameter set to true if the graph value is a constant initializer. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(ValueInfo_IsConstantInitializer, _In_ const OrtValueInfo* value_info, + _Out_ bool* is_constant_initializer); + + /** \brief Returns a boolean indicating if the given value is defined in an outer scope. + * + * Certain operator types (e.g., If and Loop) contain nested subgraphs. This function enables + * determining whether a value is defined in a parent node's graph. + * + * \param[in] value_info The OrtValueInfo instance representing the graph value. + * \param[out] is_from_outer_scope Output parameter set to true if the value is defined in an outer + * scope (i.e., a parent graph). + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(ValueInfo_IsFromOuterScope, _In_ const OrtValueInfo* value_info, + _Out_ bool* is_from_outer_scope); + + /// @} + + /// \name OrtGraph + /// @{ + + /** \brief Returns a graph's name. + * + * \param[in] graph The OrtGraph instance. + * \param[out] graph_name Output parameter set to the graph's name. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(Graph_GetName, _In_ const OrtGraph* graph, _Outptr_ const char** graph_name); + + /** \brief Get the filepath to the model from which an OrtGraph is constructed. + * + * \note The model's filepath is empty if the filepath is unknown, such as when the model is loaded from bytes + * via CreateSessionFromArray. + * + * \param[in] graph The OrtGraph instance. + * \param[out] model_path Output parameter set to the model's null-terminated filepath. + * Set to an empty path string if unknown. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(Graph_GetModelPath, _In_ const OrtGraph* graph, _Outptr_ const ORTCHAR_T** model_path); + + /** \brief Returns the ONNX IR version. + * + * \param[in] graph The OrtGraph instance. + * \param[out] onnx_ir_version Output parameter set to the ONNX IR version. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(Graph_GetOnnxIRVersion, _In_ const OrtGraph* graph, _Out_ int64_t* onnx_ir_version); + + /** \brief Returns the number of operator sets that the graph's model uses. + * + * \note An operator set is uniquely identified by the (domain, opset_version) pair. All models must have at + * least one entry that specifies which entry of the ONNX operator set is used. The ONNX domain is represented by + * an empty string. + * + * \param[in] graph The OrtGraph instance. + * \param[out] num_operator_sets Output parameter set to the number of operator sets that the graph's model uses. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(Graph_GetNumOperatorSets, _In_ const OrtGraph* graph, _Out_ size_t* num_operator_sets); + + /** \brief Returns the operator sets that the graph's model uses. + * + * \note An operator set is uniquely identified by the (domain, opset_version) pair. All models must have at + * least one entry that specifies which entry of the ONNX operator set is used. The ONNX domain is represented by + * an empty string. + * + * \param[in] graph The OrtGraph instance. + * \param[out] domains Pre-allocated array of `num_operator_sets` elements that is filled with + * null-terminated domain names. + * \param[out] opset_versions Pre-allocated array of `num_operator_sets` elements that is filled with + * the opset version of the corresponding domain in the `domains` array. + * \param[in] num_operator_sets The size of the `domains` and `opset_versions` arrays. + * Typical usage sets this to the result of Graph_GetNumOperatorSets(). + * An error status is returned if `num_operator_sets` is less than the actual number + * of operator sets. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(Graph_GetOperatorSets, _In_ const OrtGraph* graph, + _Out_writes_(num_operator_sets) const char** domains, + _Out_writes_(num_operator_sets) int64_t* opset_versions, _In_ size_t num_operator_sets); + + /** \brief Returns the number of graph inputs. + * + * \note The count includes initializers that are included in the list of graph inputs. + * + * \param[in] graph The OrtGraph instance. + * \param[out] num_inputs Output parameter set to the number of graph inputs. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(Graph_GetNumInputs, _In_ const OrtGraph* graph, _Out_ size_t* num_inputs); + + /** \brief Returns the graph's inputs as OrtValueInfo instances. + * + * \note The result includes initializers that are included in the list of graph inputs. + * + * \param[in] graph The OrtGraph instance. + * \param[out] inputs Pre-allocated array of `num_inputs` elements that is filled with the graph's inputs. + * \param[in] num_inputs The size of the `inputs` array. + * Typical usage sets this to the result of Graph_GetNumInputs(). An error status is + * returned if `num_inputs` is less than the number of graph inputs. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(Graph_GetInputs, _In_ const OrtGraph* graph, + _Out_writes_(num_inputs) const OrtValueInfo** inputs, _In_ size_t num_inputs); + + /** \brief Returns the number of graph outputs. + * + * \param[in] graph The OrtGraph instance. + * \param[out] num_outputs Output parameter set to the number of graph outputs. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(Graph_GetNumOutputs, _In_ const OrtGraph* graph, _Out_ size_t* num_outputs); + + /** \brief Returns the graph's outputs as OrtValueInfo instances. + * + * \param[in] graph The OrtGraph instance. + * \param[out] outputs Pre-allocated array of `num_outputs` elements that is filled with the graph's outputs. + * \param[in] num_outputs The size of the `outputs` array. + * Typical usage sets this to the result of Graph_GetNumOutputs(). An error status is + * returned if `num_outputs` is less than the number of graph outputs. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(Graph_GetOutputs, _In_ const OrtGraph* graph, + _Out_writes_(num_outputs) const OrtValueInfo** outputs, _In_ size_t num_outputs); + + /** \brief Returns the number of graph initializers. + * + * Counts constant and non-constant initializers. + * + * \param[in] graph The OrtGraph instance. + * \param[out] num_initializers Output parameter set to the number of graph initializers. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(Graph_GetNumInitializers, _In_ const OrtGraph* graph, _Out_ size_t* num_initializers); + + /** \brief Returns the graph's initializers as OrtValueInfo instances. + * + * Includes constant and non-constant initializers. + * + * For ONNX IR version < 4, all initializers are constant. + * + * For ONNX IR version >= 4, an initializer with a name that matches a graph input is considered a + * non-constant initializer. + * + * Call ValueInfo_GetInitializerValue to get the initializer's data. + * + * \param[in] graph The OrtGraph instance. + * \param[out] initializers Pre-allocated array of `num_outputs` elements that is filled with the initializers. + * \param[in] num_initializers The size of the `initializers` array. Typical usage sets this to the + * result of Graph_GetNumInitializers(). An error status is returned if + * `num_initializers` is less than the number of graph initializers. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(Graph_GetInitializers, _In_ const OrtGraph* graph, + _Out_writes_(num_initializers) const OrtValueInfo** initializers, + _In_ size_t num_initializers); + + /** \brief Returns the number of graph nodes. + * + * \param[in] graph The OrtGraph instance. + * \param[out] num_nodes Output parameter set to the number of graph nodes. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(Graph_GetNumNodes, _In_ const OrtGraph* graph, _Out_ size_t* num_nodes); + + /** \brief Returns the graph's nodes as OrtNode instances. + * + * The nodes are sorted using a stable topological ordering. Callers are responsible for maintaining their + * own node ordering if a different order is required. + * + * \param[in] graph The OrtGraph instance. + * \param[out] nodes Pre-allocated array of `num_nodes` elements that is filled with the graph's nodes. + * \param[in] num_nodes The size of the `nodes` array. Typical usage sets this to the + * result of Graph_GetNumNodes(). An error status is returned if + * `num_nodes` is less than the number of graph nodes. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(Graph_GetNodes, _In_ const OrtGraph* graph, + _Out_writes_(num_nodes) const OrtNode** nodes, _In_ size_t num_nodes); + + /** \brief Get the parent node for the given graph, if any exists. + * + * Certain operator types (e.g., If and Loop) contain nested subgraphs. This function enables + * access to the parent node (e.g., the If and Loop node) from a nested subgraph. + * + * \param[in] graph The OrtGraph instance. + * \param[out] node Output parameter that is set to the graph's parent node. + * Set to NULL if a parent node does not exist (e.g., for a top-level graph). + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(Graph_GetParentNode, _In_ const OrtGraph* graph, _Outptr_result_maybenull_ const OrtNode** node); + + /** \brief Returns an OrtGraph that contains a subset of nodes in the source OrtGraph. + * + * \note The lifetime of "dst_graph" is tied to that of "src_graph", as they both internally reference + * the same underlying graph. "dst_graph" preserves the input order of "src_graph", and + * its output order corresponds to the outputs produced by the nodes in "nodes" with the given order. + * + * \param[in] src_graph The source OrtGraph instance. + * \param[in] nodes A subset of the nodes/OrtNodes in 'graph'. + * \param[in] num_nodes Number of nodes. + * \param[out] dst_graph An OrtGraph created from a given set of nodes. Must be released by calling ReleaseGraph. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(Graph_GetGraphView, _In_ const OrtGraph* src_graph, _In_ const OrtNode** nodes, + _In_ size_t num_nodes, _Outptr_ OrtGraph** dst_graph); + + /// @} + + /// \name OrtNode + /// @{ + + /** \brief Returns a node's identifier. + * + * The node's identifier is only unique in the node's parent graph. Different nested subgraphs + * (e.g., subgraphs contained by If and Loop nodes) may reuse identifiers. + * + * \param[in] node The OrtNode instance. + * \param[out] node_id Output parameter set to the node's identifier. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(Node_GetId, _In_ const OrtNode* node, _Out_ size_t* node_id); + + /** \brief Returns a node's name. Can be an empty string. + * + * \param[in] node The OrtNode instance. + * \param[out] node_name Output parameter set to the node's name. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(Node_GetName, _In_ const OrtNode* node, _Outptr_ const char** node_name); + + /** \brief Returns a node's operator type (e.g., "Conv"). + * + * \param[in] node The OrtNode instance. + * \param[out] operator_type Output parameter set to the name of the node's operator type. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(Node_GetOperatorType, _In_ const OrtNode* node, _Outptr_ const char** operator_type); + + /** \brief Returns a node's domain name. + * + * \param[in] node The OrtNode instance. + * \param[out] domain_name Output parameter set to the node's domain name. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(Node_GetDomain, _In_ const OrtNode* node, _Outptr_ const char** domain_name); + + /** \brief Get the opset version in which the given node's operator type was first defined. + * + * \param[in] node The OrtNode instance. + * \param[out] since_version The opset version in which the node's operator type was first defined. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(Node_GetSinceVersion, _In_ const OrtNode* node, _Out_ int* since_version); + + /** \brief Returns the number of node inputs. + * + * \param[in] node The OrtNode instance. + * \param[out] num_inputs Output parameter set to the number of node inputs. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(Node_GetNumInputs, _In_ const OrtNode* node, _Out_ size_t* num_inputs); + + /** \brief Returns the node's inputs as OrtValueInfo instances. + * + * \param[in] node The OrtNode instance. + * \param[out] inputs Pre-allocated array of `num_inputs` elements that is filled with the node's inputs. + * \param[in] num_inputs The size of the `inputs` array. + * Typical usage sets this to the result of Node_GetNumInputs(). An error status is + * returned if `num_inputs` is less than the number of node inputs. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(Node_GetInputs, _In_ const OrtNode* node, + _Out_writes_(num_inputs) const OrtValueInfo** inputs, _In_ size_t num_inputs); + + /** \brief Returns the number of node outputs. + * + * \param[in] node The OrtNode instance. + * \param[out] num_outputs Output parameter set to the number of node outputs. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(Node_GetNumOutputs, _In_ const OrtNode* node, _Out_ size_t* num_outputs); + + /** \brief Returns the node's outputs as OrtValueInfo instances. + * + * \param[in] node The OrtNode instance. + * \param[out] outputs Pre-allocated array of `num_outputs` elements that is filled with the node's outputs. + * \param[in] num_outputs The size of the `outputs` array. + * Typical usage sets this to the result of Node_GetNumOutputs(). An error status is + * returned if `num_outputs` is less than the number of node outputs. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(Node_GetOutputs, _In_ const OrtNode* node, + _Out_writes_(num_outputs) const OrtValueInfo** outputs, _In_ size_t num_outputs); + + /** \brief Returns the number of node implicit inputs. + * + * Certain operator types (e.g., If and Loop) contain nested subgraphs. The internal nodes within the nested subgraphs + * may use values from the outer scope. Those "outer scope" values are considered implicit inputs to the node that + * contains the subgraphs (e.g., the If or Loop node). + * + * \param[in] node The OrtNode instance. + * \param[out] num_implicit_inputs Output parameter set to the number of node implicit inputs. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(Node_GetNumImplicitInputs, _In_ const OrtNode* node, _Out_ size_t* num_implicit_inputs); + + /** \brief Get the implicit inputs, as OrtValueInfo instances, that are used within the given node's subgraphs. + * + * \note Only certain operator types (e.g., If and Loop) contain nested subgraphs. + * The internal nodes within the nested subgraphs may use values from the outer scope. Those "outer scope" values + * are considered implicit inputs to the node that contains the subgraphs (e.g., the If or Loop node). + * + * \param[in] node The OrtNode instance. + * \param[out] implicit_inputs Pre-allocated array of `num_implicit_inputs` elements that is filled the node's + * implicit inputs. + * \param[in] num_implicit_inputs The size of the `implicit_inputs` array. Typical usage sets this to the result + * of Node_GetNumImplicitInputs(). An error status is returned if + * `num_implicit_inputs` is less than the number of node implicit inputs. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(Node_GetImplicitInputs, _In_ const OrtNode* node, + _Out_writes_(num_implicit_inputs) const OrtValueInfo** implicit_inputs, + _In_ size_t num_implicit_inputs); + + /** \brief Returns the number of node attributes. + * + * \param[in] node The OrtNode instance. + * \param[out] num_attributes Output parameter set to the number of node attributes. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(Node_GetNumAttributes, _In_ const OrtNode* node, _Out_ size_t* num_attributes); + + /** \brief Returns a node's attributes as OrtOpAttr instances. + * + * \param[in] node The OrtNode instance. + * \param[out] attributes Pre-allocated array of `num_attributes` elements that is filled with the node's attributes. + * \param[in] num_attributes The size of the `num_attributes` array. + * Typical usage sets this to the result of Node_GetNumAttributes(). An error status is + * returned if `num_attributes` is less than the number of node attributes. + * + * \note ONNX Runtime automatically sets optional (unset) attributes to their default values if the default value + * is a constant expression that does not depend on other tensor/model characteristics. Conv's 'kernel_shape' + * attribute is an example of an optional attribute that does not have a constant default value. This function + * does not provide any unset optional attributes without a constant default value. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(Node_GetAttributes, _In_ const OrtNode* node, + _Out_writes_(num_attributes) const OrtOpAttr** attributes, _In_ size_t num_attributes); + + /** \brief Gets the OrtNode's attribute as OrtOpAttr by name. + * + * \param[in] node The OrtNode instance. + * \param[in] attribute_name The name of the attribute + * \param[out] attribute Output parameter set to the OrtOpAttr instance if an attribute by the given name exists. + * For an unset optional attribute, `attribute` is set to NULL and a non-error status is + * returned. For an invalid attribute name, `attribute` is set to NULL and an error status with + * code ORT_NOT_FOUND is returned. + * + * \note ONNX Runtime automatically sets optional (unset) attributes to their default values if the default value + * is a constant expression that does not depend on other tensor/model characteristics. Conv's 'kernel_shape' + * attribute is an example of an optional attribute that does not have a constant default value. This function + * does not provide any unset optional attributes without a constant default value. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(Node_GetAttributeByName, _In_ const OrtNode* node, _In_ const char* attribute_name, + _Outptr_result_maybenull_ const OrtOpAttr** attribute); + + /** \brief Get the OrtNode's 'TENSOR' attribute as an OrtValue. + * + * \param[in] attribute The OrtOpAttr instance. + * \param[out] attr_tensor If successful, contains the 'TENSOR' attribute as a newly created OrtValue. + Must be freed with OrtApi::ReleaseValue. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(OpAttr_GetTensorAttributeAsOrtValue, _In_ const OrtOpAttr* attribute, + _Outptr_result_maybenull_ OrtValue** attr_tensor); + + /** \brief Get the attribute type as OrtOpAttrType from an OrtOpAttr. + * + * \param[in] attribute The OrtOpAttr instance. + * \param[out] type Output the attribute type as OrtOpAttrType. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(OpAttr_GetType, _In_ const OrtOpAttr* attribute, _Out_ OrtOpAttrType* type); + + /** \brief Get the attribute name from an OrtOpAttr. + * + * \param[in] attribute The OrtOpAttr instance. + * \param[out] name Output parameter set to the attribute's name. The name is a null-terminated string. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(OpAttr_GetName, _In_ const OrtOpAttr* attribute, _Outptr_ const char** name); + + /** \brief Returns the number of subgraphs contained by the given node. + * + * \note Only certain operator types (e.g., If and Loop) contain nested subgraphs. + * + * \param[in] node The OrtNode instance. + * \param[out] num_subgraphs Output parameter set to the number of node subgraphs. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(Node_GetNumSubgraphs, _In_ const OrtNode* node, _Out_ size_t* num_subgraphs); + + /** \brief Get the subgraphs, as OrtGraph instances, contained by the given node. + * + * \note Only certain operator types (e.g., If and Loop) contain nested subgraphs. ONNX nodes store subgraphs in + * their attributes, however, this function must be used to obtain subgraphs from an OrtNode. + * + * \param[in] node The OrtNode instance. + * \param[out] subgraphs Pre-allocated array of `num_subgraphs` elements that is filled with the node's subgraphs. + * \param[in] num_subgraphs The size of the `num_subgraphs` array. + * Typical usage sets this to the result of Node_GetNumSubgraphs(). An error status is + * returned if `num_subgraphs` is less than the number of node subgraphs. + * \param[out] attribute_names Optional pre-allocated array of `num_subgraphs` elements that is filled with the + * attribute names that correspond to the subgraphs. Ignored if set to NULL. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(Node_GetSubgraphs, _In_ const OrtNode* node, + _Out_writes_(num_subgraphs) const OrtGraph** subgraphs, _In_ size_t num_subgraphs, + _Out_writes_opt_(num_subgraphs) const char** attribute_names); + + /** \brief Get the node's parent OrtGraph instance. + * + * Can return NULL if the OrtNode was created without an owning graph. + * In another case, this API may also return NULL if `node` is obtained by calling Graph_GetParentNode() + * on an OrtGraph that is a subgraph of a control-flow op, and the parent graph has not been created yet, + * for example during ORT's GetCapability() when processing the innermost subgraph. + * + * \param[in] node The OrtNode instance. + * \param[out] graph Output parameter set to the node's OrtGraph. Can be set to NULL + * if the node is not currently contained by a graph. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(Node_GetGraph, _In_ const OrtNode* node, _Outptr_result_maybenull_ const OrtGraph** graph); + + /** \brief Returns the execution provider name that this node is assigned to run on. + * Returns NULL if the node has not been assigned to any execution provider yet. + * For plugin execution providers, the name is the one returned by OrtEp::GetName. + * + * \param[in] node The OrtNode instance. + * \param[out] out Output execution provider type and can be NULL if node has not been assigned. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(Node_GetEpName, _In_ const OrtNode* node, _Outptr_result_maybenull_ const char** out); + + /// @} + + /// \name OrtExternalInitializerInfo + /// @{ + + /** \brief Release an OrtExternalInitializerInfo instance. + * + * \param[in] input OrtExternalInitializerInfo instance to be released. + * + * \since Version 1.23. + */ + ORT_CLASS_RELEASE(ExternalInitializerInfo); + + /** \brief Get the relative path to the file that stores the initializer's data. + * + * \note The path is relative to the filesystem directory where the ONNX model was stored. + * Caller can use Graph_GetModelPath to get the model's full path and construct the absolute path to the + * external initializer file if necessary. + * + * \param[in] info The OrtExternalInitializerInfo instance. + * \return The relative path to the file that stores the initializer's data. Do NOT free this pointer. + * + * \since Version 1.23. + */ + ORT_API_T(const ORTCHAR_T*, ExternalInitializerInfo_GetFilePath, _In_ const OrtExternalInitializerInfo* info); + + /** \brief Get the byte offset within the file where the initializer's data is stored. + * + * \param[in] info The OrtExternalInitializerInfo instance. + * \return The byte offset where the initializer's data is stored within the file. + * + * \since Version 1.23. + */ + ORT_API_T(int64_t, ExternalInitializerInfo_GetFileOffset, _In_ const OrtExternalInitializerInfo* info); + + /** \brief Get the size in bytes of the initializer's data within the file. + * + * \param[in] info The OrtExternalInitializerInfo instance. + * \return The size in bytes of the initializer's data within the file. + * + * \since Version 1.23. + */ + ORT_API_T(size_t, ExternalInitializerInfo_GetByteSize, _In_ const OrtExternalInitializerInfo* info); + + /// @} + + /// \name OrtRunOptions + /// @{ + + /** \brief Get a run configuration entry. + * + * If a run configuration entry with key `config_key` doesn't exist, `config_value` will be set to NULL. + * + * `config_key`s are defined in onnxruntime_run_options_config_keys.h. + * + * \param[in] options The OrtRunOptions instance. + * \param[in] config_key The configuration entry key. A null-terminated string. + * \return The configuration entry value. Either a null-terminated string if the entry was found. nullptr otherwise. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23 + */ + ORT_API_T(const char*, GetRunConfigEntry, _In_ const OrtRunOptions* options, + _In_z_ const char* config_key); + + /// @} + + /** \brief Get the OrtMemoryInfo for the device. + * + * \param[in] ep_device The OrtEpDevice instance to query. + * \param[in] memory_type The memory type to return. + * \return A pointer to the OrtMemoryInfo for the device. This may be nullptr if not set. + * If memory_type is OrtDeviceMemoryType_DEFAULT and nullptr is returned the EP uses CPU memory. + * + * \since Version 1.23 + */ + ORT_API_T(const OrtMemoryInfo*, EpDevice_MemoryInfo, _In_ const OrtEpDevice* ep_device, + _In_ OrtDeviceMemoryType memory_type); + + /** \brief Create/replace a shared allocator for the OrtEpDevice in the OrtEnv. + * + * OrtEpDevice maps to the EP factory, and the factory provides the allocator implementation. + * + * Both OrtDeviceMemoryType_DEFAULT and OrtDeviceMemoryType_HOST_ACCESSIBLE are optional for an EP to provide. + * It is EP implementation dependent as to what is available. + * + * If a shared allocator already exists for the OrtEpDevice and OrtDeviceMemoryType, it is replaced. This allows + * changing the shared allocator configuration from the default. e.g. adding an arena. + * + * \param[in] env The OrtEnv instance to create the shared allocator in. + * \param[in] ep_device The OrtEpDevice instance to create the shared allocator for. + * \param[in] mem_type The memory type to use for the shared allocator. + * \param[in] allocator_type The type of allocator to create. Only OrtDeviceAllocator is valid currently. + * \param[in] allocator_options Optional key-value pairs to configure the allocator. If arena based, see + * include/onnxruntime/core/framework/allocator.h for the keys and values that can be + * used. + * \param[out] allocator A pointer to the created shared allocator. Owned by the OrtEnv instance. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23 + */ + ORT_API2_STATUS(CreateSharedAllocator, _In_ OrtEnv* env, _In_ const OrtEpDevice* ep_device, + _In_ OrtDeviceMemoryType mem_type, _In_ OrtAllocatorType allocator_type, + _In_opt_ const OrtKeyValuePairs* allocator_options, + _Outptr_opt_ OrtAllocator** allocator); + + /** \brief Get a shared allocator from the OrtEnv. + * + * By default there is a shared allocator created for all OrtEpDevice instances, so if you get the OrtMemoryInfo + * from the OrtEpDevice using EpDevice_MemoryInfo a shared allocator is guaranteed to exist. + * + * This will also match and return custom allocators added with RegisterAllocator. + * + * It is not an error to not find a matching allocator. + * + * \param[in] env The OrtEnv instance to get the shared allocator from. + * \param[in] mem_info The OrtMemoryInfo instance to get the shared allocator for. + * \param[out] allocator A pointer to the shared allocator, or nullptr if no shared allocator exists for + * the given memory info. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23 + */ + ORT_API2_STATUS(GetSharedAllocator, _In_ OrtEnv* env, _In_ const OrtMemoryInfo* mem_info, + _Outptr_result_maybenull_ OrtAllocator** allocator); + + /** \brief Release a shared allocator from the OrtEnv for the OrtEpDevice and memory type. + * + * This will release the shared allocator for the given OrtEpDevice and memory type. + * If no shared allocator exists, this is a no-op. + * + * \param[in] env The OrtEnv instance to release the shared allocator from. + * \param[in] ep_device The OrtEpDevice instance to release the shared allocator for. + * \param[in] mem_type The memory type of the shared allocator to release. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23 + */ + ORT_API2_STATUS(ReleaseSharedAllocator, _In_ OrtEnv* env, _In_ const OrtEpDevice* ep_device, + _In_ OrtDeviceMemoryType mem_type); + + /** \brief Get a const pointer to the raw data inside a tensor + * + * Used to read the internal tensor data directly. + * \note The returned pointer is valid until the OrtValue is destroyed. + * + * \param[in] value A tensor type (string tensors are not supported) + * \param[out] out Filled in with a pointer to the internal storage + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23 + */ + ORT_API2_STATUS(GetTensorData, _In_ const OrtValue* value, _Outptr_ const void** out); + + /** \brief Get Session configuration entries. + * + * \param[in] options The session options. + * \param[out] out A pointer to a newly created OrtKeyValuePairs instance. + * + * An OrtKeyValuePairs instance containing all session configuration entries. + * Note: the user should call OrtApi::ReleaseKeyValuePairs. + * + * \since Version 1.23. + */ + ORT_API2_STATUS(GetSessionOptionsConfigEntries, _In_ const OrtSessionOptions* options, _Outptr_ OrtKeyValuePairs** out); + + /** \brief Get the OrtMemoryInfo for each input of the session. + * + * The memory info can be used to determine where the input tensors are required. + * + * The session must be fully initialized before calling this function as the input locations are not known until + * this has occurred. + * + * \param[in] session The OrtSession instance. + * \param[out] inputs_memory_info Pre-allocated array of size `num_inputs` that will be filled with the + * OrtMemoryInfo* value for each input. + * The order is the same as returned by SessionGetInputName. + * \param[in] num_inputs The number of inputs in the session. Must match SessionGetInputCount. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23 + */ + ORT_API2_STATUS(SessionGetMemoryInfoForInputs, _In_ const OrtSession* session, + _Out_writes_(num_inputs) const OrtMemoryInfo** inputs_memory_info, + _In_ size_t num_inputs); + + /** \brief Get the OrtMemoryInfo for each output of the session. + * + * The memory info can be used to determine the device the output tensors are produced on. + * The user can pre-allocate an OrtValue using this information or use IOBinding to keep the data on the device. + * ORT will copy the output to CPU otherwise. + * + * The session must be fully initialized before calling this function as the output locations are not known until + * this has occurred. + * + * \param[in] session The OrtSession instance. + * \param[out] outputs_memory_info Pre-allocated array of size `num_outputs` that will be filled with + * OrtMemoryInfo* values for each output. + * The order is the same as returned by SessionGetOutputName. + * \param[in] num_outputs The number of outputs in the session. Must match SessionGetOutputCount. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23 + */ + ORT_API2_STATUS(SessionGetMemoryInfoForOutputs, _In_ const OrtSession* session, + _Out_writes_(num_outputs) const OrtMemoryInfo** outputs_memory_info, + _In_ size_t num_outputs); + + /** \brief Get the OrtEpDevice (if available) for each input of the session. + * + * An OrtEpDevice will be available if auto EP selection is enabled by calling + * SessionOptionsSetEpSelectionPolicy or SessionOptionsSetEpSelectionPolicyDelegate, + * or if the OrtEpDevice was manually added to the session using SessionOptionsAppendExecutionProvider_V2. + * + * If an OrtEpDevice is not available for the input a nullptr is returned. + * + * The returned OrtEpDevice can be used to create an OrtSyncStream via CreateSyncStreamForEpDevice to asynchronously + * provide input to the inference session Run. + * + * The session must be fully initialized before calling this function as the assigned EPs are not known until + * this has occurred. + * + * \param[in] session The OrtSession instance. + * \param[out] inputs_ep_devices Pre-allocated array of size `num_inputs` that will be filled with + * OrtEpDevice* values for each input. + * The order is the same as returned by SessionGetInputName. + * \param[in] num_inputs The number of inputs in the session. Must match SessionGetInputCount. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23 + */ + ORT_API2_STATUS(SessionGetEpDeviceForInputs, _In_ const OrtSession* session, + _Out_writes_(num_inputs) const OrtEpDevice** inputs_ep_devices, + _In_ size_t num_inputs); + + /** \brief Create an OrtSyncStream for the given OrtEpDevice. + * + * The OrtSyncStream can be used to enable asynchronous operations. + * e.g. async usage of CopyTensors to provide input to an OrtSession Run call. + * + * An error code of ORT_NOT_IMPLEMENTED will be returned if the EP does not support OrtSyncStream. + * + * \param[in] ep_device The OrtEpDevice instance to create the sync stream for. + * \param[in] stream_options Options for OrtSyncStream creation. May be nullptr. + * \param[out] stream Output parameter set to the created OrtSyncStream instance. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23 + */ + ORT_API2_STATUS(CreateSyncStreamForEpDevice, _In_ const OrtEpDevice* ep_device, + _In_opt_ const OrtKeyValuePairs* stream_options, + _Outptr_ OrtSyncStream** stream); + + /** \brief Get the native handle of the sync stream. + * + * This returns the native handle for the stream. e.g. cudaStream_t for CUDA streams. + * + * \param[in] stream The OrtSyncStream instance to get the handle from. + * + * \returns The native handle of the stream. + * + * \since Version 1.23 + */ + ORT_API_T(void*, SyncStream_GetHandle, _In_ OrtSyncStream* stream); + + ORT_CLASS_RELEASE(SyncStream); + + /** \brief Copy OrtValue instances containing Tensors between devices. + * + * The overall copy must be between a single source device and a single destination device. i.e. + * - all src_tensors must have matching OrtMemoryInfo, + * - all dst_tensors must have matching OrtMemoryInfo. + * + * OrtValue instances can be created by: + * - Use GetSharedAllocator to get the shared allocator for the OrtMemoryInfo if you need to allocate memory + * on the device. + * - Use CreateTensorAsOrtValue, CreateTensorWithDataAsOrtValue or CreateTensorWithDataAndDeleterAsOrtValue + * to create an OrtValue containing a tensor depending on whether you have existing data or not, and whether + * you want ORT to free the existing data once it is done with the OrtValue. + * + * \param[in] env The OrtEnv instance to use. The data transfer implementation is provided by an execution provider + * that is registered in this OrtEnv. + * \param[in] src_tensors Array of OrtValue instances containing the source tensors to copy. + * \param[in] dst_tensors Array of OrtValue instances to copy the source tensors to. + * \param[in] stream Optional OrtSyncStream that can be used to perform the copy asynchronously. May be nullptr. + * \param[in] num_tensors The number of tensors to copy. The size of `src_tensors` and `dst_tensors` must match. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23 + */ + ORT_API2_STATUS(CopyTensors, _In_ const OrtEnv* env, + _In_reads_(num_tensors) const OrtValue* const* src_tensors, + _In_reads_(num_tensors) OrtValue* const* dst_tensors, + _In_opt_ OrtSyncStream* stream, + _In_ size_t num_tensors); + + /** \brief Get ::OrtModelMetadata from an ::OrtGraph + * + * \param[in] graph The OrtGraph instance. + * \param[out] out Newly created ::OrtModelMetadata. Must be freed using OrtApi::ReleaseModelMetadata. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(Graph_GetModelMetadata, _In_ const OrtGraph* graph, _Outptr_ OrtModelMetadata** out); + + /** \brief Validate a compiled model's compatibility information for one or more EP devices. + * + * \param[in] ep_devices The EP devices to validate against (e.g., from GetEpDevices). + * All devices must belong to the same execution provider. + * \param[in] num_ep_devices The number of EP devices provided. + * \param[in] compatibility_info The compatibility info string produced when the model was compiled. + * \param[out] out_status The resulting compatibility status for the EP devices. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(GetModelCompatibilityForEpDevices, + _In_reads_(num_ep_devices) const OrtEpDevice* const* ep_devices, + _In_ size_t num_ep_devices, + _In_ const char* compatibility_info, + _Out_ OrtCompiledModelCompatibility* out_status); + + /// \name OrtExternalInitializerInfo + /// @{ + + /** \brief Creates an OrtExternalInitializerInfo instance. + * + * \param[in] filepath The relative path to the file that stores the initializer's data. ORT copies this path string. + * \param[in] file_offset The byte offset where the initializer's data is stored within the file. + * \param[in] byte_size The size in bytes of the initializer's data within the file. + * \param[out] out Output parameter set to the new OrtExternalInitializerInfo instance. + * Must be released by calling ReleaseExternalInitializerInfo(). + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(CreateExternalInitializerInfo, _In_ const ORTCHAR_T* filepath, _In_ int64_t file_offset, + _In_ size_t byte_size, _Outptr_ OrtExternalInitializerInfo** out); + + /// @} + /** \brief Fetch whether the tensor has shape information. + * \param[in] info The OrtTensorTypeAndShapeInfo instance. + * \return true if the tensor has shape information, false otherwise. + * + * \since Version 1.24 + */ + ORT_API_T(bool, TensorTypeAndShape_HasShape, _In_ const OrtTensorTypeAndShapeInfo* info); + + /** \brief Get all config entries from ::OrtKernelInfo. + * + * Gets all configuration entries from the ::OrtKernelInfo object as key-value pairs. + * Config entries are set on the ::OrtSessionOptions and are accessible in custom operator kernels. + * + * Used in the CreateKernel callback of an OrtCustomOp to access all session configuration entries + * during kernel construction. + * + * \param[in] info An instance of ::OrtKernelInfo. + * \param[out] out A pointer to a newly created OrtKeyValuePairs instance containing all config entries. + * Note: the user should call OrtApi::ReleaseKeyValuePairs. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * \since Version 1.24 + */ + ORT_API2_STATUS(KernelInfo_GetConfigEntries, _In_ const OrtKernelInfo* info, _Outptr_ OrtKeyValuePairs** out); + + /** \brief Get the graph node's operator domain from ::OrtKernelInfo. + * + * If `out` is nullptr, the value of `size` is set to the size of the operator domain + * string (including null-terminator), and a success status is returned. + * + * If the `size` parameter is greater than or equal to the string's size and `out` is not nullptr, + * the value of `size` is set to the true size of the string (including null-terminator), + * the provided memory is filled with the string's contents, and a success status is returned. + * + * If the `size` parameter is less than the actual string's size and `out` + * is not nullptr, the value of `size` is set to the true size of the string + * and a failure status with error code ORT_INVALID_ARGUMENT is returned. + * + * \param[in] info An instance of ::OrtKernelInfo. + * \param[out] out Memory location into which to write the UTF-8 null-terminated string representing the + * operator domain. + * \param[in,out] size Pointer to the size of the `out` buffer. See above comments for details. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * \since Version 1.24 + */ + ORT_API2_STATUS(KernelInfo_GetOperatorDomain, _In_ const OrtKernelInfo* info, _Out_opt_ char* out, + _Inout_ size_t* size); + + /** \brief Get the graph node's operator type from ::OrtKernelInfo. + * + * If `out` is nullptr, the value of `size` is set to the size of the operator type + * string (including null-terminator), and a success status is returned. + * + * If the `size` parameter is greater than or equal to the string's size and `out` is not nullptr, + * the value of `size` is set to the true size of the string (including null-terminator), + * the provided memory is filled with the string's contents, and a success status is returned. + * + * If the `size` parameter is less than the actual string's size and `out` + * is not nullptr, the value of `size` is set to the true size of the string + * and a failure status with error code ORT_INVALID_ARGUMENT is returned. + * + * \param[in] info An instance of ::OrtKernelInfo. + * \param[out] out Memory location into which to write the UTF-8 null-terminated string representing the + * operator type. + * \param[in,out] size Pointer to the size of the `out` buffer. See above comments for details. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * \since Version 1.24 + */ + ORT_API2_STATUS(KernelInfo_GetOperatorType, _In_ const OrtKernelInfo* info, _Out_opt_ char* out, + _Inout_ size_t* size); + + /** \brief Get the opset version in which the given node's operator type was first defined from ::OrtKernelInfo. + * + * \param[in] info An instance of ::OrtKernelInfo. + * \param[out] since_version The opset version in which the node's operator type was first defined. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * \since Version 1.24 + */ + ORT_API2_STATUS(KernelInfo_GetOperatorSinceVersion, _In_ const OrtKernelInfo* info, + _Out_ int* since_version); + + /** \brief Get the EP Interop API instance. + * + * Get the Interop API instance to work with external resources. This API provides functions + * for importing external GPU memory and semaphores for zero-copy sharing between ORT inference + * and other GPU workloads. + * + * \return Interop API struct instance. + * + * \since Version 1.24. + */ + const OrtInteropApi*(ORT_API_CALL* GetInteropApi)(void); + + /** \brief Get the EP device assigned to each session output. + * + * Returns the OrtEpDevice assigned to each output of the session after graph partitioning. + * This allows validation that outputs are placed on the expected device for external resource sharing. + * + * The EP device for each output is determined by which execution provider will produce that output + * during inferencing. This information is useful for: + * - Validating that outputs will be placed on the expected device for external resource sharing + * - Deciding whether to use external memory handles for outputs + * + * \param[in] session The OrtSession instance to query. + * \param[out] outputs_ep_devices An array to be filled with the EP device for each output. + * The array must be allocated by the caller with space for + * OrtEpDevice* values for each output. + * The order is the same as returned by SessionGetOutputName. + * \param[in] num_outputs The number of outputs in the session. Must match SessionGetOutputCount. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24 + */ + ORT_API2_STATUS(SessionGetEpDeviceForOutputs, _In_ const OrtSession* session, + _Out_writes_(num_outputs) const OrtEpDevice** outputs_ep_devices, + _In_ size_t num_outputs); + /** \brief Get the number of available hardware devices. + * + * Returns the count of hardware devices discovered on the system. + * Use this to allocate an array before calling GetHardwareDevices(). + * + * \param[in] env The OrtEnv instance where device discovery results are stored. + * \param[out] num_devices The number of OrtHardwareDevice instances available. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24. + */ + ORT_API2_STATUS(GetNumHardwareDevices, _In_ const OrtEnv* env, _Out_ size_t* num_devices); + + /** \brief Get the list of available hardware devices. + * + * Enumerates hardware devices available on the system. + * Populates a user-provided array with pointers to OrtHardwareDevice instances. The caller is responsible + * for allocating the array with sufficient space (use GetNumHardwareDevices() to get the count). + * + * The returned pointers reference internal ORT data structures that are discovered once at process + * startup and remain valid for the lifetime of the OrtEnv. The caller does not need to release these + * pointers, but should not use them after calling ReleaseEnv(). + * + * \param[in] env The OrtEnv instance where device discovery results are stored. + * \param[out] devices User-allocated array to receive pointers to OrtHardwareDevice instances. + * The array must have space for at least num_devices elements. + * \param[in] num_devices The size of the user-allocated devices array. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24. + */ + ORT_API2_STATUS(GetHardwareDevices, _In_ const OrtEnv* env, + _Out_writes_(num_devices) const OrtHardwareDevice** devices, + _In_ size_t num_devices); + + /** \brief Check for known incompatibility issues between hardware device and a specific execution provider. + * + * This function checks for known incompatibility issues between the specified hardware device + * and a specific execution provider. + * If returned incompatibility details have non-zero reasons, it indicates the device is not compatible. + * However, if returned detail have reason == 0, it doesn't guarantee 100% compatibility for all models, + * as models may have specific requirements. + * + * Note: This method should only be called when the OrtEnv has been initialized with execution + * providers (after RegisterExecutionProviderLibrary is called). + * + * \param[in] env The OrtEnv instance with registered execution providers. + * \param[in] ep_name The name of the execution provider to check. Required and cannot be null or empty. + * \param[in] hw The hardware device to check for incompatibility. + * \param[out] details Compatibility details including reasons for incompatibility if any. + * Must be freed with OrtApi::ReleaseDeviceEpIncompatibilityDetails. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24. + */ + ORT_API2_STATUS(GetHardwareDeviceEpIncompatibilityDetails, _In_ const OrtEnv* env, + _In_ const char* ep_name, + _In_ const OrtHardwareDevice* hw, + _Outptr_ OrtDeviceEpIncompatibilityDetails** details); + + /// \name OrtDeviceEpIncompatibilityDetails + /// Accessor functions for device incompatibility details + /// @{ + + /** \brief Get the incompatibility reasons bitmask from OrtDeviceEpIncompatibilityDetails. + * + * \param[in] details The OrtDeviceEpIncompatibilityDetails instance to query. + * \param[out] reasons_bitmask Pointer to store the bitmask of incompatibility reasons. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24. + */ + ORT_API2_STATUS(DeviceEpIncompatibilityDetails_GetReasonsBitmask, + _In_ const OrtDeviceEpIncompatibilityDetails* details, + _Out_ uint32_t* reasons_bitmask); + + /** \brief Get the notes from OrtDeviceEpIncompatibilityDetails. + * + * \param[in] details The OrtDeviceEpIncompatibilityDetails instance to query. + * \param[out] notes Pointer to the notes string. May be nullptr if no notes are available. + * The returned string is owned by the details object and should not be freed. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24. + */ + ORT_API2_STATUS(DeviceEpIncompatibilityDetails_GetNotes, + _In_ const OrtDeviceEpIncompatibilityDetails* details, + _Outptr_result_maybenull_ const char** notes); + + /** \brief Get the execution provider error code from OrtDeviceEpIncompatibilityDetails. + * + * This allows Independent Hardware Vendors (IHVs) to define their own error codes + * to provide additional details about device incompatibility. + * + * \param[in] details The OrtDeviceEpIncompatibilityDetails instance to query. + * \param[out] error_code Pointer to store the EP-specific error code. A value of 0 indicates no error code was set. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24. + */ + ORT_API2_STATUS(DeviceEpIncompatibilityDetails_GetErrorCode, + _In_ const OrtDeviceEpIncompatibilityDetails* details, + _Out_ int32_t* error_code); + + /** \brief Release an OrtDeviceEpIncompatibilityDetails instance. + * + * \since Version 1.24. + */ + ORT_CLASS_RELEASE(DeviceEpIncompatibilityDetails); + + /// @} + + /// \name Model Compatibility APIs + /// @{ + + /** \brief Extract EP compatibility info from a precompiled model file. + * + * Parses the model file to extract the compatibility info string for a specific execution provider + * from the model's metadata properties. This is only applicable to models that have been precompiled + * for an EP (e.g., via OrtCompileApi). Standard ONNX models do not contain this information. + * + * The compatibility info string must be valid UTF-8 without embedded NUL characters. + * + * \note This API performs standalone model parsing, separate from session creation. This means + * the protobuf parsing cost is incurred here and again during session creation. It is intended + * for scenarios where applications need to check compatibility before deciding whether to proceed + * with session creation, such as providing early user feedback. + * + * \note This operation parses the full ONNX ModelProto from disk. For very large models, consider + * using GetCompatibilityInfoFromModelBytes with a pre-loaded buffer if the model is already in memory. + * + * The compatibility info can then be passed to GetModelCompatibilityForEpDevices to check if a + * precompiled model is compatible with the current system. + * + * \param[in] model_path Path to the ONNX model file. + * \param[in] ep_type The execution provider type string. Must be non-empty. + * Use OrtApi::EpDevice_EpName to get this value from an OrtEpDevice. + * \param[in] allocator Allocator to use for the output string. Use OrtApi::GetAllocatorWithDefaultOptions. + * \param[out] compatibility_info Output pointer to the compatibility info string. + * Returns nullptr if no compatibility info exists for the specified EP. + * Caller must free with OrtApi::AllocatorFree when non-null. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24. + */ + ORT_API2_STATUS(GetCompatibilityInfoFromModel, + _In_ const ORTCHAR_T* model_path, + _In_ const char* ep_type, + _Inout_ OrtAllocator* allocator, + _Outptr_result_maybenull_ char** compatibility_info); + + /** \brief Extract EP compatibility info from precompiled model bytes in memory. + * + * Same as GetCompatibilityInfoFromModel but reads from a memory buffer instead of a file. + * Useful when precompiled models are loaded from encrypted storage, network, or other non-file sources. + * + * \note This API performs standalone model parsing, separate from session creation. This means + * the protobuf parsing cost is incurred here and again during session creation. It is intended + * for scenarios where applications need to check compatibility before deciding whether to proceed + * with session creation, such as providing early user feedback. + * + * \param[in] model_data Pointer to the model data in memory. + * \param[in] model_data_length Size of the model data in bytes. + * \param[in] ep_type The execution provider type string. Must be non-empty. + * \param[in] allocator Allocator to use for the output string. Use OrtApi::GetAllocatorWithDefaultOptions. + * \param[out] compatibility_info Output pointer to the compatibility info string. + * Returns nullptr if no compatibility info exists for the specified EP. + * Caller must free with OrtApi::AllocatorFree when non-null. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24. + */ + ORT_API2_STATUS(GetCompatibilityInfoFromModelBytes, + _In_reads_(model_data_length) const void* model_data, + _In_ size_t model_data_length, + _In_ const char* ep_type, + _Inout_ OrtAllocator* allocator, + _Outptr_result_maybenull_ char** compatibility_info); + + /// @} + + /** \brief Create an OrtEnv instance with the given options. + * + * \note Invoking this function will return the same instance of the environment as that returned by a previous call + * to another env creation function; all arguments to this function will be ignored. + * + * \param[in] options The OrtEnvCreationOptions instance that contains creation options. + * \param[out] out Output parameter set to the new OrtEnv instance. Must be freed with OrtApi::ReleaseEnv. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24 + */ + ORT_API2_STATUS(CreateEnvWithOptions, _In_ const OrtEnvCreationOptions* options, _Outptr_ OrtEnv** out); + + /** \brief Get information about the subgraphs assigned to each execution provider (EP) and the nodes within. + * + * Each returned OrtEpAssignedSubgraph instance contains details of the subgraph/nodes assigned to an execution + * provider, including the execution provider's name, and the name, domain, and operator type for every node. + * + * For compiling execution providers, a single OrtEpAssignedSubgraph instance contains information about the + * nodes that are fused and compiled within a single subgraph assigned to the execution provider. + * + * For execution providers that use kernel registration (e.g., CPU EP), each node with a registered kernel is + * contained in its own OrtEpAssignedSubgraph instance. + * + * \note The caller must enable the collection of this information by enabling the session + * configuration entry "session.record_ep_graph_assignment_info" during session creation. + * Refer to onnxruntime_session_options_config_keys.h. Otherwise, if not enabled, this function returns a + * status with error code ORT_FAIL. + * + * \note The information reported by this function is obtained immediately after running basic optimizations on the + * original graph if the session optimization level is set to ORT_ENABLE_BASIC or higher. If the session + * optimization level is set to ORT_DISABLE_ALL, only minimal/required optimizations are run before + * the information is collected. + * + * \param[in] session The OrtSession instance. + * \param[out] ep_subgraphs Output parameter set to the array of OrtEpAssignedSubgraph instances. + * \param[out] num_ep_subgraphs Output parameter set to the number of elements in the `ep_subgraphs` array. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24. + */ + ORT_API2_STATUS(Session_GetEpGraphAssignmentInfo, _In_ const OrtSession* session, + _Outptr_ const OrtEpAssignedSubgraph* const** ep_subgraphs, + _Out_ size_t* num_ep_subgraphs); + + /** \brief Get the name of the execution provider to which the subgraph was assigned. + * + * \param[in] ep_subgraph The OrtEpAssignedSubgraph instance. + * \param[out] out Output parameter set to the execution provider's name as a UTF-8 null-terminated string. + * Owned by the OrtEpAssignedSubgraph instance (do not free). + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24. + */ + ORT_API2_STATUS(EpAssignedSubgraph_GetEpName, _In_ const OrtEpAssignedSubgraph* ep_subgraph, + _Outptr_ const char** out); + + /** \brief Get the nodes in a subgraph assigned to a specific execution provider. + * + * \param[in] ep_subgraph The OrtEpAssignedSubgraph instance. + * \param[out] ep_nodes Output parameter set to the array of OrtEpAssignedNode instances. + * \param[out] num_ep_nodes Output parameter set to the number of OrtEpAssignedNode instance returned. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24. + */ + ORT_API2_STATUS(EpAssignedSubgraph_GetNodes, _In_ const OrtEpAssignedSubgraph* ep_subgraph, + _Outptr_ const OrtEpAssignedNode* const** ep_nodes, _Out_ size_t* num_ep_nodes); + + /** \brief Get the name of the node assigned to an execution provider. + * + * \param[in] ep_node The OrtEpAssignedNode instance. + * \param[out] out Output parameter set to the node's name as a UTF-8 null-terminated string. + * Owned by the OrtEpAssignedNode instance (do not free). + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24. + */ + ORT_API2_STATUS(EpAssignedNode_GetName, _In_ const OrtEpAssignedNode* ep_node, _Outptr_ const char** out); + + /** \brief Get the domain of the node assigned to an execution provider. + * + * \param[in] ep_node The OrtEpAssignedNode instance. + * \param[out] out Output parameter set to the node's domain as a UTF-8 null-terminated string. + * Owned by the OrtEpAssignedNode instance (do not free). + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24. + */ + ORT_API2_STATUS(EpAssignedNode_GetDomain, _In_ const OrtEpAssignedNode* ep_node, _Outptr_ const char** out); + + /** \brief Get the operator type of the node assigned to an execution provider. + * + * \param[in] ep_node The OrtEpAssignedNode instance. + * \param[out] out Output parameter set to the node's operator type as a UTF-8 null-terminated string. + * Owned by the OrtEpAssignedNode instance (do not free). + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24. + */ + ORT_API2_STATUS(EpAssignedNode_GetOperatorType, _In_ const OrtEpAssignedNode* ep_node, _Outptr_ const char** out); + + /** \brief Sets OrtSyncStream for the run options + * + * OrtSyncStream is used to synchronize the execution of the model run for the device + * of the stream. It overrides the existing stream for the duration of the Run(). + * The stream instance must be alive for the duration of the Run() call. + * + * \param[in] options + * \param[in] sync_stream The synchronization stream. Pass nullptr to clear previous setting. + * + * \since 1.24 + */ + ORT_API_T(void, RunOptionsSetSyncStream, _Inout_ OrtRunOptions* options, _In_ OrtSyncStream* sync_stream); + + /** \brief Get the element data type and shape for an OrtValue that represents a Tensor (scalar, dense, or sparse). + * + * \note This function is an alternative to OrtApi::GetTensorTypeAndShape that does not allocate a new array for + * the shape data. The OrtValue instance's internal shape data is returned directly. + * + * \note Returns an error if the underlying OrtValue is not a Tensor. + * + * \param[in] value The OrtValue instance. + * \param[out] elem_type Output parameter set to the tensor element data type. + * \param[out] shape_data Output parameter set to the OrtValue instance's internal shape data array. + * For a scalar, `shape_data` is NULL and `shape_data_count` is 0. + * Must not be released as it is owned by the OrtValue instance. This pointer becomes invalid + * when the OrtValue is released or if the underlying shape data is updated or reallocated. + * \param[out] shape_data_count Output parameter set to the number of elements in `shape_data`. + * `shape_data_count` is 0 for a scalar. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24. + */ + ORT_API2_STATUS(GetTensorElementTypeAndShapeDataReference, _In_ const OrtValue* value, + _Out_ ONNXTensorElementDataType* elem_type, + _Outptr_result_maybenull_ const int64_t** shape_data, + _Out_ size_t* shape_data_count); + + /** \brief Enable profiling for this run + * + * \param[in] options + * \param[in] profile_file_prefix The prefix for the profile file. The actual filename will be: + * [profile_file_prefix]_[timestamp].json + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.25. + */ + ORT_API2_STATUS(RunOptionsEnableProfiling, _Inout_ OrtRunOptions* options, _In_ const ORTCHAR_T* profile_file_prefix); + + /** \brief Disable profiling for this run + * + * \param[in] options + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.25. + */ + ORT_API2_STATUS(RunOptionsDisableProfiling, _Inout_ OrtRunOptions* options); + + /** \brief Fetch an array of strings stored as an attribute in the graph node + * + * If `out` is nullptr, the value of `size` is set to the true size of the attribute + * array and a success status is returned. + * + * Otherwise, the strings and pointer array are allocated using `allocator`. + * The caller must free each string and the pointer array with `allocator`. + * If the attribute array is empty, `*out` is set to nullptr and `*size` is set to 0. + * + * \param[in] info instance + * \param[in] name name of the attribute to be parsed + * \param[in] allocator allocator used to allocate the returned string array and strings + * \param[out] out pointer to the returned array of null-terminated UTF-8 strings + * \param[out] size actual size of attribute array + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.25. + */ + ORT_API2_STATUS(KernelInfoGetAttributeArray_string, _In_ const OrtKernelInfo* info, _In_ const char* name, + _Inout_ OrtAllocator* allocator, _Outptr_result_buffer_maybenull_(*size) char*** out, _Out_ size_t* size); + + /// \name OrtEnv + /// @{ + + /** \brief Set thread pool work callbacks for per-session thread pools. + * + * Stores callbacks on the Env that will be applied to per-session thread pools + * created by subsequent sessions. Does not affect sessions already created or + * global thread pools created via OrtApi::CreateEnvWithGlobalThreadPools. + * + * Requires ORT built with --enable_session_threadpool_callbacks. + * + * \param[in] env OrtEnv instance. + * \param[in] config Versioned callbacks configuration. ORT copies the struct contents; + * the caller does not need to keep the struct alive after this call returns. + * However, all pointers within the struct must remain valid for the + * lifetime of any session that uses these callbacks. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.25. + */ + ORT_API2_STATUS(SetPerSessionThreadPoolCallbacks, _Inout_ OrtEnv* env, + _In_ const OrtThreadPoolCallbacksConfig* config); + + /// @} + + /** \brief Check if the memory pattern optimization is enabled in the session options. + * + * \param[in] options + * \param[out] out Set to 1 if the memory pattern optimization is enabled, 0 otherwise. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.27. + * + * \see OrtApi::EnableMemPattern, OrtApi::DisableMemPattern + */ + ORT_API2_STATUS(GetMemPatternEnabled, _In_ const OrtSessionOptions* options, _Out_ int* out); + + /** \brief Get the current execution mode setting. + * + * \param[in] options + * \param[out] out Set to the current execution mode (ORT_SEQUENTIAL or ORT_PARALLEL). + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.27. + * + * \see OrtApi::SetSessionExecutionMode + */ + ORT_API2_STATUS(GetSessionExecutionMode, _In_ const OrtSessionOptions* options, _Out_ ExecutionMode* out); + + /** \brief Release a previously captured graph and its associated resources. + * + * When graph capture is enabled, the EP records information during initial runs (e.g., GPU commands) + * and replays them on subsequent runs. This function releases the captured resources for a specific + * graph annotation ID, freeing memory. + * + * \param[in] session The OrtSession instance. + * \param[in] graph_annotation_id The annotation ID of the captured graph to release. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.27. + */ + ORT_API2_STATUS(SessionReleaseCapturedGraph, _In_ OrtSession* session, _In_ int graph_annotation_id); + + /** \brief Retrieve an experimental function pointer by name. + * + * Experimental functions are not part of the stable ABI and may be added or removed between releases without notice. + * Use the companion header onnxruntime_experimental_c_api.h for typedefs, name constants, and (for C++) typed + * accessors. + * + * \param[in] name The null-terminated name of the experimental function to look up. + * Names follow the pattern "__SinceV". + * Name constants are defined in onnxruntime_experimental_c_api.h. + * \return The function pointer cast to ::OrtExperimentalFnPtr, or nullptr if the function is not available in this + * build. The caller must cast the returned pointer to the correct function pointer type before calling. + * Function pointer typedefs are defined in onnxruntime_experimental_c_api.h. + * + * \since Version 1.28. + */ + ORT_API_T(OrtExperimentalFnPtr, GetExperimentalFunction, _In_ const char* name); + + /** \brief Get the framework synchronization stream associated with a kernel context. + * + * This returns the framework stream wrapper for the execution provider stream used by this kernel invocation. + * It is intended for APIs that need a stable framework stream object for stream-aware allocation and + * synchronization bookkeeping. Use KernelContext_GetGPUComputeStream when launching native GPU work. + * + * \param[in] context OrtKernelContext instance. + * \param[out] out Returns the framework synchronization stream, or nullptr if the kernel has no stream. + * Do not free or mutate the returned pointer. It is owned by the underlying session. + * The pointer may be stored and used for stream-aware allocation and synchronization + * bookkeeping beyond the Compute call (e.g. an allocator may persist it in arena + * chunks); it remains valid until the owning Session::Run() completes its teardown. + * Do not retain or dereference it after the run that produced this kernel context ends. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.28. + */ + ORT_API2_STATUS(KernelContext_GetSyncStream, _In_ const OrtKernelContext* context, + _Outptr_result_maybenull_ OrtSyncStream** out); +}; + +/* + * Steps to use a custom op: + * 1 Create an OrtCustomOpDomain with the domain name used by the custom ops + * 2 Create an OrtCustomOp structure for each op and add them to the domain + * 3 Call OrtAddCustomOpDomain to add the custom domain of ops to the session options + */ + +// Specifies some characteristics of inputs/outputs of custom ops: +// Specify if the inputs/outputs are one of: +// 1) Non-optional (input/output must be present in the node) +// 2) Optional (input/output may be absent in the node) +// 3) Variadic: A variadic input or output specifies N (i.e., the minimum arity) or more operands. +// Only the last input or output of a custom op may be marked as variadic. +// The homogeneity of the variadic input or output determines whether all operands must be of the same +// tensor element type. +typedef enum OrtCustomOpInputOutputCharacteristic { + INPUT_OUTPUT_REQUIRED = 0, + INPUT_OUTPUT_OPTIONAL, + INPUT_OUTPUT_VARIADIC, +} OrtCustomOpInputOutputCharacteristic; + +/* + * The OrtCustomOp structure defines a custom op's schema and its kernel callbacks. The callbacks are filled in by + * the implementor of the custom op. + */ +struct OrtCustomOp { + uint32_t version; // Initialize to ORT_API_VERSION. ORT will cap the API version used to select the OrtApi passed + // to CreateKernel/CreateKernelV2 callbacks, so custom ops compiled against a newer ORT can load + // on an older runtime if they only use APIs available at the runtime version. Newer OrtCustomOp + // function pointers are gated by per-function version checks within ORT. + + // This callback creates the kernel, which is a user defined + // parameter that is passed to the Kernel* callbacks below. It is + // recommended to use CreateKernelV2 which allows for a safe error + // propagation by returning an OrtStatusPtr. + void*(ORT_API_CALL* CreateKernel)(_In_ const struct OrtCustomOp* op, _In_ const OrtApi* api, + _In_ const OrtKernelInfo* info); + + // Returns the name of the op + const char*(ORT_API_CALL* GetName)(_In_ const struct OrtCustomOp* op); + + // Returns the type of the execution provider, return nullptr to use CPU execution provider + const char*(ORT_API_CALL* GetExecutionProviderType)(_In_ const struct OrtCustomOp* op); + + // Returns the count and types of the input & output tensors + ONNXTensorElementDataType(ORT_API_CALL* GetInputType)(_In_ const struct OrtCustomOp* op, _In_ size_t index); + size_t(ORT_API_CALL* GetInputTypeCount)(_In_ const struct OrtCustomOp* op); + ONNXTensorElementDataType(ORT_API_CALL* GetOutputType)(_In_ const struct OrtCustomOp* op, _In_ size_t index); + size_t(ORT_API_CALL* GetOutputTypeCount)(_In_ const struct OrtCustomOp* op); + + // Perform a computation step. It is recommended to use + // KernelComputeV2 which allows for a safe error propagation by + // returning an OrtStatusPtr. + void(ORT_API_CALL* KernelCompute)(_In_ void* op_kernel, _In_ OrtKernelContext* context); + void(ORT_API_CALL* KernelDestroy)(_In_ void* op_kernel); + + // Returns the characteristics of the input & output tensors + OrtCustomOpInputOutputCharacteristic(ORT_API_CALL* GetInputCharacteristic)(_In_ const struct OrtCustomOp* op, _In_ size_t index); + OrtCustomOpInputOutputCharacteristic(ORT_API_CALL* GetOutputCharacteristic)(_In_ const struct OrtCustomOp* op, _In_ size_t index); + + // Returns the memory type of the input tensors. This API allows the custom op + // to place the inputs on specific devices. By default, it returns + // OrtMemTypeDefault, which means the input is placed on the default device for + // the execution provider. If the inputs need to be with different memory types, + // this function can be overridden to return the specific memory types. + OrtMemType(ORT_API_CALL* GetInputMemoryType)(_In_ const struct OrtCustomOp* op, _In_ size_t index); + + // Returns the minimum number of input arguments expected for the variadic input. + // Applicable only for custom ops that have a variadic input. + int(ORT_API_CALL* GetVariadicInputMinArity)(_In_ const struct OrtCustomOp* op); + + // Returns true (non-zero) if all arguments of a variadic input have to be of the same type (homogeneous), + // and false (zero) otherwise. + // Applicable only for custom ops that have a variadic input. + int(ORT_API_CALL* GetVariadicInputHomogeneity)(_In_ const struct OrtCustomOp* op); + + // Returns the minimum number of output values expected for the variadic output. + // Applicable only for custom ops that have a variadic output. + int(ORT_API_CALL* GetVariadicOutputMinArity)(_In_ const struct OrtCustomOp* op); + + // Returns true (non-zero) if all outputs values of a variadic output have to be of the same type (homogeneous), + // and false (zero) otherwise. + // Applicable only for custom ops that have a variadic output. + int(ORT_API_CALL* GetVariadicOutputHomogeneity)(_In_ const struct OrtCustomOp* op); + + // Create the kernel state which is passed to each compute call. + OrtStatusPtr(ORT_API_CALL* CreateKernelV2)(_In_ const struct OrtCustomOp* op, _In_ const OrtApi* api, + _In_ const OrtKernelInfo* info, + _Out_ void** kernel); + + // Perform the computation step. + OrtStatusPtr(ORT_API_CALL* KernelComputeV2)(_In_ void* op_kernel, _In_ OrtKernelContext* context); + + OrtStatusPtr(ORT_API_CALL* InferOutputShapeFn)(_In_ const struct OrtCustomOp* op, _In_ OrtShapeInferContext*); + + // Get start range + int(ORT_API_CALL* GetStartVersion)(_In_ const struct OrtCustomOp* op); + int(ORT_API_CALL* GetEndVersion)(_In_ const struct OrtCustomOp* op); + + // Get the inplace_map that defines which output can reuse which input + // Callers will provide 2 raw int* and pass in their address, this function will fill these 2 arrays + // when return, output (*output_index)[i] may reuse the input (*input_index[i]). + // The return value is the size of these 2 arrays. + // Callers are responsible to delete these 2 arrays after use by calling OrtCustomOp::ReleaseMayInplace(). + size_t(ORT_API_CALL* GetMayInplace)(_Out_ int** input_index, _Out_ int** output_index); + + // Release the pointer input_index and output_index allocated from GetMayInplace() function. + // If GetMayInplace() is defined, this function MUST be defined as well. + void(ORT_API_CALL* ReleaseMayInplace)(_Frees_ptr_opt_ int* input_index, _Frees_ptr_opt_ int* output_index); + + // Same as GetMayInplace() and ReleaseMayInplace() + size_t(ORT_API_CALL* GetAliasMap)(_Out_ int** input_index, _Out_ int** output_index); + void(ORT_API_CALL* ReleaseAliasMap)(_Frees_ptr_opt_ int* input_index, _Frees_ptr_opt_ int* output_index); +}; + +/** + * ORT Model Editor API + */ + +/** + * \brief The OrtModelEditorApi struct provides functions to create or edit an ONNX model. + * + * See onnxruntime/test/shared_lib/test_model_editor_api.cc for example usage. + * + * \since Version 1.22. + */ +struct OrtModelEditorApi { + // Model building/editing requires a full build. We return nullptr from GetModelEditorApi if this is a minimal + // build, so it doesn't matter if there are no function pointers in this struct as a user will never get an + // OrtModelEditorApi instance. We do however need a dummy field to avoid empty struct warning. +#if defined(ORT_MINIMAL_BUILD) + const bool not_defined_in_this_build; +#else + /** \brief Create an OrtTypeInfo instance for a Tensor. + * + * Create an OrtTypeInfo instance for a Tensor to use as graph inputs/outputs with the Model Editor API. + * + * User can release `tensor_info` after creating the OrtTypeInfo. + * + * \param[in] tensor_info Tensor type and shape information. + * \param[out] type_info TypeInfo instance for the tensor. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.22. + */ + ORT_API2_STATUS(CreateTensorTypeInfo, _In_ const OrtTensorTypeAndShapeInfo* tensor_info, + _Outptr_ OrtTypeInfo** type_info); + + /** \brief Create an OrtTypeInfo instance for a SparseTensor. + * + * Create an OrtTypeInfo instance for a SparseTensor to use as graph inputs/outputs with the Model Editor API. + * + * User can release `tensor_info` after creating the OrtTypeInfo. + * + * \param[in] tensor_info SparseTensor type and shape information. + * \param[out] type_info TypeInfo instance for the tensor. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.22. + */ + ORT_API2_STATUS(CreateSparseTensorTypeInfo, _In_ const OrtTensorTypeAndShapeInfo* tensor_info, + _Outptr_ OrtTypeInfo** type_info); + + /** \brief Create an OrtTypeInfo instance for a Map. + * + * Create an OrtTypeInfo instance for a Map to use as graph inputs/outputs with the Model Editor API. + * + * User can release `map_value_type` after creating the OrtTypeInfo. + * + * \param[in] map_key_type Key type for the map. + * \param[in] map_value_type Value type for the map. + * \param[out] type_info TypeInfo instance for the map. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.22. + */ + ORT_API2_STATUS(CreateMapTypeInfo, ONNXTensorElementDataType map_key_type, _In_ const OrtTypeInfo* map_value_type, + _Outptr_ OrtTypeInfo** type_info); + + /** \brief Create an OrtTypeInfo instance for a Sequence. + * + * Create an OrtTypeInfo instance for a Sequence to use as graph inputs/outputs with the Model Editor API. + * + * User can release `sequence_type` after creating the OrtTypeInfo. + * + * \param[in] sequence_type Sequence type and shape information. + * \param[out] type_info TypeInfo instance for the sequence. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.22. + */ + ORT_API2_STATUS(CreateSequenceTypeInfo, _In_ const OrtTypeInfo* sequence_type, _Outptr_ OrtTypeInfo** type_info); + + /** \brief Create an OrtTypeInfo instance for an Optional. + * + * Create an OrtTypeInfo instance for an Optional to use as graph inputs/outputs with the Model Editor API. + * + * User can release `contained_type` after creating the OrtTypeInfo. + * + * \param[in] contained_type Tensor type and shape information. + * \param[out] type_info TypeInfo instance for the tensor. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.22. + */ + ORT_API2_STATUS(CreateOptionalTypeInfo, _In_ const OrtTypeInfo* contained_type, _Outptr_ OrtTypeInfo** type_info); + + /** \brief Create an OrtValueInfo for use as an OrtGraph input or output. + * + * \param[in] name The name of the input or output. + * \param[in] type_info The type information for the input or output. The provided value is copied. + * \param[out] value_info The OrtValueInfo instance. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.22. + */ + ORT_API2_STATUS(CreateValueInfo, _In_ const char* name, _In_ const OrtTypeInfo* type_info, + _Outptr_ OrtValueInfo** value_info); + + /** \brief Create an OrtNode to add to an OrtGraph. + * + * Create an OrtNode. + * + * Create attributes with CreateOpAttr. OrtOpAttr instances are copied. + * + * \param[in] operator_name The name of the operator. + * \param[in] domain_name The domain of the operator. Use an empty string for ONNX operators. + * \param[in] node_name The name of the node. + * \param[in] input_names The names of the inputs. + * \param[in] input_names_len The number of input names. + * \param[in] output_names The names of the outputs. + * \param[in] output_names_len The number of output names. + * \param[in] attributes The optional attributes of the node. + * \param[in] attribs_len The number of attributes. May be zero. + * \param[out] node The OrtNode instance. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.22. + */ + ORT_API2_STATUS(CreateNode, _In_ const char* operator_name, _In_ const char* domain_name, _In_ const char* node_name, + _In_reads_(input_names_len) const char* const* input_names, size_t input_names_len, + _In_reads_(output_names_len) const char* const* output_names, size_t output_names_len, + _In_reads_(attribs_len) _In_opt_ OrtOpAttr** attributes, _In_ size_t attribs_len, + _Outptr_ OrtNode** node); + + /** \brief Create an OrtGraph + * \snippet{doc} snippets.dox OrtStatus Return Value + * \since Version 1.22. + */ + ORT_API2_STATUS(CreateGraph, _Outptr_ OrtGraph** graph); + + /** \brief Set the inputs for the OrtGraph. + * + * Set the graph inputs. This will replace any existing inputs with the new values. + * + * Atomicity / all-or-nothing: on success, the OrtGraph takes ownership of every OrtValueInfo in + * `inputs`, and each entry in the array is reset to nullptr to make the transfer explicit. On any + * failure, the graph's previous inputs are preserved and ownership of NONE of the OrtValueInfo + * instances is transferred -- the caller remains responsible for releasing them. Calling + * OrtApi::ReleaseValueInfo on an entry whose array slot has been nulled out would cause a double-free. + * + * \param[in] graph The OrtGraph instance to update. + * \param[in] inputs The input OrtValueInfo instances. + * \param[in] inputs_len The number of input OrtValueInfo instances. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.22. + */ + ORT_API2_STATUS(SetGraphInputs, _Inout_ OrtGraph* graph, + _In_reads_(inputs_len) _Inout_ OrtValueInfo** inputs, _In_ size_t inputs_len); + + /** \brief Set the outputs for the OrtGraph. + * + * Set the graph outputs. This will replace any existing outputs with the new values. + * + * Atomicity / all-or-nothing: on success, the OrtGraph takes ownership of every OrtValueInfo in + * `outputs`, and each entry in the array is reset to nullptr to make the transfer explicit. On any + * failure, the graph's previous outputs are preserved and ownership of NONE of the OrtValueInfo + * instances is transferred -- the caller remains responsible for releasing them. Calling + * OrtApi::ReleaseValueInfo on an entry whose array slot has been nulled out would cause a double-free. + * + * \param[in] graph The OrtGraph instance to update. + * \param[in] outputs The output OrtValueInfo instances. + * \param[in] outputs_len The number of output OrtValueInfo instances. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.22. + */ + ORT_API2_STATUS(SetGraphOutputs, _Inout_ OrtGraph* graph, + _In_reads_(outputs_len) _Inout_ OrtValueInfo** outputs, _In_ size_t outputs_len); + + /** \brief Add an initializer to the OrtGraph + * + * Atomicity / all-or-nothing: on success, the OrtGraph takes ownership of `tensor` and the caller + * must NOT call OrtApi::ReleaseValue on it. On any failure, ownership is NOT transferred and the caller + * remains responsible for releasing the OrtValue. Calling OrtApi::ReleaseValue after a successful + * transfer would cause a double-free. + * Adding the same OrtValue pointer twice (under any name) is rejected with ORT_INVALID_ARGUMENT. + * Adding a duplicate initializer name is also rejected. The Model Editor API does not currently + * support removing or replacing an initializer once it has been added. + * + * Two options: + * + * Allocated memory: + * Use OrtApi::CreateTensorAsOrtValue (allocates memory) and populate the tensor with the data. + * Set `data_is_external` to false. + * + * Pre-existing memory: + * Use OrtApi::CreateTensorWithDataAsOrtValue or OrtApi::CreateTensorWithDataAndDeleterAsOrtValue to create an OrtValue + * with a tensor that contains a pointer to the existing data. + * Set `data_is_external` to true. + * + * The pointer must remain valid for the duration of the inference session. + * If using OrtApi::CreateTensorWithDataAsOrtValue you are responsible for freeing the memory after the inference session + * is released. + * If using OrtApi::CreateTensorWithDataAndDeleterAsOrtValue, ORT will free the memory using the provided deleter as + * soon as the OrtValue is no longer in use. + * + * NOTE: A tensor containing pre-existing memory MUST have 128 bytes of data or more. + * For smaller tensors use OrtApi::CreateTensorAsOrtValue. + * + * ONNX shape inferencing does not support external data. An initializer involved in shape inferencing is + * typically small (a single value or limited by the rank of a tensor) and uses less than 128 bytes of + * memory, so this limit acts as a simple catch-all rule to avoid issues. + * e.g. Reshape's `shape`, Clip's `min` and `max`, various ops `axes`. + * + * \param[in] graph The OrtGraph instance to update. + * \param[in] name The value name for the initializer. + * \param[in] tensor The OrtValue instance containing the tensor data. + * \param[in] data_is_external Set to true if the data is external and should not be copied. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.22. + */ + ORT_API2_STATUS(AddInitializerToGraph, _Inout_ OrtGraph* graph, _In_ const char* name, _Inout_ OrtValue* tensor, + bool data_is_external); + + /** \brief Add an OrtNode to an OrtGraph + * + * Atomicity / all-or-nothing: on success, the OrtGraph takes ownership of `node` and the caller + * must NOT call OrtApi::ReleaseNode. On any failure, ownership is NOT transferred and the caller remains + * responsible for releasing the OrtNode. Calling OrtApi::ReleaseNode after a successful transfer would + * cause a double-free. + * Adding the same OrtNode pointer twice is rejected with ORT_INVALID_ARGUMENT. + * + * \param[in] graph The OrtGraph instance to update. + * \param[in] node The OrtNode instance to add to the graph. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.22. + */ + ORT_API2_STATUS(AddNodeToGraph, _Inout_ OrtGraph* graph, _Inout_ OrtNode* node); + + /** \brief Create an OrtModel. + * + * Create an OrtModel. + * + * This can be used to build a new model, or to augment an existing model. + * + * \param[in] domain_names The domain names for the model. + * If augmenting an existing model add additional domains if needed. + * \param[in] opset_versions The opset versions for the model. + * If augmenting an existing model add additional opset versions if needed. + * \param[in] opset_entries_len The number of domain_names and opset_versions entries. + * Domain and opset entries should be 1:1 + * \param[out] model The OrtModel instance. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.22. + */ + ORT_API2_STATUS(CreateModel, + _In_reads_(opset_entries_len) const char* const* domain_names, + _In_reads_(opset_entries_len) const int* opset_versions, + size_t opset_entries_len, + _Outptr_ OrtModel** model); + + /** \brief Add an OrtGraph to an OrtModel. + * + * Add the graph to a model. Each OrtModel may hold at most one OrtGraph; a second call is rejected + * with ORT_INVALID_ARGUMENT. + * + * Atomicity / all-or-nothing: on success, the OrtModel takes ownership of `graph` and the caller + * must NOT call OrtApi::ReleaseGraph. On any failure, ownership is NOT transferred and the caller + * remains responsible for releasing the OrtGraph. Calling OrtApi::ReleaseGraph after a successful + * transfer would cause a double-free. + * + * \param[in] model The OrtModel instance to update. + * \param[in] graph The OrtGraph instance to add to the model. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.22. + */ + ORT_API2_STATUS(AddGraphToModel, _Inout_ OrtModel* model, _Inout_ OrtGraph* graph); + + /** \brief Create an OrtSession using the OrtModel. + * + * Create an inference session using the OrtModel instance. + * The OrtModel should have been populated with an OrtGraph containing nodes and initializers, and SetGraphInputs + * and SetGraphOutputs must have been called. + * This will validate the model, run optimizers, and prepare the session for inferencing. + * + * OrtApi::ReleaseModel must be called to free the OrtModel after session creation. + * + * \param[in] env The OrtEnv instance. + * \param[in] model The OrtModel instance. + * \param[in] options The OrtSessionOptions instance. + * \param[out] out The OrtSession instance. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.22. + */ + ORT_API2_STATUS(CreateSessionFromModel, _In_ const OrtEnv* env, _In_ const OrtModel* model, + _In_ const OrtSessionOptions* options, _Outptr_ OrtSession** out); + + /** \brief Create an OrtSession to augment an existing model. + * + * Create an OrtSession with an existing model that will be augmented with additional nodes and initializers. + * Nodes can be added before or after the existing nodes in the model. ONNX Runtime will connect the nodes when the + * model is finalized. + * + * To add nodes and initializers to the existing model, first create an OrtModel using CreateModel(). + * Add nodes and initializers to the OrtModel using AddNodeToGraph() and AddInitializerToGraph(). + * Graph inputs/outputs should be updated with SetGraphInputs() and SetGraphOutputs() as needed to reflect changes made + * by the new nodes. The list of graph inputs/outputs should be for the overall model and not just the new nodes. + * + * Add the new information from the OrtModel to the original model using ApplyModelToSession(), and prepare the + * session for inferencing by calling FinalizeModelEditorSession(). + * + * \param{in} env The OrtEnv instance. + * \param{in} model_path The path to the existing ONNX model to augment. + * \param{in} options The OrtSessionOptions instance. + * \param{out} out The created OrtSession instance. + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.22. + */ + ORT_API2_STATUS(CreateModelEditorSession, _In_ const OrtEnv* env, _In_ const ORTCHAR_T* model_path, + _In_ const OrtSessionOptions* options, + _Outptr_ OrtSession** out); + + /** \brief Create an OrtSession to augment an existing model. + * + * Create an OrtSession with an existing model that will be augmented with additional nodes and initializers. + * Nodes can be added before or after the existing nodes in the model. ONNX Runtime will connect the nodes when the + * model is finalized. + * + * To add nodes and initializers to the existing model, first create an OrtModel using CreateModel(). + * Add nodes and initializers to the OrtModel using AddNodeToGraph() and AddInitializerToGraph(). + * Graph inputs/outputs should be updated with SetGraphInputs() and SetGraphOutputs() as needed to reflect changes made + * by the new nodes. The list of graph inputs/outputs should be for the overall model and not just the new nodes. + * + * Add the new information from the OrtModel to the original model using ApplyModelToSession(), and prepare the + * session for inferencing by calling FinalizeModelEditorSession(). + * + * \param{in} env The OrtEnv instance. + * \param{in} model_data The model data for the existing model to augment. + * \param{in} model_data_length The length of the model data. + * \param{in} options The OrtSessionOptions instance. + * \param{out} out The created OrtSession instance. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.22. + */ + ORT_API2_STATUS(CreateModelEditorSessionFromArray, _In_ const OrtEnv* env, + _In_ const void* model_data, size_t model_data_length, + _In_ const OrtSessionOptions* options, + _Outptr_ OrtSession** out); + + /** \brief Query the session for the opset version of a domain. + * + * When using the Model Editor API to augment a model, any new nodes must conform to the opset version of the + * original model. To do that the user must be able to discover that opset version. + * Returns an error if the domain is not used in the model. + * + * \param[in] session OrtSession to query + * \param[in] domain Domain to query. The ONNX domain is an empty string. + * \param[out] opset The opset version of the domain. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.22. + */ + ORT_API2_STATUS(SessionGetOpsetForDomain, _In_ const OrtSession* session, _In_ const char* domain, _Out_ int* opset); + + /** \brief Apply changes to augment the ONNX model in a session created using CreateModelEditorSession[FromArray] + * + * Adds new nodes and updates graph inputs/outputs using `model` to augment the original ONNX model in the session. + * All changes will be validated. + * Call FinalizeModelEditorSession to prepare the session for inferencing. + * + * Existing input/outputs will only be updated if the OrtGraph inputs/outputs are set in the OrtModel. + * i.e. you don't need to call SetGraphInputs/SetGraphOutputs if they are unchanged. + * + * ReleaseOrtModel must be called to free the OrtModel after it is applied to the session. + * + * \param[in] session OrtSession to update. Session must have been created using CreateModelEditorSession[FromArray]. + * \param[in] model OrtModel containing new nodes, new initializers, and updated graph input and/or output info. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.22. + */ + ORT_API2_STATUS(ApplyModelToModelEditorSession, _Inout_ OrtSession* session, _In_ OrtModel* model); + + /** \brief Finalize the Model Editor session that was created using CreateModelEditorSession[FromArray]. + * + * Finalize the Model Editor session that augmented an ONNX model by adding new nodes. + * This will run optimizers and prepare the session for inferencing. + * + * \param[in] session OrtSession to finalize. Session must have been created using CreateModelEditorSession[FromArray]. + * \param[in] options OrtSessionOptions to use for the session. + * \param[in] prepacked_weights_container Optional OrtPrepackedWeightsContainer to use for the session. + Set to nullptr if not used. + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.22. + */ + ORT_API2_STATUS(FinalizeModelEditorSession, _Inout_ OrtSession* session, _In_ const OrtSessionOptions* options, + _In_opt_ OrtPrepackedWeightsContainer* prepacked_weights_container); +#endif // !defined(ORT_MINIMAL_BUILD) +}; + +/** + * ORT Compile API + */ + +/** \brief Flags representing options to enable when compiling a model. + */ +typedef enum OrtCompileApiFlags { + // Default. Do not enable any additional compilation options. + OrtCompileApiFlags_NONE = 0, + + // Force compilation to return an error (ORT_FAIL) if no nodes were compiled. + // Otherwise, a model with basic optimizations (ORT_ENABLE_BASIC) is still generated by default. + OrtCompileApiFlags_ERROR_IF_NO_NODES_COMPILED = 1 << 0, + + // Force compilation to return an error (ORT_FAIL) if a file with the same filename as the output model exists. + // Otherwise, compilation will automatically overwrite the output file if it exists. + OrtCompileApiFlags_ERROR_IF_OUTPUT_FILE_EXISTS = 1 << 1, +} OrtCompileApiFlags; + +/** + * \brief The OrtCompileApi struct provides functions to compile ONNX models. + * + * Execution providers that support compilation fuse a subgraph into an EPContext node that wraps a provider-specific + * binary representation of the subgraph. + * For more details about the EPContext design, refer to: + * \htmlonly + * EPContext design document. + * \endhtmlonly + * + * Example (error handling not shown): + * OrtStatus* status = NULL; + * OrtCompileApi* compile_api = ort_api->GetCompileApi(); + * OrtModelCompilationOptions* compile_options = NULL; + * + * status = compile_api->CreateModelCompilationOptionsFromSessionOptions(env, session_options, &compile_options); + * status = compile_api->ModelCompilationOptions_SetInputModelPath(compile_options, ORT_TSTR("model.onnx")); + * status = compile_api->ModelCompilationOptions_SetOutputModelPath(compile_options, ORT_TSTR("model.compiled.onnx")); + * status = compile_api->CompileModel(env, compile_options); + * compile_api->ReleaseModelCompilationOptions(compile_options); + * + * \since Version 1.22. + */ +struct OrtCompileApi { + /// \name OrtModelCompilationOptions + /// @{ + ORT_CLASS_RELEASE(ModelCompilationOptions); + + /** \brief Creates an OrtModelCompilationOptions object from an existing OrtSessionOptions object. + * + * An OrtModelCompilationOptions object contains the settings used to generate a compiled ONNX model. + * The OrtSessionOptions object has the execution providers with which the model will be compiled. + * + * ReleaseOrtModelCompilationsOptions must be called to free the OrtModelCompilationOptions after calling + * CompileModel. + * + * \note By default, the GraphOptimizationLevel is set to ORT_DISABLE_ALL. Use + * ModelCompilationOptions_SetGraphOptimizationLevel to enable graph optimizations. + * + * \param[in] env OrtEnv object. + * \param[in] session_options The OrtSessionOptions instance from which to create the OrtModelCompilationOptions. + * \param[out] out The created OrtModelCompilationOptions instance. + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.22. + */ + ORT_API2_STATUS(CreateModelCompilationOptionsFromSessionOptions, _In_ const OrtEnv* env, + _In_ const OrtSessionOptions* session_options, _Outptr_ OrtModelCompilationOptions** out); + + /** \brief Sets the file path to the input ONNX model to compile. + * + * The input model's location (e.g., file path or memory buffer) must be set with either + * ModelCompilationOptions_SetInputModelPath or ModelCompilationOptions_SetInputModelFromBuffer. + * + * \param[in] model_compile_options The OrtModelCompilationOptions instance. + * \param[in] input_model_path Null terminated string of the path (wchar on Windows, char otherwise). + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.22. + */ + ORT_API2_STATUS(ModelCompilationOptions_SetInputModelPath, _In_ OrtModelCompilationOptions* model_compile_options, + _In_ const ORTCHAR_T* input_model_path); + + /** \brief Sets the buffer that stores the bytes of the loaded ONNX model to compile. + * + * The input model's location (e.g., file path or memory buffer) must be set with either + * ModelCompilationOptions_SetInputModelPath or ModelCompilationOptions_SetInputModelFromBuffer. + * + * \param[in] model_compile_options The OrtModelCompilationOptions instance. + * \param[in] input_model_data Buffer containing the loaded ONNX model bytes. + * \param[in] input_model_data_size The number of bytes in the `input_model_data` buffer. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.22. + */ + ORT_API2_STATUS(ModelCompilationOptions_SetInputModelFromBuffer, + _In_ OrtModelCompilationOptions* model_compile_options, + _In_ const void* input_model_data, + size_t input_model_data_size); + + /** \brief Sets the file path for the output ONNX model generated by CompileModel. + * + * The output model's location (e.g., file path or memory buffer) can be set with either + * ModelCompilationOptions_SetOutputModelPath or ModelCompilationOptions_SetOutputModelBuffer. + * + * If the output model's location is not set, ONNX Runtime will generate an output file with a path based on + * the input model's file path. Examples: + * /Path/my_model.onnx -> /Path/my_model_ctx.onnx + * /Path/my_model -> /Path/my_model_ctx.onnx + * + * \param[in] model_compile_options The OrtModelCompilationOptions instance. + * \param[in] output_model_path Null terminated string of the path (wchar on Windows, char otherwise). + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.22. + */ + ORT_API2_STATUS(ModelCompilationOptions_SetOutputModelPath, _In_ OrtModelCompilationOptions* model_compile_options, + _In_ const ORTCHAR_T* output_model_path); + + /** \brief Optionally sets the file that should store external initializers for the compiled ONNX model. + * If not set, initializers are stored within the model. + * + * Only initializers for nodes that were not compiled are stored in the external initializers file. + * Compiled nodes contain their initializer data within the `ep_cache_context` attribute of EPContext nodes. + * Refer to ModelCompilationOptions_SetEpContextEmbedMode. + * + * \param[in] model_compile_options The OrtModelCompilationOptions instance. + * \param[in] external_initializers_file_path Null terminated string of the path to the file. + * \param[in] external_initializers_size_threshold Initializers larger than this threshold are stored in the file. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.22. + */ + ORT_API2_STATUS(ModelCompilationOptions_SetOutputModelExternalInitializersFile, + _In_ OrtModelCompilationOptions* model_compile_options, + _In_ const ORTCHAR_T* external_initializers_file_path, + size_t external_initializers_size_threshold); + + /** \brief Configures model compilation to store the output compiled ONNX model in a buffer. + * + * The caller passes an OrtAllocator that ONNX Runtime uses to allocate memory for the buffer. + * + * The output model's location (e.g., file path or memory buffer) can be set with either + * ModelCompilationOptions_SetOutputModelPath or ModelCompilationOptions_SetOutputModelBuffer. + * + * If the output model's location is not set, ONNX Runtime will generate an output file with a path based on + * the input model's file path. Examples: + * /Path/my_model.onnx -> /Path/my_model_ctx.onnx + * /Path/my_model -> /Path/my_model_ctx.onnx + * + * \param[in] model_compile_options The OrtModelCompilationOptions instance. + * \param[in] allocator The allocator used to allocate the buffer for the compiled model. + * \param[out] output_model_buffer_ptr Pointer to the buffer that stores the compiled model. + * \param[out] output_model_buffer_size_ptr Pointer set to the size of output model in bytes. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.22. + */ + ORT_API2_STATUS(ModelCompilationOptions_SetOutputModelBuffer, + _In_ OrtModelCompilationOptions* model_compile_options, + _Inout_ OrtAllocator* allocator, + _Outptr_ void** output_model_buffer_ptr, + _Out_ size_t* output_model_buffer_size_ptr); + + /** \brief Enables or disables the embedding of EPContext binary data into the `ep_cache_context` attribute + * of EPContext nodes. Defaults to false. + * + * If enabled, the `ep_cache_context` attribute of EPContext nodes will store the context binary data, which may + * include weights for compiled subgraphs. + * + * If disabled, the `ep_cache_context` attribute of EPContext nodes will contain the path to the file containing the + * context binary data. The path is set by the execution provider creating the EPContext node. + * + * More details relate to EPContext design refers to: + * \htmlonly + * EPContext design document. + * \endhtmlonly + * + * \param[in] model_compile_options The OrtModelCompilationOptions instance. + * \param[in] embed_ep_context_in_model True to embed EPContext binary data into the EPContext node + * `ep_cache_context` attributes. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.22. + */ + ORT_API2_STATUS(ModelCompilationOptions_SetEpContextEmbedMode, _In_ OrtModelCompilationOptions* model_compile_options, + bool embed_ep_context_in_model); + + /** \brief Compiles an input ONNX model with the given compilation options. + * + * \param[in] env OrtEnv object. + * \param[in] model_options The compilation options that defines compilation options for a model. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.22. + */ + ORT_API2_STATUS(CompileModel, _In_ const OrtEnv* env, _In_ const OrtModelCompilationOptions* model_options); + + /** \brief Sets flags from OrtCompileApiFlags that represent one or more boolean options to enable. + * + * \param[in] model_compile_options The OrtModelCompilationOptions instance. + * \param[in] flags bitwise OR of flags in OrtCompileApiFlags to enable. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(ModelCompilationOptions_SetFlags, _In_ OrtModelCompilationOptions* model_compile_options, + uint32_t flags); + + /** Sets information related to EP context binary file. + * + * EP uses this information to decide the location and context binary file name. + * Used while compiling model with input and output in memory buffer + * + * \param[in] model_compile_options The OrtModelCompilationOptions instance. + * \param[in] output_directory Null terminated string of the path (wchar on Windows, char otherwise). + * \param[in] model_name Null terminated string of the model name (wchar on Windows, char otherwise). + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(ModelCompilationOptions_SetEpContextBinaryInformation, + _In_ OrtModelCompilationOptions* model_compile_options, + _In_ const ORTCHAR_T* output_directory, + _In_ const ORTCHAR_T* model_name); + + /** Set the graph optimization level. + * + * \param[in] model_compile_options The OrtModelCompilationOptions instance. + * \param[in] graph_optimization_level The graph optimization level. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(ModelCompilationOptions_SetGraphOptimizationLevel, + _In_ OrtModelCompilationOptions* model_compile_options, + _In_ GraphOptimizationLevel graph_optimization_level); + + /** \brief Sets a OrtWriteBufferFunc function that is called by ORT to write out the output model's serialized + * ONNX bytes. + * + * The provided write function may be called repeatedly until then entire output model has been written out. Each call + * to the write function is expected to consume the entire input buffer. + * + * The output model's destination (e.g., file path, memory buffer, or stream) can be set with any of the functions + * that begin with ModelCompilationOptions_SetOutputModel____. + * + * \param[in] model_compile_options The OrtModelCompilationOptions instance. + * \param[in] write_func The OrtWriteBufferFunc function called by ORT when writing out the model. + * \param[in] state Opaque state passed as the first argument to OrtWriteBufferFunc. Can be NULL. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(ModelCompilationOptions_SetOutputModelWriteFunc, + _In_ OrtModelCompilationOptions* model_compile_options, + _In_ OrtWriteBufferFunc write_func, _In_ void* state); + + /** \brief Sets a OrtGetInitializerLocationFunc function that is called by ORT for every initializer in the generated + * model. Allows implementer to specify whether initializers should be stored within the model or externally. + * + * \param[in] model_compile_options The OrtModelCompilationOptions instance. + * \param[in] get_initializer_location_func The OrtGetInitializerLocationFunc function called by ORT when + * to determine the location of the initializer. + * \param[in] state Opaque state passed as the first argument to OrtGetInitializerLocationFunc. Can be NULL. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(ModelCompilationOptions_SetOutputModelGetInitializerLocationFunc, + _In_ OrtModelCompilationOptions* model_compile_options, + _In_ OrtGetInitializerLocationFunc get_initializer_location_func, _In_ void* state); + + /** \brief Sets the OrtModel to compile. + * + * Sets an OrtModel created via the Model Editor API as the input for compilation. + * + * The input model's source (file path, memory buffer, or OrtModel) must be set with + * one of: ModelCompilationOptions_SetInputModelPath, ModelCompilationOptions_SetInputModelFromBuffer, + * or ModelCompilationOptions_SetInputModel. + * + * The OrtModel must have a complete graph with inputs, outputs, and nodes defined. + * The caller retains ownership of the OrtModel and must not release it until after + * CompileModel returns. + * + * \param[in] model_compile_options The OrtModelCompilationOptions instance. + * \param[in] model The OrtModel to compile. The model is borrowed (not copied or owned). + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24. + */ + ORT_API2_STATUS(ModelCompilationOptions_SetInputModel, + _In_ OrtModelCompilationOptions* model_compile_options, + _In_ const OrtModel* model); +}; + +/** + * \brief The OrtInteropApi struct provides functions for external resource interop with execution providers. + * + * This API enables importing external GPU resources (memory and semaphores) for zero-copy sharing + * between ORT inference and other GPU workloads (e.g., D3D12 applications, media pipelines). + * + * The API is designed to be EP-agnostic and can be extended to support various GPU interop mechanisms + * (D3D12 shared handles, CUDA external memory, Vulkan, etc.). + * + * Example usage (error handling not shown): + * const OrtInteropApi* interop_api = ort_api->GetInteropApi(); + * OrtExternalResourceImporter* importer = NULL; + * + * status = interop_api->CreateExternalResourceImporterForDevice(ep_device, &importer); + * if (importer == nullptr) { + * // External resource import is optional for EPs to implement + * return; + * } + * bool can_import = false; + * status = interop_api->CanImportMemory(importer, ORT_EXTERNAL_MEMORY_HANDLE_TYPE_D3D12_RESOURCE, &can_import); + * if (can_import) { + * OrtExternalMemoryHandle* mem_handle = NULL; + * status = interop_api->ImportMemory(importer, &mem_desc, &mem_handle); + * // ... use mem_handle to create tensors ... + * interop_api->ReleaseExternalMemoryHandle(mem_handle); + * } + * interop_api->ReleaseExternalResourceImporter(importer); + * + * \since Version 1.24. + */ +struct OrtInteropApi { + /// \name OrtExternalResourceImporter + /// @{ + + /** \brief Create an external resource importer for a specific EP device. + * + * The external resource importer is a capability object that provides methods for importing + * external GPU memory and semaphores for zero-copy import with an execution provider. + * + * This is an optional EP capability. If the EP does not support external resource import, + * out_importer is set to nullptr and the function returns success (nullptr status). + * This allows callers to use the simple "if (status != nullptr) handle_error()" pattern + * and check out_importer separately for capability detection. + * + * \param[in] ep_device The OrtEpDevice instance to create the importer for. + * \param[out] out_importer Output parameter set to the created OrtExternalResourceImporter instance, + * or nullptr if the EP does not support external resource import. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24. + */ + ORT_API2_STATUS(CreateExternalResourceImporterForDevice, + _In_ const OrtEpDevice* ep_device, + _Outptr_result_maybenull_ OrtExternalResourceImporter** out_importer); + + /** \brief Release an OrtExternalResourceImporter instance. + * + * \param[in] input The OrtExternalResourceImporter instance to release. May be nullptr. + * + * \since Version 1.24. + */ + ORT_CLASS_RELEASE(ExternalResourceImporter); + + /// @} + /// \name Memory Import + /// @{ + + /** \brief Check if the external resource importer can import a specific memory handle type. + * + * \param[in] importer The OrtExternalResourceImporter instance. + * \param[in] handle_type The type of external memory handle to check. + * \param[out] out_supported Set to true if the handle type is supported. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24. + */ + ORT_API2_STATUS(CanImportMemory, + _In_ const OrtExternalResourceImporter* importer, + _In_ OrtExternalMemoryHandleType handle_type, + _Out_ bool* out_supported); + + /** \brief Import external memory into the execution provider. + * + * \param[in] importer The OrtExternalResourceImporter instance. + * \param[in] desc Descriptor containing the external memory handle and properties. + * \param[out] out_handle Output parameter set to the created OrtExternalMemoryHandle. + * The caller owns the returned handle and must call ReleaseExternalMemoryHandle to free it. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24. + */ + ORT_API2_STATUS(ImportMemory, + _In_ OrtExternalResourceImporter* importer, + _In_ const OrtExternalMemoryDescriptor* desc, + _Outptr_ OrtExternalMemoryHandle** out_handle); + + /** \brief Release an OrtExternalMemoryHandle instance. + * + * \param[in] input The OrtExternalMemoryHandle instance to release. May be nullptr. + * + * \since Version 1.24. + */ + ORT_CLASS_RELEASE(ExternalMemoryHandle); + + /** \brief Create a tensor backed by imported external memory. + * + * The created tensor is a view over the imported memory and does not copy data. + * The OrtExternalMemoryHandle must remain valid for the lifetime of the tensor. + * + * \param[in] importer The OrtExternalResourceImporter instance. + * \param[in] mem_handle The imported external memory handle. + * \param[in] tensor_desc Descriptor specifying tensor element type, shape, and optional offset. + * \param[out] out_tensor Output parameter set to the created OrtValue containing the tensor. + * The caller owns the returned tensor and must call ReleaseValue to free it. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24. + */ + ORT_API2_STATUS(CreateTensorFromMemory, + _In_ OrtExternalResourceImporter* importer, + _In_ const OrtExternalMemoryHandle* mem_handle, + _In_ const OrtExternalTensorDescriptor* tensor_desc, + _Outptr_ OrtValue** out_tensor); + + /// @} + /// \name Semaphore Import + /// @{ + + /** \brief Check if the external resource importer can import a specific semaphore type. + * + * \param[in] importer The OrtExternalResourceImporter instance. + * \param[in] type The type of external semaphore to check. + * \param[out] out_supported Set to true if the semaphore type is supported. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24. + */ + ORT_API2_STATUS(CanImportSemaphore, + _In_ const OrtExternalResourceImporter* importer, + _In_ OrtExternalSemaphoreType type, + _Out_ bool* out_supported); + + /** \brief Import an external semaphore into the execution provider. + * + * The returned OrtExternalSemaphoreHandle can be used with WaitSemaphore and an OrtSyncStream + * to synchronize execution with external operations. + * + * \param[in] importer The OrtExternalResourceImporter instance. + * \param[in] desc Descriptor containing the external semaphore handle and type. + * \param[out] out_handle Output parameter set to the created OrtExternalSemaphoreHandle. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24. + */ + ORT_API2_STATUS(ImportSemaphore, + _In_ OrtExternalResourceImporter* importer, + _In_ const OrtExternalSemaphoreDescriptor* desc, + _Outptr_ OrtExternalSemaphoreHandle** out_handle); + + /** \brief Release an OrtExternalSemaphoreHandle instance. + * + * \param[in] input The OrtExternalSemaphoreHandle instance to release. May be nullptr. + * + * \since Version 1.24. + */ + ORT_CLASS_RELEASE(ExternalSemaphoreHandle); + + /** \brief Wait on an external semaphore on the EP's stream. + * + * Inserts a wait operation into the EP's stream that blocks until the semaphore + * reaches the specified value. This is used to synchronize with external GPU work + * (e.g., D3D12 timeline fence). + * + * \param[in] importer The OrtExternalResourceImporter instance. + * \param[in] semaphore_handle The imported external semaphore. + * \param[in] stream The OrtSyncStream to wait on. + * \param[in] value The fence/semaphore value to wait for. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24. + */ + ORT_API2_STATUS(WaitSemaphore, + _In_ OrtExternalResourceImporter* importer, + _In_ OrtExternalSemaphoreHandle* semaphore_handle, + _In_ OrtSyncStream* stream, + _In_ uint64_t value); + + /** \brief Signal an external semaphore from the EP's stream. + * + * Inserts a signal operation into the EP's stream that sets the semaphore + * to the specified value when reached. This is used to notify external GPU work + * (e.g., D3D12 timeline fence) that ORT inference is complete. + * + * \param[in] importer The OrtExternalResourceImporter instance. + * \param[in] semaphore_handle The imported external semaphore. + * \param[in] stream The OrtSyncStream to signal from. + * \param[in] value The fence/semaphore value to signal. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24. + */ + ORT_API2_STATUS(SignalSemaphore, + _In_ OrtExternalResourceImporter* importer, + _In_ OrtExternalSemaphoreHandle* semaphore_handle, + _In_ OrtSyncStream* stream, + _In_ uint64_t value); + + /// @} + /// \name Graphics interop + /// @{ + + /** \brief Initialize graphics interop for an execution provider device. + * + * Requests the EP factory to set up graphics interop for the given ep_device using the + * provided config. How the factory uses the config (e.g. creating a context, affecting + * later stream creation) is implementation-defined. The config (OrtGraphicsInteropConfig) + * supplies the graphics API and optional handles; the command_queue member is optional. + * Passing command_queue and calling this before CreateSyncStreamForEpDevice for the same + * ep_device may allow the factory to enable more efficient GPU-side sync; see the EP + * documentation for details. + * + * Initialization is tied to the OrtEpDevice instance and applies across all sessions + * that use that ep_device (i.e. it is global per ep_device, not per-session). + * + * \param[in] ep_device The OrtEpDevice to initialize graphics interop for. + * \param[in] config Configuration (OrtGraphicsInteropConfig): required fields are version (ORT_API_VERSION) + * and graphics_api; optional handles include command_queue and additional_options. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.25. + */ + ORT_API2_STATUS(InitGraphicsInteropForEpDevice, _In_ const OrtEpDevice* ep_device, + _In_ const OrtGraphicsInteropConfig* config); + + /** \brief Deinitialize graphics interop for an execution provider device. + * + * This function cleans up the graphics interop context that was created by InitGraphicsInteropForEpDevice. + * Should be called when graphics interop is no longer needed for the ep_device. + * + * The caller must release all resources that use the interop context before calling this function: + * - OrtSyncStream instances created for this ep_device via CreateSyncStreamForEpDevice + * - OrtExternalSemaphoreHandle and OrtExternalResourceImporter instances created for this ep_device + * Failure to do so may lead to undefined behavior if the implementation destroys the underlying context. + * + * \param[in] ep_device The OrtEpDevice to deinitialize graphics interop for. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.25. + */ + ORT_API2_STATUS(DeinitGraphicsInteropForEpDevice, _In_ const OrtEpDevice* ep_device); + + /// @} +}; + +/* + * This is the old way to add the CUDA provider to the session, please use SessionOptionsAppendExecutionProvider_CUDA above to access the latest functionality + * This function always exists, but will only succeed if Onnxruntime was built with CUDA support and the CUDA provider shared library exists + * + * \param device_id CUDA device id, starts from zero. + */ +ORT_API_STATUS(OrtSessionOptionsAppendExecutionProvider_CUDA, _In_ OrtSessionOptions* options, int device_id); + +/* + * This is the old way to add the ROCm provider to the session, please use + * SessionOptionsAppendExecutionProvider_ROCM above to access the latest functionality + * This function always exists, but will only succeed if Onnxruntime was built with + * HIP support and the ROCm provider shared library exists + * + * \param device_id HIP device id, starts from zero. + */ +ORT_API_STATUS(OrtSessionOptionsAppendExecutionProvider_ROCM, _In_ OrtSessionOptions* options, int device_id); + +/* + * This is the old way to add the MIGraphX provider to the session, please use + * SessionOptionsAppendExecutionProvider_MIGraphX above to access the latest functionality + * This function always exists, but will only succeed if Onnxruntime was built with + * HIP support and the MIGraphX provider shared library exists + * + * \param device_id HIP device id, starts from zero. + */ +ORT_API_STATUS(OrtSessionOptionsAppendExecutionProvider_MIGraphX, _In_ OrtSessionOptions* options, int device_id); + +/* + * This is the old way to add the oneDNN provider to the session, please use + * SessionOptionsAppendExecutionProvider_oneDNN above to access the latest functionality + * This function always exists, but will only succeed if Onnxruntime was built with + * oneDNN support and the oneDNN provider shared library exists + * + * \param use_arena zero: false. non-zero: true. + */ +ORT_API_STATUS(OrtSessionOptionsAppendExecutionProvider_Dnnl, _In_ OrtSessionOptions* options, int use_arena); + +/* + * This is the old way to add the TensorRT provider to the session, please use SessionOptionsAppendExecutionProvider_TensorRT_V2 above to access the latest functionality + * This function always exists, but will only succeed if Onnxruntime was built with TensorRT support and the TensorRT provider shared library exists + * + * \param device_id CUDA device id, starts from zero. + */ +ORT_API_STATUS(OrtSessionOptionsAppendExecutionProvider_Tensorrt, _In_ OrtSessionOptions* options, int device_id); + +#ifdef __cplusplus +} +#endif +/// @} + +#include "onnxruntime_ep_c_api.h" diff --git a/go/onnxruntime/onnxruntime_ep_c_api.h b/go/onnxruntime/onnxruntime_ep_c_api.h new file mode 100644 index 0000000000000..683528304f7a0 --- /dev/null +++ b/go/onnxruntime/onnxruntime_ep_c_api.h @@ -0,0 +1,3102 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +// Do not include this file directly. Please include "onnxruntime_c_api.h" instead. + +#if defined(__DOXYGEN__) +// When running a Doxygen build, include onnxruntime_c_api.h. Doxygen expects header files to be self-contained. +#include "onnxruntime_c_api.h" +#else +// In normal usage, do not include onnxruntime_c_api.h. This file is explicitly included in onnxruntime_c_api.h. +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +/** \addtogroup Global + * @{ + */ +ORT_RUNTIME_CLASS(Ep); +ORT_RUNTIME_CLASS(EpFactory); +ORT_RUNTIME_CLASS(EpGraphSupportInfo); +ORT_RUNTIME_CLASS(MemoryDevice); // opaque class to wrap onnxruntime::OrtDevice +ORT_RUNTIME_CLASS(NodeComputeContext); + +ORT_RUNTIME_CLASS(DataTransferImpl); +ORT_RUNTIME_CLASS(SyncNotificationImpl); +ORT_RUNTIME_CLASS(SyncStreamImpl); + +ORT_RUNTIME_CLASS(ExternalResourceImporterImpl); + +ORT_RUNTIME_CLASS(OpSchema); +ORT_RUNTIME_CLASS(OpSchemaTypeConstraint); +ORT_RUNTIME_CLASS(ProfilingEventsContainer); +ORT_RUNTIME_CLASS(ProfilingEvent); // Based on the Trace Event Format's "complete event" +/// @} + +/** \brief Base struct for imported external memory handles. + * + * EPs derive from this struct to add EP-specific fields (e.g., CUdeviceptr for CUDA). + * EP is responsible for creating and releasing instances of the derived type. + * + * Example derived type for CUDA EP: + * \code + * struct MyCudaExternalMemoryHandle : OrtExternalMemoryHandle { + * CUexternalMemory ext_memory; + * CUdeviceptr mapped_ptr; + * bool is_dedicated; + * }; + * \endcode + * + * \since Version 1.24. + */ +struct OrtExternalMemoryHandle { + uint32_t version; ///< Must be ORT_API_VERSION + const OrtEpDevice* ep_device; ///< EP device that created this handle + OrtExternalMemoryDescriptor descriptor; ///< External memory descriptor + + /** \brief Release callback for this handle. EP sets this to its release function. + * + * ORT calls this when ReleaseExternalMemoryHandle is invoked. The EP's callback + * should cast the handle to its derived type and delete it. + */ + void(ORT_API_CALL* Release)(_In_ OrtExternalMemoryHandle* handle); +}; + +/** \brief Base struct for imported external semaphore handles. + * + * EPs derive from this struct to add EP-specific fields (e.g., CUexternalSemaphore for CUDA). + * EP is responsible for creating and releasing instances of the derived type. + * + * Example derived type for CUDA EP: + * \code + * struct MyCudaExternalSemaphoreHandle : OrtExternalSemaphoreHandle { + * CUexternalSemaphore ext_semaphore; + * }; + * \endcode + * + * \since Version 1.24. + */ +struct OrtExternalSemaphoreHandle { + uint32_t version; ///< Must be ORT_API_VERSION + const OrtEpDevice* ep_device; ///< EP device that created this handle + OrtExternalSemaphoreDescriptor descriptor; ///< External semaphore descriptor + + /** \brief Release callback for this handle. EP sets this to its release function. + * + * ORT calls this when ReleaseExternalSemaphoreHandle is invoked. The EP's callback + * should cast the handle to its derived type and delete it. + */ + void(ORT_API_CALL* Release)(_In_ OrtExternalSemaphoreHandle* handle); +}; + +// Opaque types for kernel-based EPs +ORT_RUNTIME_CLASS(KernelRegistry); +ORT_RUNTIME_CLASS(KernelDefBuilder); +ORT_RUNTIME_CLASS(KernelDef); +ORT_RUNTIME_CLASS(DataType); // combination of ONNXType (e.g., Tensor, Map, Sequence) and ONNXTensorElementDataType +ORT_RUNTIME_CLASS(SharedPrePackedWeightCache); + +/** \brief Struct that an EP implements for IDataTransfer to copy between devices it uses and CPU. + * + * \since Version 1.23. + */ +struct OrtDataTransferImpl { + uint32_t ort_version_supported; ///< Must be initialized to ORT_API_VERSION + + /** \brief Release the OrtDataTransferImpl instance. + * + * This is called by ORT when the OrtDataTransferImpl instance is no longer needed. + * The implementation should release any resources held by the instance. + * + * \param[in] this_ptr Pointer to the OrtDataTransferImpl instance. + * + * \since Version 1.23. + */ + ORT_API_T(void, Release, _In_ OrtDataTransferImpl* this_ptr); + + /** \brief Check if the implementation can copy between the source and destination memory devices. + * + * \param[in] this_ptr Pointer to the OrtDataTransferImpl instance. + * \param[in] src_memory_device Source OrtMemoryDevice to copy from. + * \param[in] dst_memory_device Destination OrtMemoryDevice to copy to. + * \return True if the implementation can copy between the devices. + * + * \since Version 1.23. + */ + ORT_API_T(bool, CanCopy, _In_ const OrtDataTransferImpl* this_ptr, + _In_ const OrtMemoryDevice* src_memory_device, _In_ const OrtMemoryDevice* dst_memory_device); + + /** \brief Copy tensors from src_tensors to dst_tensors using the provided streams. + * + * The implementation can use the provided streams to perform asynchronous copies if supported. + * If a stream is not available, the copy is performed synchronously. + * + * \param[in] this_ptr Pointer to the OrtDataTransferImpl instance. + * \param[in] src_tensors Array of source OrtValue pointers to copy from. + * \param[in] dst_tensors Array of destination OrtValue pointers to copy to. + * \param[in] streams Array of OrtSyncStream pointers for the copy operations, if the execution provider is stream + * aware. nullptr if it is not. + * \param[in] num_tensors Number of tensors to copy. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(CopyTensors, _In_ OrtDataTransferImpl* this_ptr, + _In_reads_(num_tensors) const OrtValue** src_tensors, + _In_reads_(num_tensors) OrtValue** dst_tensors, + _In_reads_(num_tensors) OrtSyncStream** streams, + _In_ size_t num_tensors); +}; + +/** \brief Struct that an EP implements for Stream Notifications. + * + * \since Version 1.23. + */ +struct OrtSyncNotificationImpl { + uint32_t ort_version_supported; ///< Must be initialized to ORT_API_VERSION + + /** \brief Release the OrtSyncNotificationImpl instance. + * + * This is called by ORT when the OrtSyncNotificationImpl instance is no longer needed. + * The implementation should release any resources held by the instance. + * + * \param[in] this_ptr Pointer to the OrtSyncNotificationImpl instance. + * + * \since Version 1.23. + */ + ORT_API_T(void, Release, _In_ OrtSyncNotificationImpl* this_ptr); + + /** \brief Called by ORT to activate the notification. + * + * \param[in] this_ptr Pointer to the OrtSyncNotificationImpl instance. + * + * \since Version 1.23. + */ + ORT_API2_STATUS(Activate, _In_ OrtSyncNotificationImpl* this_ptr); + + /** \brief Wait for a device to device operation to complete. + * + * \param[in] this_ptr Pointer to the OrtSyncNotificationImpl instance. + * \param[in] consumer_stream The OrtSyncStream instance that will wait on this notification to be activated. + * + * \since Version 1.23. + */ + ORT_API2_STATUS(WaitOnDevice, _In_ OrtSyncNotificationImpl* this_ptr, _In_ OrtSyncStream* consumer_stream); + + /** \brief Wait for a device to host operation to complete. + * + * \param[in] this_ptr Pointer to the OrtSyncNotificationImpl instance. + * + * \since Version 1.23. + */ + ORT_API2_STATUS(WaitOnHost, _In_ OrtSyncNotificationImpl* this_ptr); +}; + +/** \brief Struct that an EP implements if it wishes to implement Stream support. + * + * This struct provides the overrides for onnxruntime::Stream's virtual methods. + * + * \since Version 1.23. + */ +struct OrtSyncStreamImpl { + uint32_t ort_version_supported; ///< Must be initialized to ORT_API_VERSION + + /** \brief Release the OrtSyncStreamImpl instance. + * + * This is called by ORT when the OrtSyncStreamImpl instance is no longer needed. + * The implementation should release any resources held by the instance. + * + * \param[in] this_ptr Pointer to the OrtSyncStreamImpl instance. + * + * \since Version 1.23. + */ + ORT_API_T(void, Release, _In_ OrtSyncStreamImpl* this_ptr); + + /** \brief Get the handle of the stream. + * + * This returns the native handle for the stream. e.g. cudaStream_t for CUDA streams. + * + * \param[in] this_ptr Pointer to the OrtSyncStreamImpl instance. + * \return The handle of the stream. + * + * \since Version 1.23. + */ + ORT_API_T(void*, GetHandle, _In_ OrtSyncStreamImpl* this_ptr); + + /** \brief Create an OrtSyncNotificationImpl for the OrtSyncStreamImpl instance. + * + * \param[in] this_ptr Pointer to the OrtSyncStreamImpl instance + * \param[out] notification The new OrtSyncNotificationImpl instance. + * + * \since Version 1.23. + */ + ORT_API2_STATUS(CreateNotification, _In_ OrtSyncStreamImpl* this_ptr, + _Outptr_ OrtSyncNotificationImpl** notification); + + /** \brief Flush the stream. + * + * This is called by ORT to flush the stream, ensuring that all operations submitted to the stream are completed. + * + * \param[in] this_ptr Pointer to the OrtSyncStreamImpl instance. + * + * \since Version 1.23. + */ + ORT_API2_STATUS(Flush, _In_ OrtSyncStreamImpl* this_ptr); + + /** \brief Notify the stream that a session run has ended. + * + * This is called by ORT to notify the stream that a session run has ended, allowing the stream to perform any + * necessary cleanup or finalization. + * + * \param[in] this_ptr Pointer to the OrtSyncStreamImpl instance. + * + * \since Version 1.23. + */ + ORT_API2_STATUS(OnSessionRunEnd, _In_ OrtSyncStreamImpl* this_ptr); +}; + +/** \brief Struct that an EP implements for external resource import (memory + semaphore import). + * + * This capability object provides methods for importing external GPU memory and semaphores + * for zero-copy import. EPs that support D3D12, CUDA, HIP, or Vulkan external resource APIs + * can implement this interface. + * + * \since Version 1.24. + */ +struct OrtExternalResourceImporterImpl { + uint32_t ort_version_supported; ///< Must be initialized to ORT_API_VERSION + + // Memory operations (stream-independent) + + /** \brief Check if the implementation can import external memory of the given handle type. + * + * \param[in] this_ptr Pointer to the OrtExternalResourceImporterImpl instance. + * \param[in] handle_type The type of external memory handle to check. + * \return True if the handle type is supported. + * + * \since Version 1.24. + */ + ORT_API_T(bool, CanImportMemory, + _In_ const OrtExternalResourceImporterImpl* this_ptr, + _In_ OrtExternalMemoryHandleType handle_type); + + /** \brief Import external memory. + * + * The EP creates a derived type of OrtExternalMemoryHandle and returns a pointer to the base. + * EP is responsible for the lifetime of the handle (release via ReleaseMemory). + * + * \param[in] this_ptr Pointer to the OrtExternalResourceImporterImpl instance. + * \param[in] desc Descriptor containing the external memory handle and properties. + * \param[out] out_handle Output parameter set to the created OrtExternalMemoryHandle (EP's derived type). + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24. + */ + ORT_API2_STATUS(ImportMemory, + _In_ OrtExternalResourceImporterImpl* this_ptr, + _In_ const OrtExternalMemoryDescriptor* desc, + _Outptr_ OrtExternalMemoryHandle** out_handle); + + /** \brief Release an imported external memory handle. + * + * The EP deletes its derived type instance. + * + * \param[in] this_ptr Pointer to the OrtExternalResourceImporterImpl instance. + * \param[in] handle The OrtExternalMemoryHandle to release (EP casts to its derived type). + * + * \since Version 1.24. + */ + ORT_API_T(void, ReleaseMemory, + _In_ OrtExternalResourceImporterImpl* this_ptr, + _In_ OrtExternalMemoryHandle* handle); + + /** \brief Create a tensor backed by imported external memory. + * + * The created tensor is a view over the imported memory and does not copy data. + * + * \param[in] this_ptr Pointer to the OrtExternalResourceImporterImpl instance. + * \param[in] mem_handle The imported external memory handle (EP casts to its derived type). + * \param[in] tensor_desc Descriptor specifying tensor element type, shape, and optional offset. + * \param[out] out_tensor Output parameter set to the created OrtValue containing the tensor. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24. + */ + ORT_API2_STATUS(CreateTensorFromMemory, + _In_ OrtExternalResourceImporterImpl* this_ptr, + _In_ const OrtExternalMemoryHandle* mem_handle, + _In_ const OrtExternalTensorDescriptor* tensor_desc, + _Outptr_ OrtValue** out_tensor); + + // Semaphore operations (require stream) + + /** \brief Check if the implementation can import external semaphores of the given type. + * + * \param[in] this_ptr Pointer to the OrtExternalResourceImporterImpl instance. + * \param[in] type The type of external semaphore to check. + * \return True if the semaphore type is supported. + * + * \since Version 1.24. + */ + ORT_API_T(bool, CanImportSemaphore, + _In_ const OrtExternalResourceImporterImpl* this_ptr, + _In_ OrtExternalSemaphoreType type); + + /** \brief Import an external semaphore. + * + * The EP creates a derived type of OrtExternalSemaphoreHandle and returns a pointer to the base. + * EP is responsible for the lifetime of the handle (release via ReleaseSemaphore). + * + * \param[in] this_ptr Pointer to the OrtExternalResourceImporterImpl instance. + * \param[in] desc Descriptor containing the external semaphore handle and type. + * \param[out] out_handle Output parameter set to the created OrtExternalSemaphoreHandle (EP's derived type). + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24. + */ + ORT_API2_STATUS(ImportSemaphore, + _In_ OrtExternalResourceImporterImpl* this_ptr, + _In_ const OrtExternalSemaphoreDescriptor* desc, + _Outptr_ OrtExternalSemaphoreHandle** out_handle); + + /** \brief Release an imported external semaphore handle. + * + * The EP deletes its derived type instance. + * + * \param[in] this_ptr Pointer to the OrtExternalResourceImporterImpl instance. + * \param[in] handle The OrtExternalSemaphoreHandle to release (EP casts to its derived type). + * + * \since Version 1.24. + */ + ORT_API_T(void, ReleaseSemaphore, + _In_ OrtExternalResourceImporterImpl* this_ptr, + _In_ OrtExternalSemaphoreHandle* handle); + + /** \brief Wait on an external semaphore on the EP's stream. + * + * Inserts a wait operation into the EP's stream that blocks until the semaphore + * reaches the specified value. + * + * \param[in] this_ptr Pointer to the OrtExternalResourceImporterImpl instance. + * \param[in] handle The imported external semaphore (EP casts to its derived type). + * \param[in] stream The OrtSyncStream to wait on. + * \param[in] value The fence/semaphore value to wait for. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24. + */ + ORT_API2_STATUS(WaitSemaphore, + _In_ OrtExternalResourceImporterImpl* this_ptr, + _In_ OrtExternalSemaphoreHandle* handle, + _In_ OrtSyncStream* stream, + _In_ uint64_t value); + + /** \brief Signal an external semaphore from the EP's stream. + * + * Inserts a signal operation into the EP's stream that sets the semaphore + * to the specified value when reached. + * + * \param[in] this_ptr Pointer to the OrtExternalResourceImporterImpl instance. + * \param[in] handle The imported external semaphore (EP casts to its derived type). + * \param[in] stream The OrtSyncStream to signal from. + * \param[in] value The fence/semaphore value to signal. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24. + */ + ORT_API2_STATUS(SignalSemaphore, + _In_ OrtExternalResourceImporterImpl* this_ptr, + _In_ OrtExternalSemaphoreHandle* handle, + _In_ OrtSyncStream* stream, + _In_ uint64_t value); + + // Release the capability object itself + + /** \brief Release the OrtExternalResourceImporterImpl instance. + * + * This is called by ORT when the OrtExternalResourceImporterImpl instance is no longer needed. + * The implementation should release any resources held by the instance. + * + * \param[in] this_ptr Pointer to the OrtExternalResourceImporterImpl instance. + * + * \since Version 1.24. + */ + ORT_API_T(void, Release, _In_ OrtExternalResourceImporterImpl* this_ptr); +}; + +/** \brief The event category for profiling events reported by an execution provider. + * + * \since Version 1.25. + */ +typedef enum OrtProfilingEventCategory { + OrtProfilingEventCategory_SESSION = 0, ///< Session-level event + OrtProfilingEventCategory_NODE = 1, ///< Node-level event + OrtProfilingEventCategory_KERNEL = 2, ///< Kernel-level event + OrtProfilingEventCategory_API = 3, ///< API-level event +} OrtProfilingEventCategory; + +struct OrtEpProfilerImpl; +typedef struct OrtEpProfilerImpl OrtEpProfilerImpl; + +/** \brief Struct that an EP implements for profiling support. + * + * An execution provider optionally implements this struct to participate in ONNX Runtime's profiling system. + * The EP creates and returns an instance of this struct via OrtEp::CreateProfiler. + * + * ORT calls the function pointers at appropriate times during a profiling session: + * - StartProfiling once when profiling begins. + * - [Optional] StartEvent / StopEvent around each ORT event (operator executions, session events, etc.). + * - EndProfiling once when profiling ends to collect EP events. + * - Release when ORT no longer needs the profiler. + * + * Profiling scenarios: + * - ORT session profiling: Captures session initialization and one or more runs with a single ORT session. + * - Enabled via OrtApi::EnableProfiling(session_options). + * - There is one ORT session, one OrtEp, and one OrtEpProfilerImpl + * - Concurrency notes: An application may use a single ORT session (and the single OrtEpProfilerImpl) to run + * multiple inferences concurrently. The OrtEpProfilerImpl::StartEvent and OrtEpProfilerImpl::StopEvent + * functions will be called with ORT events from multiple concurrent runs. The OrtEpProfilerImpl::EndProfiling + * function is expected to return all EP events from all runs with correct correlations with the original ORT + * events. + * - ORT run profiling: Captures events for a given run. An application can either enable session profiling or run + * profiling, but not both at the same time. + * - Enabled via OrtApi::RunOptionsEnableProfiling(run_options). + * - There is one ORT session, one OrtEp, and multiple OrtEpProfilerImpl instances (one per profiled run). + * - Concurrency notes: each OrtEpProfilerImpl only receives calls for its specific run. + * OrtEpProfilerImpl::EndProfiling must only return EP events for its specific run. + * + * \since Version 1.25. + */ +struct OrtEpProfilerImpl { + uint32_t ort_version_supported; ///< Must be initialized to ORT_API_VERSION. + + /** \brief Release the OrtEpProfilerImpl instance. + * + * Called by ORT when the profiler is no longer needed. + * The implementation should release any resources held by the instance. + * + * \param[in] this_ptr Pointer to the OrtEpProfilerImpl instance. + * + * \note Implementation of this function is required. + * + * \since Version 1.25. + */ + ORT_API_T(void, Release, _In_ OrtEpProfilerImpl* this_ptr); + + /** \brief Called when profiling starts. + * + * Allows the EP profiler to initialize profiling utilities and record the profiling start time. + * + * An EP profiler should record its own clock's current time when this function is called. This allows the EP to + * later compute ORT-relative event timestamps by combining `ep_profiling_start_offset_ns` with the EP's own + * elapsed time since this call. The formula is: + * + * event_timestamp_us = (ep_profiling_start_offset_ns + (ep_event_time_ns - ep_profiling_start_time_ns)) / 1000 + * + * where `ep_event_time_ns` and `ep_profiling_start_time_ns` are measured using the EP's own clock. + * + * \param[in] this_ptr Pointer to the OrtEpProfilerImpl instance. + * \param[in] ep_profiling_start_offset_ns The elapsed time in nanoseconds (using ORT's profiling clock) between + * ORT's profiling start and this call to StartProfiling. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \note An error OrtStatus returned from this function is logged by ORT (does not end execution). + * \note Implementation of this function is required. + * + * \since Version 1.25. + */ + ORT_API2_STATUS(StartProfiling, _In_ OrtEpProfilerImpl* this_ptr, _In_ int64_t ep_profiling_start_offset_ns); + + /** \brief Called when an ORT event (e.g., session initialization, node kernel execution, etc.) begins. + * + * ORT pairs every StartEvent call with a corresponding call to StopEvent with the same ORT event correlation ID. + * EP profiler implementations may use the calls to StartEvent and StopEvent to maintain a stack of ORT event + * correlation IDs that can be correlated with EP events (e.g., GPU kernel events). For example: + * + * OrtEpProfilerImpl::StartEvent(x) -> EP ort event stack: [x] <- top of stack + * [EP events are tagged with 'x'] + * OrtEpProfilerImpl::StartEvent(y) -> EP ort event stack: [x, y] <- top of stack + * [EP events are tagged with 'y'] + * OrtEpProfilerImpl::StopEvent(y) -> EP ort event stack: [x] <- top of stack + * OrtEpProfilerImpl::StartEvent(z) -> EP ort event stack: [x, z] <- top of stack + * [EP events are tagged with 'z'] + * OrtEpProfilerImpl::StopEvent(z) -> EP ort event stack: [x] <- top of stack + * OrtEpProfilerImpl::StopEvent(x) -> EP ort event stack: [ ] <- top of stack + * + * Tagging EP events with the ORT event correlation ID enables the EP to annotate its own events + * with metadata from the parent ORT event (e.g., operator name). + * + * \note **Threading:** For a given ORT event, StartEvent, the kernel's Compute() entry point, and StopEvent + * are all invoked on the same CPU thread (Compute() may asynchronously launch device work). + * Different ORT events may run on different threads (e.g., inter-op parallelism), so correlation + * stacks (e.g., for CUPTI) should use thread-local storage. + * + * \note The ORT event correlation ID is an absolute, epoch-based timestamp in microseconds. It is computed + * from the ORT event's start time using std::chrono::high_resolution_clock (platform-defined epoch). + * Because it is absolute rather than relative to profiling start, it is practically unique across + * concurrent profiling sessions within the same process (collisions require sub-microsecond event + * concurrency) and can be used directly as a correlation ID for EP profiling utilities + * (e.g., CUPTI or ROCTracer). + * + * \param[in] this_ptr Pointer to the OrtEpProfilerImpl instance. + * \param[in] ort_event_correlation_id Absolute, epoch-based correlation ID for the ORT event that is starting. + * The same value is passed to the corresponding StopEvent call. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \note An error OrtStatus returned from this function is logged by ORT (does not end execution). + * \note Implementation of this function is optional. If set to NULL, it is not called. + * + * \since Version 1.25. + */ + ORT_API2_STATUS(StartEvent, _In_ OrtEpProfilerImpl* this_ptr, _In_ uint64_t ort_event_correlation_id); + + /** \brief Called when a profiled ORT event (e.g., session initialization, node kernel execution, etc.) ends. + * + * ORT pairs every StartEvent call with a corresponding call to StopEvent with the same ORT event correlation ID. + * EP profiler implementations may use the calls to StartEvent and StopEvent to maintain a stack of ORT event + * correlation IDs that can be correlated with EP events (e.g., GPU kernel events). For example: + * + * OrtEpProfilerImpl::StartEvent(x) -> EP ort event stack: [x] <- top of stack + * [EP events are tagged with 'x'] + * OrtEpProfilerImpl::StartEvent(y) -> EP ort event stack: [x, y] <- top of stack + * [EP events are tagged with 'y'] + * OrtEpProfilerImpl::StopEvent(y) -> EP ort event stack: [x] <- top of stack + * OrtEpProfilerImpl::StartEvent(z) -> EP ort event stack: [x, z] <- top of stack + * [EP events are tagged with 'z'] + * OrtEpProfilerImpl::StopEvent(z) -> EP ort event stack: [x] <- top of stack + * OrtEpProfilerImpl::StopEvent(x) -> EP ort event stack: [ ] <- top of stack + * + * Tagging EP events with the ORT event correlation ID enables the EP to annotate its own events + * with metadata from the parent ORT event (e.g., operator name). + * + * \note **Threading:** For a given ORT event, StartEvent, the kernel's Compute() entry point, and StopEvent + * are all invoked on the same CPU thread (Compute() may asynchronously launch device work). + * Different ORT events may run on different threads (e.g., inter-op parallelism), so correlation + * stacks (e.g., for CUPTI) should use thread-local storage. + * + * \note The ORT event correlation ID is an absolute, epoch-based timestamp in microseconds. It is computed + * from the ORT event's start time using std::chrono::high_resolution_clock (platform-defined epoch). + * Because it is absolute rather than relative to profiling start, it is practically unique across + * concurrent profiling sessions within the same process (collisions require sub-microsecond event + * concurrency) and can be used directly as a correlation ID for EP profiling utilities + * (e.g., CUPTI or ROCTracer). + * + * \param[in] this_ptr Pointer to the OrtEpProfilerImpl instance. + * \param[in] ort_event_correlation_id Absolute, epoch-based correlation ID for the ORT event that is ending. + * The same value was passed to the corresponding StartEvent call. + * \param[in] ort_event Opaque pointer to the ORT profiling event. Valid only during this call. + * Use OrtEpApi accessor functions to read event fields. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \note An error OrtStatus returned from this function is logged by ORT (does not end execution). + * \note Implementation of this function is optional. If set to NULL, it is not called. + * + * \since Version 1.25. + */ + ORT_API2_STATUS(StopEvent, _In_ OrtEpProfilerImpl* this_ptr, _In_ uint64_t ort_event_correlation_id, + _In_ const OrtProfilingEvent* ort_event); + + /** \brief Called when profiling ends to collect the EP's new profiling events since the call to StartProfiling. + * + * An EP profiler converts its events to OrtProfilingEvent instances and adds them into the provided + * OrtProfilingEventsContainer container. Call OrtEpApi::CreateProfilingEvent to create a new + * OrtProfilingEvent instance. Then call OrtEpApi::ProfilingEventsContainer_AddEvents to add one or more + * events to the container. + * + * After this function returns, ORT appends the EP's events to the profiling timeline. + * + * \param[in] this_ptr The OrtEpProfilerImpl instance. + * \param[in] events_container Event container to which the EP profiler adds its new events. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \note An error OrtStatus returned from this function is logged by ORT (does not end execution). + * \note Implementation of this function is required. + * + * \since Version 1.25. + */ + ORT_API2_STATUS(EndProfiling, _In_ OrtEpProfilerImpl* this_ptr, + _In_ OrtProfilingEventsContainer* events_container); +}; + +struct OrtNodeFusionOptions; +typedef struct OrtNodeFusionOptions OrtNodeFusionOptions; + +struct OrtNodeComputeInfo; +typedef struct OrtNodeComputeInfo OrtNodeComputeInfo; + +/** + * \brief The OrtNodeFusionOptions struct specifies options for fusing nodes supported by an execution provider. + * + * Refer to OrtEpApi::EpGraphSupportInfo_AddNodesToFuse. + * + * \since Version 1.23. + */ +struct OrtNodeFusionOptions { + /** \brief The ONNX Runtime version the OrtNodeFusionOptions was compiled with. + * + * Implementation should set to ORT_API_VERSION. + * ORT will use this to ensure it does not use members that were not available when the EP library was compiled. + * + * \since Version 1.23. + */ + uint32_t ort_version_supported; + + /** \brief If set to true, specify that the execution provider does not require ONNX Runtime to provide constant + * initializers as inputs to the fused node during model inference. This is used when the execution + * provider saves a copy of constant initializers, and allows ONNX Runtime to release constant initializers that + * are not used by any execution provider. + * + * If not specified, defaults to false. That is, ONNX Runtime provides constant initializers as inputs to + * the fused node by default. + * + * \since Version 1.23. + */ + bool drop_constant_initializers; + + // const OrtNode* fused_node_schema; +}; + +/** + * \brief The OrtNodeComputeInfo struct provides functions that an OrtEp implements to specify the compute + * function for a compiled OrtGraph instance. + * \since Version 1.23. + */ +struct OrtNodeComputeInfo { + /** \brief The ONNX Runtime version the OrtNodeComputeInfo was compiled with. + * + * Implementation should set to ORT_API_VERSION. + * ORT will use this to ensure it does not call functions that were not available when the EP library was compiled. + * + * \since Version 1.23. + */ + uint32_t ort_version_supported; + + /** \brief Creates an opaque compute state object that is then passed to the Compute() function during inference. + * \param[in] this_ptr The OrtNodeComputeInfo instance. + * \param[in] compute_context OrtNodeComputeContext instance that contains compiled/fused node's name and host + * memory allocation functions. Can optionally be used to build the compute state. + * \param[out] compute_state Output parameter that is assigned the opaque computation state. ONNX Runtime calls + * ReleaseState() (after calling Compute()) to allow the implementer to release the + * compute state. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + OrtStatus*(ORT_API_CALL* CreateState)(_In_ OrtNodeComputeInfo* this_ptr, + _In_ OrtNodeComputeContext* compute_context, + _Outptr_ void** compute_state); + + /** \brief Computation function called to execute the fused node compiled by an OrtEp instance. + * \param[in] this_ptr The OrtNodeComputeInfo instance. + * \param[in] compute_state The opaque computation state returned by CreateState(). + * \param[in] kernel_context The OrtKernelContext instance used to access inputs/outputs. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + OrtStatus*(ORT_API_CALL* Compute)(_In_ OrtNodeComputeInfo* this_ptr, _In_ void* compute_state, + _In_ OrtKernelContext* kernel_context); + + /** \brief Releases the compute state returned by CreateState(). + * \param[in] this_ptr The OrtNodeComputeInfo instance. + * \param[inout] compute_state The opaque compute state returned by CreateState(). + * + * \since Version 1.23. + */ + void(ORT_API_CALL* ReleaseState)(_In_ OrtNodeComputeInfo* this_ptr, _Frees_ptr_opt_ void* compute_state); +}; + +struct OrtKernelImpl; +typedef struct OrtKernelImpl OrtKernelImpl; + +/** + * \brief Contains functions that an OrtEp implements to specify the computation for an operator kernel. + * \since Version 1.24. + */ +struct OrtKernelImpl { + uint32_t ort_version_supported; ///< Must be initialized to ORT_API_VERSION + uint32_t flags; ///< EP must initialize to 0. Used internally by ORT. + + /** \brief Computation function called to execute the kernel on an EP. + * + * \note Implementation of this function is required. + * + * \param[in] this_ptr The OrtKernelImpl instance. + * \param[in] context The OrtKernelContext instance that provides access to the inputs and outputs. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24. + */ + ORT_API2_STATUS(Compute, _In_ OrtKernelImpl* this_ptr, _In_ OrtKernelContext* context); + + /** \brief Called by ORT to release the OrtKernelImpl instance and its resources. + * + * \note Implementation of this function is required. + * + * \param[in] this_ptr The OrtKernelImpl instance. + * + * \since Version 1.24. + */ + ORT_API_T(void, Release, _In_ OrtKernelImpl* this_ptr); + + /** \brief Optional function to pre-pack a constant tensor (i.e., a weight) to the kernel's preferred data layout. + * + * For example, a Conv kernel can define this function to pack input W to the channel-last data layout + * before inference. + * + * Pre-packing can operate in three different modes: no pre-packing mode, sharing mode, and non-sharing mode. + * 1) No pre-packing mode: The kernel can forgo any weight pre-packing for the given `input_index` by setting + * `is_packed` to false and returning a successful OrtStatus. In this mode, the kernel's + * OrtKernelImpl::SetSharedPrePackedWeight() function is not called for that specific + * `input_index`. + * 2) Sharing mode: Sharing is allowed if the `prepacked_weight_cache` argument is not NULL and the EP stores + * weight data in CPU-accessible memory. In this case, the kernel can optionally choose + * to share the packed weight with other kernels that use the same weight + * (compared by content hash). To do so, the kernel must allocate the packed weight with the + * provided `allocator`, then it stores the packed weight data into `prepacked_weight_cache` + * via SharedPrePackedWeightCache_StoreWeightData(), sets `is_packed` to true, and returns a + * successful OrtStatus. ORT will subsequently call OrtKernelImpl::SetSharedPrePackedWeight() + * to provide this kernel with the actual shared weight data, whose memory location could + * differ (i.e., if shared data was allocated by a previously processed kernel). + * 3) Non-sharing mode: In non-sharing mode, the `prepacked_weight_cache` argument is ignored. In this mode, + * the implementation allocates the packed data with the provided `allocator`, sets + * `is_packed` to true, and returns a successful OrtStatus. The kernel is ultimately + * responsible for releasing the packed data for the weight with `allocator`. + * ORT may release the original (unpacked) weight, which must not be accessed in + * OrtKernelImpl::Compute(). Note that in this mode, the kernel's + * OrtKernelImpl::SetSharedPrePackedWeight() function is not called by ORT for that specific + * `input_index`. + * + * \note This function is based on the internal OpKernel::PrePack() virtual function used within ORT. + * + * \param[in] this_ptr The OrtKernelImpl instance. + * \param[in] tensor The OrtValue instance representing the constant tensor (weight). Do not cache in the kernel. + * \param[in] input_index The input index of the tensor in this kernel. + * \param[in] allocator Allocator for allocating the pre-packed data. Its use is required in sharing mode and + * recommended, but not required, in the non-sharing mode. This will be an allocator set by + * the application for the session/environment (e.g., via CreateAndRegisterAllocator[V2] + * or RegisterAllocator), or an allocator on the OrtEpDevice (read-only or default) otherwise. + * The allocator remains valid throughout the lifetime of the OrtKernelImpl instance. + * \param[in] prepacked_weight_cache May be NULL. If not NULL, the kernel may choose to share a packed weight by + * first storing it in the OrtSharedPrePackedWeightCache instance and then + * receiving the actual shared weight data in the call to + * OrtKernelImpl::SetSharedPrePackedWeight(). See the above description for + * "sharing mode". + * \param[out] is_packed Output parameter that the implementation sets to true if the kernel packed the tensor data. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \note Implementation of this function is optional. If not implemented (set to NULL), ORT assumes the kernel + * does not pre-pack weight data (i.e., `is_packed` defaults to false). + * + * \since Version 1.24. + */ + ORT_API2_STATUS(PrePackWeight, _In_ OrtKernelImpl* this_ptr, _In_ const OrtValue* tensor, + _In_ int input_index, _Inout_ OrtAllocator* allocator, + _In_opt_ OrtSharedPrePackedWeightCache* prepacked_weight_cache, _Out_ bool* is_packed); + + /** \brief Optional function that receives data for a shared pre-packed weight from ORT. + * + * ORT calls this function after calling OrtKernelImpl::PrePackWeight for a specific `input_index` if: + * - OrtKernelImpl::PrePackWeight set the output parameter `is_packed` to true. + * - OrtKernelImpl::PrePackWeight stored weight data to share into the provided OrtSharedPrePackedWeightCache + * parameter (`prepacked_weight_cache`) via the API SharedPrePackedWeightCache_StoreWeightData. + * + * Refer to the description of the "sharing-mode" in the documentation for OrtKernelImpl::PrePackWeight(). + * + * \note ORT will not call this function for an `input_index` that a previous call to + * OrtKernelImpl::PrePackWeight() did not elect to pre-pack and share. + * + * \note This function is based on the internal OpKernel::UseSharedPrePackedBuffers() virtual function used + * within ORT. + * + * \param[in] this_ptr The OrtKernelImpl instance. + * \param[in] buffer_data_ptrs An array of buffer data pointers that collectively hold the pre-packed data for a + * single shared weight. The buffers are provided in the same order and with the same + * contents (in a potentially different memory location) as the buffers + * passed into SharedPrePackedWeightCache_StoreWeightData() within the + * OrtKernelImpl::PrePackWeight() call for the same `input_index`. + * \param[in] buffer_data_sizes An array of buffer byte sizes, one per element in `buffer_data_ptrs`. + * \param[in] num_buffers The number of buffers used to store the data for the shared pre-packed weight. + * Specifies the number of elements in the `buffer_data_ptrs` and `buffer_data_sizes` arrays. + * \param[in] input_index The input index of the tensor in this kernel. This index identifies the identity of + * the weight. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \note Implementation of this function is generally optional. It is only required if OrtKernelImpl::PrePack() + * elects to share pre-packed weights. + * + * \since Version 1.24. + */ + ORT_API2_STATUS(SetSharedPrePackedWeight, _In_ OrtKernelImpl* this_ptr, + _In_reads_(num_buffers) const void* const* buffer_data_ptrs, + _In_reads_(num_buffers) const size_t* buffer_data_sizes, + _In_ size_t num_buffers, _In_ int input_index); +}; + +/** \brief Type definition for a function that creates an OrtKernelImpl instance for an operator kernel. + * + * \param[in] kernel_create_func_state Opaque state initially provided by the EP that registered the kernel. + * Refer to OrtEpApi::KernelRegistry_AddKernel(). May be null. + * \param[in] info The OrtKernelInfo instance that provides access to the kernel's input and output characteristics. + * \param[out] kernel_out Output parameter set to the new OrtKernelImpl instance. On success, ownership of this + * OrtKernelImpl instance transfers to ORT, which will call OrtKernelImpl::Release() to + * release the instance when it is no longer used. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24. + */ +typedef OrtStatus*(ORT_API_CALL* OrtKernelCreateFunc)(_In_ void* kernel_create_func_state, + _In_ const OrtKernelInfo* info, + _Outptr_result_maybenull_ OrtKernelImpl** kernel_out); + +struct OrtLoopKernelHelper; +typedef struct OrtLoopKernelHelper OrtLoopKernelHelper; + +/** + * \brief Contains helper functions for a Loop OrtKernelImpl created via OrtEpApi::CreateLoopKernel. + * \since Version 1.24. + */ +struct OrtLoopKernelHelper { + uint32_t ort_version_supported; ///< Must be initialized to ORT_API_VERSION + + /** \brief Called by ORT to release the OrtLoopKernelHelper instance and its resources. + * + * \param[in] this_ptr The OrtLoopKernelHelper instance. + * + * \since Version 1.24. + */ + ORT_API_T(void, Release, _In_ OrtLoopKernelHelper* this_ptr); + + /** \brief Helper function that concatenates OrtValue instances from each loop iteration into a single + * pre-allocated output buffer. + * + * \note Implementing this function is required for all Loop opset versions. + * + * \param[in] this_ptr The OrtLoopKernelHelper instance. + * \param[in] stream_handle Optional native stream handle that enables asynchronous operations. May be NULL. + * \param[in] per_iteration_outputs Array of OrtValue instances from each iteration. All OrtValue elements have the + * same shape. + * \param[in] num_per_iteration_outputs The number of OrtValue* elements in the `per_iteration_outputs` array. + * \param[out] output The pre-allocated output buffer. Memory is allocated on the device for the EP running the + * Loop node. + * \param[in] output_size_in_bytes The size in bytes of the `output` buffer. It is guaranteed to be large enough + * to hold the concatenated data of each element in `per_iteration_outputs`. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24. + */ + ORT_API2_STATUS(ConcatOutput, _In_ OrtLoopKernelHelper* this_ptr, _In_opt_ void* stream_handle, + _In_reads_(num_per_iteration_outputs) const OrtValue* const* per_iteration_outputs, + _In_ size_t num_per_iteration_outputs, _Out_writes_bytes_all_(output_size_in_bytes) void* output, + _In_ size_t output_size_in_bytes); +}; + +struct OrtScanKernelHelper; +typedef struct OrtScanKernelHelper OrtScanKernelHelper; + +/** + * \brief Contains helper functions for a Scan OrtKernelImpl created via OrtEpApi::CreateScanKernel. + * \since Version 1.24. + */ +struct OrtScanKernelHelper { + uint32_t ort_version_supported; ///< Must be initialized to ORT_API_VERSION + + /** \brief Called by ORT to release the OrtScanKernelHelper instance and its resources. + * + * \param[in] this_ptr The OrtScanKernelHelper instance. + * + * \since Version 1.24. + */ + ORT_API_T(void, Release, _In_ OrtScanKernelHelper* this_ptr); + + /** \brief Helper function that transposes an OrtValue instance during execution of a Scan kernel. + * + * \note Called for Scan (opset >= 9) when the 'scan_input_axes' or 'scan_output_axes' attributes contain + * non-zero values. Implementing this function is required for Scan opset versions >= 9. + * + * \param[in] this_ptr The OrtScanKernelHelper instance. + * \param[in] permutation An array of integers that defines how the input tensor's axes should be permuted. + * \param[in] num_permutation_elems The number of integer elements in the `permutation` array. + * \param[in] input The input OrtValue tensor to transpose. + * \param[in] stream An optional OrtSyncStream instance to be used for asynchronous operations. May be NULL. + * \param[out] output The pre-allocated output OrtValue instance into which to store the results of the + * transpose operation. Must not be released as it is owned by ORT. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24. + */ + ORT_API2_STATUS(Transpose, _In_ OrtScanKernelHelper* this_ptr, + _In_reads_(num_permutation_elems) const size_t* permutation, _In_ size_t num_permutation_elems, + _In_ const OrtValue* input, _In_opt_ OrtSyncStream* stream, _Inout_ OrtValue* output); +}; + +/** + * \brief Discriminator for the resource count type stored in an OrtResourceCount. + * + * New resource accounting types can be added by appending new enum values. + * The OrtResourceCount union storage is large enough to hold all current and future types. + * + * \since Version 1.26. + */ +typedef enum OrtResourceCountKind { + OrtResourceCountKind_None = 0, ///< Unset / zero-cost sentinel. + OrtResourceCountKind_TotalBytes = 1, ///< Single uint64_t: byte count (cost or budget). +} OrtResourceCountKind; + +/** + * \brief ABI-stable tagged union representing a resource cost or budget. + * + * This struct is a C-safe variant that can be passed by value across the plugin DLL boundary. + * The `kind` field selects which member of the `value` union is active. The + * `value.reserved_words` storage reserves space for future resource types without changing + * the struct layout. + * + * Adding new resource types requires only: (a) a new OrtResourceCountKind enum value, + * (b) a new union member. No new C API functions are needed. + * + * \since Version 1.26. + */ +typedef struct OrtResourceCount { + uint32_t kind; /**< OrtResourceCountKind discriminator. */ + uint32_t reserved; /**< Must be zero. Ensures natural alignment for the value union. */ + + union { + uint64_t total_bytes; /**< Active when kind == OrtResourceCountKind_TotalBytes. */ + uint64_t reserved_words[6]; /**< 48 bytes fixed storage for future resource types. */ + } value; + +#ifdef __cplusplus + /** Default-construct a None (unset) resource count. */ + OrtResourceCount() noexcept : kind{OrtResourceCountKind_None}, reserved{0}, value{} {} + + /** Construct a zero/unset resource count. */ + static OrtResourceCount None() noexcept { + return OrtResourceCount{}; + } + + /** Construct a resource count representing total bytes. */ + static OrtResourceCount FromTotalBytes(uint64_t bytes) noexcept { + OrtResourceCount rc{}; + rc.kind = OrtResourceCountKind_TotalBytes; + rc.value.total_bytes = bytes; + return rc; + } + + /** Read the total_bytes value (caller must check kind first). */ + uint64_t AsTotalBytes() const noexcept { + return value.total_bytes; + } +#endif +} OrtResourceCount; + +#ifdef __cplusplus +static_assert(sizeof(OrtResourceCount) == 56, "OrtResourceCount size must not change to maintain ABI stability"); +#endif + +/** + * \brief The OrtEpApi struct provides functions that are relevant to the implementation of an execution provider. + * + * \since Version 1.22. + */ +struct OrtEpApi { + /** \brief Create an OrtEpDevice for the EP and an OrtHardwareDevice. + * \param[in] ep_factory Execution provider factory that is creating the instance. + * \param[in] hardware_device Hardware device that the EP can utilize. + * \param[in] ep_metadata Optional OrtKeyValuePairs instance for execution provider metadata that may be used + * during execution provider selection and passed to CreateEp. + * ep_device will copy this instance and the user should call ReleaseKeyValuePairs. + * \param[in] ep_options Optional OrtKeyValuePairs instance for execution provider options that will be added + * to the Session configuration options if the execution provider is selected. + * ep_device will copy this instance and the user should call ReleaseKeyValuePairs. + * \param ep_device OrtExecutionDevice that is created. + * + * \since Version 1.22. + */ + ORT_API2_STATUS(CreateEpDevice, _In_ OrtEpFactory* ep_factory, + _In_ const OrtHardwareDevice* hardware_device, + _In_opt_ const OrtKeyValuePairs* ep_metadata, + _In_opt_ const OrtKeyValuePairs* ep_options, + _Out_ OrtEpDevice** ep_device); + + ORT_CLASS_RELEASE(EpDevice); + + /** \brief Specify nodes that are supported by an OrtEp and should be fused into one node. + * + * Because the nodes will be fused into one "fused node", there must not exist an unsupported node in + * a path between two of the provided nodes. Otherwise, the graph will become invalid. + * + * This function can be called multiple times. A subsequent call to this function will force the next set of + * nodes to be fused into a different node. + * + * \param[in] graph_support_info OrtEpGraphSupportInfo instance to which to add the supported nodes. + * \param[in] nodes Array of nodes supported by the EP that should be fused/compiled. + * \param[in] num_nodes The number of supported nodes. + * \param[in] node_fusion_options Optional node fusion options. Ignored if set to NULL. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(EpGraphSupportInfo_AddNodesToFuse, _In_ OrtEpGraphSupportInfo* graph_support_info, + _In_reads_(num_nodes) const OrtNode* const* nodes, _In_ size_t num_nodes, + _In_opt_ const OrtNodeFusionOptions* node_fusion_options); + + /** \brief Specify a node that is supported by an OrtEp and should be run with a registered EP kernel. + * + * \param[in] graph_support_info OrtEpGraphSupportInfo instance to which to add the supported node. + * \param[in] node The supported OrtNode instance. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(EpGraphSupportInfo_AddSingleNode, _In_ OrtEpGraphSupportInfo* graph_support_info, + _In_ const OrtNode* node); + + /** \brief Query a OrtNodeComputeContext for the name of the node that encapsulates the compiled/fused node. + * + * Used in OrtNodeComputeInfo::CreateComputeState(). + * + * \param[in] context The OrtNodeComputeContext instance to query. + * \return The node's name. + * + * \note Returned string is owned by ORT and valid only while OrtNodeComputeInfo::CreateComputeState() is called. + * + * \since Version 1.23. + */ + ORT_API_T(const char*, NodeComputeContext_NodeName, _In_ const OrtNodeComputeContext* context); + + /** \brief Register an allocator with the OrtEpDevice. + * + * This allows an EP to provide OrtMemoryInfo for DEFAULT and HOST_ACCESSIBLE memory type as needed. + * The registered values will be used in calls to OrtEpFactory::CreateAllocator to ensure the required allocator/s + * are available for EP usage. + * + * Multiple calls for the same entry type will replace a previous entry. + * + * Available entries: + * - OrtDeviceAllocator with type of OrtDeviceMemoryType_DEFAULT + * - OrtDeviceAllocator with type of OrtDeviceMemoryType_HOST_ACCESSIBLE + * - OrtReadOnlyAllocator with type of OrtDeviceMemoryType_DEFAULT + * - if provided this allocator will only be used to copy initializers to the device the EP uses. + * ORT will use the OrtDeviceAllocator if not provided. + * + * \param[in] ep_device The OrtEpDevice instance to register the OrtMemoryInfo with. + * \param[in] allocator_memory_info The OrtMemoryInfo information for the allocator. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(EpDevice_AddAllocatorInfo, _In_ OrtEpDevice* ep_device, + _In_ const OrtMemoryInfo* allocator_memory_info); + + /** \brief Get the OrtMemoryDevice from an OrtMemoryInfo instance. + * + * This is required for OrtDataTransferImpl (which implements onnxruntime::IDataTransfer) where the OrtMemoryDevice + * is used in the CanCopy and CopyTensors functions. + * + * \param[in] memory_info The OrtMemoryInfo instance to get the memory device from. + * \return The OrtMemoryDevice associated with the OrtMemoryInfo instance. + * + * \since Version 1.23. + */ + ORT_API_T(const OrtMemoryDevice*, MemoryInfo_GetMemoryDevice, _In_ const OrtMemoryInfo* memory_info); + + /** \brief Get the OrtMemoryDevice from an OrtValue instance if it contains a Tensor. + * + * \param[in] value The OrtValue instance to get the memory device from. + * \return Memory device if OrtValue contains a Tensor, nullptr otherwise. + * + * \since Version 1.23. + */ + ORT_API_T(const OrtMemoryDevice*, Value_GetMemoryDevice, _In_ const OrtValue* value); + + /** \brief Compare two OrtMemoryDevice instances for equality. + * + * This is used to check if two memory devices are the same. + * Used to implement DataTransferImpl::CanCopy. + * + * \param[in] a The first OrtMemoryDevice instance to compare. + * \param[in] b The second OrtMemoryDevice instance to compare. + * \return True if the two OrtMemoryDevice instances are equal, false otherwise. + * + * \since Version 1.23. + */ + ORT_API_T(bool, MemoryDevice_AreEqual, _In_ const OrtMemoryDevice* a, _In_ const OrtMemoryDevice* b); + + /** \brief Get the OrtMemoryInfoDeviceType value from an OrtMemoryDevice instance. + * + * \param[in] memory_device OrtMemoryDevice instance. + * \return The OrtMemoryInfoDeviceType value. + * + * \since Version 1.23. + */ + ORT_API_T(OrtMemoryInfoDeviceType, MemoryDevice_GetDeviceType, _In_ const OrtMemoryDevice* memory_device); + + /** \brief Get the OrtDeviceMemoryType value from an OrtMemoryDevice instance. + * + * \param[in] memory_device OrtMemoryDevice instance. + * \return The OrtDeviceMemoryType value. + * + * \since Version 1.23. + */ + ORT_API_T(OrtDeviceMemoryType, MemoryDevice_GetMemoryType, _In_ const OrtMemoryDevice* memory_device); + + /** \brief Get the vendor ID from an OrtMemoryDevice instance. + * + * The vendor ID is used to identify the vendor of the device, and is typically set to the PCI vendor ID. + * + * If the device is not vendor specific (e.g. CPU memory) the vendor ID is set to 0. + * + * \param[in] memory_device OrtMemoryDevice instance. + * \return The vendor ID value. + * + * \since Version 1.23. + */ + ORT_API_T(uint32_t, MemoryDevice_GetVendorId, _In_ const OrtMemoryDevice* memory_device); + + /** \brief Get the device ID from an OrtMemoryDevice instance. + * + * \param[in] memory_device OrtMemoryDevice instance. + * \return The device ID. + * + * \since Version 1.23. + */ + ORT_API_T(uint32_t, MemoryDevice_GetDeviceId, _In_ const OrtMemoryDevice* memory_device); + + /** \brief Get the OrtSyncStreamImpl associated with an OrtSyncStream instance. + * + * This allows an the plugin library to connect its OrtSyncStreamImpl instance with an OrtSyncStream if needed. + * + * \param[in] stream The OrtSyncStream instance to find an OrtSyncStreamImpl for. + * \return The associated OrtSyncStreamImpl if found. nullptr otherwise. + * + * \since Version 1.23. + * + * \remarks There should always be an OrtSyncStreamImpl associated with an OrtSyncStream instance that the EP gets. + */ + ORT_API_T(const OrtSyncStreamImpl*, SyncStream_GetImpl, _In_ const OrtSyncStream* stream); + + /** \brief Get the current sync ID for a stream. + * + * \param[in] stream The OrtSyncStream to get the sync ID for. + * \return Current sync ID. + * + * \since Version 1.23. + */ + ORT_API_T(uint64_t, SyncStream_GetSyncId, _In_ const OrtSyncStream* stream); + + /** \brief Get the sync ID for the last time the consumer_stream waited on the producer_stream. + * + * When two streams are synchronized, the sync id represents the event used in that synchronization. + * + * \param[in] producer_stream The OrtSyncStream that produced the data. + * \param[in] consumer_stream The OrtSyncStream that waited on the producer_stream. + * \return ID for last sync. 0 if no sync has occurred between the two streams. + * + * \since Version 1.23. + */ + ORT_API_T(uint64_t, GetSyncIdForLastWaitOnSyncStream, + _In_ const OrtSyncStream* producer_stream, _In_ const OrtSyncStream* consumer_stream); + + /** \brief Create an OrtHardwareDevice. + * + * \note Called within OrtEpFactory::GetSupportedDevices to create a new hardware device (e.g., virtual). + * + * \param[in] type The hardware device type. + * \param[in] vendor_id The hardware device's vendor identifier. + * \param[in] device_id The hardware device's identifier. + * \param[in] vendor_name The hardware device's vendor name as a null-terminated string. Copied by ORT. + * \param[in] metadata Optional OrtKeyValuePairs instance for hardware device metadata that may be queried by + * applications via OrtApi::GetEpDevices(). + * Refer to onnxruntime_ep_device_ep_metadata_keys.h for common OrtHardwareDevice metadata keys. + * \param[out] hardware_device Output parameter set to the new OrtHardwareDevice instance that is created. + * Must be release with ReleaseHardwareDevice(). + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24. + */ + ORT_API2_STATUS(CreateHardwareDevice, _In_ OrtHardwareDeviceType type, + _In_ uint32_t vendor_id, + _In_ uint32_t device_id, + _In_ const char* vendor_name, + _In_opt_ const OrtKeyValuePairs* metadata, + _Out_ OrtHardwareDevice** hardware_device); + + ORT_CLASS_RELEASE(HardwareDevice); + + /** \brief Creates an empty kernel registry. A kernel registry contains kernel creation information for + * every operator kernel supported by an EP. + * + * \remarks Refer to OrtEp::GetKernelRegistry, which returns an EP's kernel registry to ORT. + * + * \param[out] kernel_registry Output parameter set to the new OrtKernelRegistry instance. + * Must be released with OrtEpApi::ReleaseKernelRegistry. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24. + */ + ORT_API2_STATUS(CreateKernelRegistry, _Outptr_ OrtKernelRegistry** kernel_registry); + + ORT_CLASS_RELEASE(KernelRegistry); + + /** \brief Adds kernel creation information for a supported operator kernel to the given kernel registry. + * + * \remarks Refer to OrtEp::GetKernelRegistry, which returns an EP's kernel registry to ORT. + * + * \param[in] kernel_registry The OrtKernelRegistry instance. + * \param[in] kernel_def The kernel definition, which includes operator type, version, EP name, type constraints, etc. + * \param[in] kernel_create_func Function that creates an instance of the operator kernel as a OrtKernelImpl instance. + * \param[in] kernel_create_func_state Custom state passed to the kernel creation function. Can be null. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24. + */ + ORT_API2_STATUS(KernelRegistry_AddKernel, _In_ OrtKernelRegistry* kernel_registry, + _In_ const OrtKernelDef* kernel_def, _In_ OrtKernelCreateFunc kernel_create_func, + _In_ void* kernel_create_func_state); + + /** \brief Creates a kernel definition builder used to create instances of OrtKernelDef. + * + * \param[out] kernel_def_builder_out Output parameter set to the new OrtKernelDefBuilder instance. + * Must be released with OrtEpApi::ReleaseKernelDefBuilder(). + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24. + */ + ORT_API2_STATUS(CreateKernelDefBuilder, _Outptr_ OrtKernelDefBuilder** kernel_def_builder_out); + + ORT_CLASS_RELEASE(KernelDefBuilder); + + /** \brief Sets the kernel's operator type. + * + * \param[in] kernel_def_builder The OrtKernelDefBuilder instance. + * \param[in] op_type A null-terminated string representing the operator type. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24. + */ + ORT_API2_STATUS(KernelDefBuilder_SetOperatorType, _In_ OrtKernelDefBuilder* kernel_def_builder, + _In_ const char* op_type); + + /** \brief Sets the kernel's domain. + * + * \param[in] kernel_def_builder The OrtKernelDefBuilder instance. + * \param[in] domain A null-terminated string representing the operator's domain. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24. + */ + ORT_API2_STATUS(KernelDefBuilder_SetDomain, _In_ OrtKernelDefBuilder* kernel_def_builder, _In_ const char* domain); + + /** \brief Sets the kernel's opset version range that is supported. + * + * \param[in] kernel_def_builder The OrtKernelDefBuilder instance. + * \param[in] since_version_start The starting opset version that is supported. + * \param[in] since_version_end The ending opset version (inclusive) that is supported. + * Can be set equal to the starting version to indicate that only one + * version is supported. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24. + */ + ORT_API2_STATUS(KernelDefBuilder_SetSinceVersion, _In_ OrtKernelDefBuilder* kernel_def_builder, + _In_ int since_version_start, _In_ int since_version_end); + + /** \brief Sets the name of the kernel's intended execution provider. + * + * \param[in] kernel_def_builder The OrtKernelDefBuilder instance. + * \param[in] ep_name A null-terminated string representing the execution provider's name. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24. + */ + ORT_API2_STATUS(KernelDefBuilder_SetExecutionProvider, _In_ OrtKernelDefBuilder* kernel_def_builder, + _In_ const char* ep_name); + + /** \brief Sets the memory type for a kernel input. + * + * \param[in] kernel_def_builder The OrtKernelDefBuilder instance. + * \param[in] input_index The index of the input. + * \param[in] mem_type The input's memory type. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24. + */ + ORT_API2_STATUS(KernelDefBuilder_SetInputMemType, _In_ OrtKernelDefBuilder* kernel_def_builder, + _In_ size_t input_index, _In_ OrtMemType mem_type); + + /** \brief Sets the memory type for a kernel output. + * + * \param[in] kernel_def_builder The OrtKernelDefBuilder instance. + * \param[in] output_index The index of the output. + * \param[in] mem_type The output's memory type. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24. + */ + ORT_API2_STATUS(KernelDefBuilder_SetOutputMemType, _In_ OrtKernelDefBuilder* kernel_def_builder, + _In_ size_t output_index, _In_ OrtMemType mem_type); + + /** \brief Adds type constraints for a kernel argument represented as a string (e.g., "T"). + * + * \param[in] kernel_def_builder The OrtKernelDefBuilder instance. + * \param[in] arg_name A null-terminated string representing the argument to constrain (e.g., "T"). + * \param[in] types Array of OrtDataType instances representing allowed types for the argument. + * Must contain `num_types` elements. + * \param[in] num_types The number of OrtDataType elements in the `types` array. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24. + */ + ORT_API2_STATUS(KernelDefBuilder_AddTypeConstraint, _In_ OrtKernelDefBuilder* kernel_def_builder, + _In_ const char* arg_name, _In_reads_(num_types) const OrtDataType* const* types, + _In_ size_t num_types); + + /** \brief Adds aliases for the given input and output pairs. + * + * \note Used for operators like Identity and Reshape to allow ORT to reuse the input buffer for the output + * without modification. + * + * \param[in] kernel_def_builder The OrtKernelDefBuilder instance. + * \param[in] input_indices Array of input indices. Array must contain `num_io_indices` elements. + * \param[in] output_indices Array of output indices. Each output index is aliased with a corresponding + * input index in `input_indices`. Array must contain `num_io_indices` elements. + * \param[in] num_io_indices The number of input/output index pairs to alias. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24. + */ + ORT_API2_STATUS(KernelDefBuilder_AddInputOutputAliases, _In_ OrtKernelDefBuilder* kernel_def_builder, + _In_reads_(num_io_indices) int const* input_indices, + _In_reads_(num_io_indices) int const* output_indices, + _In_ size_t num_io_indices); + + /** \brief Adds mutable aliases for the given input and output pairs. + * + * \note Allows ORT to reuse and *modify* an input buffer (in-place) for the output buffer. + * This is also known as "MayInplace" within the ORT codebase. + * + * \param[in] kernel_def_builder The OrtKernelDefBuilder instance. + * \param[in] input_indices Array of input indices. Array must contain `num_io_indices` elements. + * \param[in] output_indices Array of output indices. Each output index is aliased with a corresponding + * input index in `input_indices`. Array must contain `num_io_indices` elements. + * \param[in] num_io_indices The number of input/output index pairs to alias. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24. + */ + ORT_API2_STATUS(KernelDefBuilder_AddInputOutputMutableAliases, _In_ OrtKernelDefBuilder* kernel_def_builder, + _In_reads_(num_io_indices) int const* input_indices, + _In_reads_(num_io_indices) int const* output_indices, + _In_ size_t num_io_indices); + + /** \brief Creates a OrtKernelDef instance from the given kernel definition builder. + * + * \param[in] kernel_def_builder The OrtKernelDefBuilder instance. + * \param[out] kernel_def_out The new OrtKernelDef instance. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24. + */ + ORT_API2_STATUS(KernelDefBuilder_Build, _In_ OrtKernelDefBuilder* kernel_def_builder, + _Outptr_ OrtKernelDef** kernel_def_out); + + ORT_CLASS_RELEASE(KernelDef); + + /** \brief Returns the operator type from the kernel definition. + * + * \param[in] kernel_def The OrtKernelDef instance. + * \return A null-terminated string representing the operator type. + * + * \since Version 1.24. + */ + ORT_API_T(const char*, KernelDef_GetOperatorType, _In_ const OrtKernelDef* kernel_def); + + /** \brief Returns the operator's domain from the kernel definition. + * + * \param[in] kernel_def The OrtKernelDef instance. + * \return A null-terminated string representing the operator's domain. + * + * \since Version 1.24. + */ + ORT_API_T(const char*, KernelDef_GetDomain, _In_ const OrtKernelDef* kernel_def); + + /** \brief Gets the kernel's opset version range that is supported. + * + * \param[in] kernel_def The OrtKernelDef instance. + * \param[out] start_version Output parameter set to the starting opset version that is supported. + * \param[out] end_version Output parameter set to the ending opset version (inclusive) that is supported. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24. + */ + ORT_API2_STATUS(KernelDef_GetSinceVersion, _In_ const OrtKernelDef* kernel_def, + _Out_ int* start_version, _Out_ int* end_version); + + /** \brief Returns the name of the kernel's intended execution provider. + * + * \param[in] kernel_def The OrtKernelDef instance. + * \return A null-terminated string representing the name of the execution provider. + * + * \since Version 1.24. + */ + ORT_API_T(const char*, KernelDef_GetExecutionProvider, _In_ const OrtKernelDef* kernel_def); + + /** \brief Gets the memory type for a kernel input. + * + * \param[in] kernel_def The OrtKernelDef instance. + * \param[in] input_index The index of the input. + * \param[out] mem_type Output parameter set to the input's memory type. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24. + */ + ORT_API2_STATUS(KernelDef_GetInputMemType, _In_ const OrtKernelDef* kernel_def, + _In_ size_t input_index, _Out_ OrtMemType* mem_type); + + /** \brief Gets the memory type for a kernel output. + * + * \param[in] kernel_def The OrtKernelDef instance. + * \param[in] output_index The index of the output. + * \param[out] mem_type Output parameter set to the output's memory type. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24. + */ + ORT_API2_STATUS(KernelDef_GetOutputMemType, _In_ const OrtKernelDef* kernel_def, + _In_ size_t output_index, _Out_ OrtMemType* mem_type); + + /** \brief Gets the OrtDataType that represents the data type for a tensor of the given element type. + * + * \param[in] elem_type The tensor's element type. + * \param[out] out Output parameter set to the OrtDataType. Owned by ORT and must not be released. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24. + */ + ORT_API2_STATUS(GetTensorDataType, _In_ ONNXTensorElementDataType elem_type, + _Outptr_ const OrtDataType** out); + + /** \brief Gets the kernel definition for a given node, if any exists for the calling execution provider. + * + * Used within OrtEp::GetCapability() to get the registered kernel definition for the given node. + * The kernel definition is set to NULL if there is no registered kernel definition for the node + * and execution provider. + * + * \param[in] graph_support_info The OrtEpGraphSupportInfo instance to query. + * \param[in] node The node for which to look up a kernel definition. + * \param[out] out_kernel_def Output parameter set to the OrtKernelDef or NULL. + * Owned by ORT and must not be released. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24. + */ + ORT_API2_STATUS(EpGraphSupportInfo_LookUpKernel, _In_ OrtEpGraphSupportInfo* graph_support_info, + _In_ const OrtNode* node, _Outptr_result_maybenull_ const OrtKernelDef** out_kernel_def); + + /** \brief Sets one or more data buffers that collectively hold the pre-packed data for a single shared weight. + * + * \note Used within the implementation of OrtKernelImpl::PrePackWeight() when the kernel wants to share pre-packed + * weight data with other kernels. The buffer data MUST be allocated with the OrtAllocator provided to + * OrtKernelImpl::PrePack. + * + * \note Ownership of weight data transfers to the OrtSharedPrePackedWeightCache instance on success. + * If this function returns an error status, the caller retains ownership of the weight data. + * + * \note Subsequent calls with the same OrtSharedPrePackedWeightCache instance release and replace the old data. + * + * \param[in] prepacked_weight_cache The OrtSharedPrePackedWeightCache instance. + * \param[in] buffer_data_ptrs An array of buffer data pointers that collectively hold the pre-packed data for a + * single shared weight. Note that sometimes a single weight may have multiple pre-packed + * buffers and it is up to the kernel implementation to determine how to split the data + * into multiple buffers (if desired). + * \param[in] buffer_data_sizes An array of buffer byte sizes, one per element in `buffer_data_ptrs`. + * \param[in] num_buffers The number of buffers used to store the data for the shared pre-packed weight. + * Specifies the number of elements in the `buffer_data_ptrs` and `buffer_data_sizes` arrays. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24. + */ + ORT_API2_STATUS(SharedPrePackedWeightCache_StoreWeightData, + _In_ OrtSharedPrePackedWeightCache* prepacked_weight_cache, + _In_reads_(num_buffers) void** buffer_data_ptrs, _In_reads_(num_buffers) size_t* buffer_data_sizes, + _In_ size_t num_buffers); + + /** \brief Get the OrtEp instance to which the node is assigned from the OrtKernelInfo. + * + * \note Used within OrtKernelImpl implementations to obtain a reference to the OrtEp. + * + * \param[in] info The ::OrtKernelInfo instance. + * \param[out] ep Output parameter set to the OrtEp instance associated with the OrtKernelInfo. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * \since Version 1.24 + */ + ORT_API2_STATUS(KernelInfo_GetEp, _In_ const OrtKernelInfo* info, _Outptr_ const OrtEp** ep); + + /** \brief Set the details of an OrtDeviceEpIncompatibilityDetails instance. + * + * Used by execution provider factories to set incompatibility details in their + * GetHardwareDeviceIncompatibilityDetails implementation. ORT creates and initializes the object + * before passing it to the EP, so calling this function is optional. The EP uses this function + * to set incompatibility information when the device is not compatible. + * + * \param[in,out] details The OrtDeviceEpIncompatibilityDetails instance to update. + * \param[in] reasons_bitmask Bitmask of OrtDeviceEpIncompatibilityReason values. (0 = no incompatibility). + * \param[in] error_code Optional EP-specific error code (0 = no error). + * \param[in] notes Optional human-readable notes. Can be null. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24. + */ + ORT_API2_STATUS(DeviceEpIncompatibilityDetails_SetDetails, _Inout_ OrtDeviceEpIncompatibilityDetails* details, + _In_ uint32_t reasons_bitmask, + _In_ int32_t error_code, + _In_opt_z_ const char* notes); + + /** \brief Creates an OrtKernelImpl instance for an If operator. + * + * Control flow operators require access to ORT session internals to orchestrate subgraph operations. + * This function allows an EP to create a properly configured OrtKernelImpl with access to ORT internals that + * the EP can add to its kernel registry. + * + * An EP is required to create an OrtKernelDef that keeps input[0] ('cond') on the CPU (i.e., OrtMemTypeCPUInput) + * as this input is used by CPU logic. The output should remain on the device (i.e., OrtMemTypeDefault), which is + * the default setting, to avoid copying to/from CPU. + * + * Example kernel definition (CXX API): + * Ort::KernelDef kernel_def = Ort::KernelDefBuilder() + * .SetDomain("").SetOperatorType("If").SetSinceVersion(21, 22) + * .SetExecutionProvider("MyEp") + * .SetInputMemType(0, OrtMemTypeCPUInput) // 'cond' on CPU + * .SetOutputMemType(0, OrtMemTypeDefault) // output on EP device + * .AddTypeConstraint("B", ...) + * .AddTypeConstraint("V", ...).Build(); + * + * \param[in] kernel_info The ::OrtKernelInfo instance for an If node. This function returns error ORT_FAIL + * if the opset version specified by `kernel_info` is unsupported. + * \param[out] kernel_out Output parameter set to the OrtKernelImpl instance for the If node. + * Must be released via OrtEpApi::ReleaseKernelImpl, unless ownership is transferred + * to ORT (see OrtKernelCreateFunc and OrtEpApi::KernelRegistry_AddKernel). + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * \since Version 1.24 + */ + ORT_API2_STATUS(CreateIfKernel, _In_ const OrtKernelInfo* kernel_info, _Outptr_ OrtKernelImpl** kernel_out); + + /** \brief Creates an OrtKernelImpl instance for a Loop operator. + * + * Control flow operators require access to ORT session internals to orchestrate subgraph operations. + * This function allows an EP to create a properly configured OrtKernelImpl with access to ORT internals that + * the EP can add to its kernel registry. + * + * An EP is required to create an OrtKernelDef that keeps input[0] ('M') and input[1] ('cond') on the CPU + * (i.e., OrtMemTypeCPUInput) as these inputs are used by CPU logic. Input[2] ('v_initial') and the output should + * remain on the device (i.e., OrtMemTypeDefault), which is the default setting, to avoid copying to/from CPU. + * + * Example kernel definition (CXX API): + * Ort::KernelDef kernel_def = Ort::KernelDefBuilder() + * .SetDomain("").SetOperatorType("Loop").SetSinceVersion(21, 22) + * .SetExecutionProvider("MyEp") + * .SetInputMemType(0, OrtMemTypeCPUInput) // 'M' on CPU + * .SetInputMemType(1, OrtMemTypeCPUInput) // 'cond' on CPU + * .SetInputMemType(2, OrtMemTypeDefault) // 'v_initial' on EP device + * .SetOutputMemType(0, OrtMemTypeDefault) // output on EP device + * .AddTypeConstraint("I", ...) + * .AddTypeConstraint("B", ...) + * .AddTypeConstraint("V", ...).Build(); + * + * \param[in] kernel_info The ::OrtKernelInfo instance for a Loop node. This function returns error ORT_FAIL + * if the opset version specified by `kernel_info` is unsupported. + * \param[in] helper A OrtLoopKernelHelper instance that contains helper functions that ORT calls during kernel + * execution to operate on tensors allocated with the EP's device memory. + * ORT will call OrtLoopKernelHelper::Release() to release the helper and its resources. + * \param[out] kernel_out Output parameter set to the OrtKernelImpl instance for the Loop node. + * Must be released via OrtEpApi::ReleaseKernelImpl, unless ownership is transferred + * to ORT (see OrtKernelCreateFunc and OrtEpApi::KernelRegistry_AddKernel). + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * \since Version 1.24 + */ + ORT_API2_STATUS(CreateLoopKernel, _In_ const OrtKernelInfo* kernel_info, _In_ OrtLoopKernelHelper* helper, + _Outptr_ OrtKernelImpl** kernel_out); + + /** \brief Creates an OrtKernelImpl instance for a Scan operator. Does not support opset versions older than 9. + * + * Control flow operators require access to ORT session internals to orchestrate subgraph operations. + * This function allows an EP to create a properly configured OrtKernelImpl with access to ORT internals that + * the EP can add to its kernel registry. + * + * It is recommended that an EP create an OrtKernelDef that keeps the inputs and outputs on the EP's + * device (i.e., OrtMemTypeDefault), which is the default setting, to avoid copying to/from CPU. + * + * Example kernel definition (CXX API): + * Ort::KernelDef kernel_def = Ort::KernelDefBuilder() + * .SetDomain("").SetOperatorType("Scan").SetSinceVersion(21, 22) + * .SetExecutionProvider("MyEp") + * .SetInputMemType(0, OrtMemTypeDefault) // input[0] on EP device + * .SetOutputMemType(0, OrtMemTypeDefault) // output[0] on EP device + * .AddTypeConstraint("V", ...).Build(); + * + * \param[in] kernel_info The ::OrtKernelInfo instance for a Scan node. This function returns error ORT_FAIL + * if the opset version specified by `kernel_info` is unsupported. + * \param[in] helper A OrtScanKernelHelper instance that contains helper functions that ORT calls during kernel + * execution to operate on tensors allocated with the EP's device memory. + * ORT will call OrtScanKernelHelper::Release() to release the helper and its resources. + * \param[out] kernel_out Output parameter set to the OrtKernelImpl instance for the Scan node. + * Must be released via OrtEpApi::ReleaseKernelImpl, unless ownership is transferred + * to ORT (see OrtKernelCreateFunc and OrtEpApi::KernelRegistry_AddKernel). + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * \since Version 1.24 + */ + ORT_API2_STATUS(CreateScanKernel, _In_ const OrtKernelInfo* kernel_info, _In_ OrtScanKernelHelper* helper, + _Outptr_ OrtKernelImpl** kernel_out); + + ORT_CLASS_RELEASE(KernelImpl); + + /** \brief Gets a new OrtKeyValuePairs instance containing a copy of all configuration entries set on the environment. + * + * \note An application provides environment-level configuration options for execution provider libraries by + * using keys with the prefix 'ep_factory.\\.'. Ex: the key 'ep_factory.my_ep.some_ep_key' represents + * a key named 'some_ep_key' that is meant to be consumed by an execution provider named 'my_ep'. Refer to + * the specific execution provider's documentation for valid keys and values. + * + * \note Refer to onnxruntime_env_config_keys.h for common configuration entry keys and their supported values. + * + * \param[out] config_entries Output parameter set to the OrtKeyValuePairs instance containing all configuration entries. + * Must be released via OrtApi::ReleaseKeyValuePairs. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * \since Version 1.24 + */ + ORT_API2_STATUS(GetEnvConfigEntries, _Outptr_ OrtKeyValuePairs** config_entries); + + /** \brief Get an operator schema from the global schema registry. + * + * Looks up a schema by name, maximum inclusive version, and domain. + * The returned pointer is owned by the caller and must be released via ReleaseOpSchema. + * If the schema is not found, *out_schema is set to nullptr (no allocation occurs). + * + * Available schemas include standard ONNX operators (domain "" or "ai.onnx"), ONNX ML operators + * (domain "ai.onnx.ml"), and ORT contrib operators (domain "com.microsoft"). + * + * \param[in] name A null-terminated string for the operator name. + * \param[in] max_inclusive_version The maximum inclusive opset version. + * \param[in] domain A null-terminated string for the operator domain. + * \param[out] out_schema Output parameter set to the schema pointer, or nullptr if not found. + * Must be released via OrtEpApi::ReleaseOpSchema. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.25. + */ + ORT_API2_STATUS(GetOpSchema, _In_ const char* name, _In_ int max_inclusive_version, + _In_ const char* domain, _Outptr_result_maybenull_ OrtOpSchema** out_schema); + + ORT_CLASS_RELEASE(OpSchema); + + /** \brief Get the first ONNX opset version that introduced this operator schema. + * + * If an operator has had no changes that break backwards compatibility, the `since_version` is + * just the first opset version that introduced the operator. However, if the operator has had breaking changes, + * then `since_version` corresponds to the opset version that introduced the breaking change. + * + * For example, suppose operator "Foo" was added in version 3 and had a breaking change in version 6. + * Then, there will be an operator schema entry for "Foo" with a since_version of 3 and another updated + * operator schema entry for "Foo" with a since_version of 6. + * + * \param[in] schema The OrtOpSchema instance. + * \param[out] out Output parameter set to the ONNX opset version. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.25. + */ + ORT_API2_STATUS(OpSchema_GetSinceVersion, _In_ const OrtOpSchema* schema, _Out_ int* out); + + /** \brief Get the number of inputs defined by the operator schema. + * + * \param[in] schema The OrtOpSchema instance. + * \param[out] out Output parameter set to the number of inputs. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.25. + */ + ORT_API2_STATUS(OpSchema_GetNumInputs, _In_ const OrtOpSchema* schema, _Out_ size_t* out); + + /** \brief Get the name of the i-th input formal parameter from an operator schema. + * + * \param[in] schema The OrtOpSchema instance. + * \param[in] index Zero-based index of the input parameter. + * \param[out] out Output parameter set to the name of the input parameter (null-terminated UTF8 string). + * Valid as long as the OrtOpSchema exists. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.25. + */ + ORT_API2_STATUS(OpSchema_GetInputName, _In_ const OrtOpSchema* schema, _In_ size_t index, + _Outptr_ const char** out); + + /** \brief Get the type constraint for the i-th input formal parameter from an operator schema. + * + * Returns a non-owning pointer to the OrtOpSchemaTypeConstraint associated with the given input. + * The returned pointer is valid as long as the parent OrtOpSchema is alive. + * If the input has no type constraint, *out is set to nullptr. + * + * Multiple inputs sharing the same type constraint (e.g., both using "T") return the same pointer. + * + * \param[in] schema The OrtOpSchema instance. + * \param[in] index Zero-based index of the input parameter. + * \param[out] out Output parameter set to the type constraint, or NULL if the input has no type constraint. + * Valid as long as the OrtOpSchema exists. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.25. + */ + ORT_API2_STATUS(OpSchema_GetInputTypeConstraint, _In_ const OrtOpSchema* schema, _In_ size_t index, + _Outptr_result_maybenull_ const OrtOpSchemaTypeConstraint** out); + + /** \brief Get the number of outputs defined by the operator schema. + * + * \param[in] schema The OrtOpSchema instance. + * \param[out] out Output parameter set to the number of outputs. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.25. + */ + ORT_API2_STATUS(OpSchema_GetNumOutputs, _In_ const OrtOpSchema* schema, _Out_ size_t* out); + + /** \brief Get the name of the i-th output formal parameter from an operator schema. + * + * \param[in] schema The OrtOpSchema instance. + * \param[in] index Zero-based index of the output parameter. + * \param[out] out Output parameter set to the name of the output parameter (null-terminated UTF8 string). + * Valid as long as the OrtOpSchema exists. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.25. + */ + ORT_API2_STATUS(OpSchema_GetOutputName, _In_ const OrtOpSchema* schema, _In_ size_t index, + _Outptr_ const char** out); + + /** \brief Get the type constraint for the i-th output formal parameter from an operator schema. + * + * Returns a non-owning pointer to the OrtOpSchemaTypeConstraint associated with the given output. + * The returned pointer is valid as long as the parent OrtOpSchema is alive. + * If the output has no type constraint, *out is set to nullptr. + * + * Multiple outputs sharing the same type constraint return the same pointer. + * Pointer equality can be used to check if two outputs share a type constraint. + * + * \param[in] schema The OrtOpSchema instance. + * \param[in] index Zero-based index of the output parameter. + * \param[out] out Output parameter set to the type constraint, or NULL if the output has no type constraint. + * Valid as long as the OrtOpSchema exists. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.25. + */ + ORT_API2_STATUS(OpSchema_GetOutputTypeConstraint, _In_ const OrtOpSchema* schema, _In_ size_t index, + _Outptr_result_maybenull_ const OrtOpSchemaTypeConstraint** out); + + /** \brief Get the number of unique type constraints in the operator schema. + * + * \param[in] schema The OrtOpSchema instance. + * \param[out] out Output set to the number of type constraints. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.25. + */ + ORT_API2_STATUS(OpSchema_GetTypeConstraintCount, _In_ const OrtOpSchema* schema, _Out_ size_t* out); + + /** \brief Get the i-th type constraint from the operator schema. + * + * Returns a non-owning pointer to the OrtOpSchemaTypeConstraint at the given index. + * The returned pointer is valid as long as the parent OrtOpSchema is alive. + * + * Constraints are returned in the order they are declared in the ONNX operator schema + * definition. The order is stable but has no semantic significance. + * + * Use this API to iterate all type constraints (e.g., to register allowed types for + * each constraint). Use OpSchema_GetInputTypeConstraint / OpSchema_GetOutputTypeConstraint + * to look up the constraint for a specific input or output. + * + * \param[in] schema The OrtOpSchema instance. + * \param[in] index Zero-based index of the type constraint. + * \param[out] out Output parameter set to the type constraint. + * Valid as long as the OrtOpSchema exists. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.25. + */ + ORT_API2_STATUS(OpSchema_GetTypeConstraint, _In_ const OrtOpSchema* schema, _In_ size_t index, + _Outptr_ const OrtOpSchemaTypeConstraint** out); + + /** \brief Get the type parameter name of a type constraint (e.g., "T", "T1"). + * + * \param[in] type_constraint The OrtOpSchemaTypeConstraint instance. + * \param[out] out Output parameter set to the type parameter name. + * Valid as long as the parent OrtOpSchema exists. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.25. + */ + ORT_API2_STATUS(OpSchemaTypeConstraint_GetTypeParamName, _In_ const OrtOpSchemaTypeConstraint* type_constraint, + _Outptr_ const char** out); + + /** \brief Get the allowed type strings for a type constraint. + * + * Returns an array of null-terminated strings representing the allowed data types + * (e.g., "tensor(float)", "tensor(double)"). The array and its contents are valid + * as long as the parent OrtOpSchema exists. + * + * \param[in] type_constraint The OrtOpSchemaTypeConstraint instance. + * \param[out] out_types Output parameter set to the output array of type strings. + * Valid as long as the parent OrtOpSchema exists. + * \param[out] num_types Output parameter set to the number of elements in the output array. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.25. + */ + ORT_API2_STATUS(OpSchemaTypeConstraint_GetAllowedTypes, _In_ const OrtOpSchemaTypeConstraint* type_constraint, + _Outptr_ const char* const** out_types, _Out_ size_t* num_types); + + /** \brief Get the input indices that use a type constraint. + * + * Returns an array of zero-based input indices whose formal parameter type string + * matches this type constraint. The array is valid as long as the parent OrtOpSchema exists. + * + * \param[in] type_constraint The OrtOpSchemaTypeConstraint instance. + * \param[out] out_indices Output parameter set to the output array of input indices. + * \param[out] count Output parameter set to the number of elements in the output array. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.25. + */ + ORT_API2_STATUS(OpSchemaTypeConstraint_GetInputIndices, _In_ const OrtOpSchemaTypeConstraint* type_constraint, + _Outptr_ const size_t** out_indices, _Out_ size_t* count); + + /** \brief Get the output indices that use a type constraint. + * + * Returns an array of zero-based output indices whose formal parameter type string + * matches this type constraint. The array is valid as long as the parent OrtOpSchema exists. + * + * \param[in] type_constraint The OrtOpSchemaTypeConstraint instance. + * \param[out] out_indices Output parameter set to the output array of output indices. + * \param[out] count Output parameter set to the number of elements in the output array. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.25. + */ + ORT_API2_STATUS(OpSchemaTypeConstraint_GetOutputIndices, _In_ const OrtOpSchemaTypeConstraint* type_constraint, + _Outptr_ const size_t** out_indices, _Out_ size_t* count); + + /** \brief Create a profiling event. + * + * An EP profiler calls this to create an event to pass to OrtEpApi::ProfilingEventsContainer_AddEvents. + * The returned event must be released via OrtEpApi::ReleaseProfilingEvent after it has been added. + * + * \param[in] category The event category (e.g., session, node, kernel, or API). + * \param[in] process_id Process ID. Set to -1 if does not apply. + * \param[in] thread_id Thread ID. Set to -1 if does not apply. + * \param[in] event_name Null-terminated string representing the event name. ORT copies this string. + * \param[in] timestamp_us Starting timestamp in microseconds relative to the profiling start time. + * An OrtEpProfilerImpl should record its own clock's profiling start time and + * use the `ep_profiling_start_offset_ns` value passed to OrtEpProfilerImpl::StartProfiling + * to compute this value as: + * timestamp_us = (ep_profiling_start_offset_ns + + * (ep_event_time_ns - ep_profiling_start_time_ns)) / 1000 + * \param[in] duration_us Duration in microseconds. + * \param[in] arg_keys Array of null-terminated argument key strings. Can be NULL if num_args is 0. + * ORT copies these strings. + * \param[in] arg_values Array of null-terminated argument value strings. Can be NULL if num_args is 0. + * ORT copies these strings. + * \param[in] num_args Number of key-value argument pairs. + * \param[out] out Output parameter set to the created profiling event. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.25. + */ + ORT_API2_STATUS(CreateProfilingEvent, + _In_ OrtProfilingEventCategory category, + _In_ int32_t process_id, + _In_ int32_t thread_id, + _In_ const char* event_name, + _In_ int64_t timestamp_us, + _In_ int64_t duration_us, + _In_reads_(num_args) const char* const* arg_keys, + _In_reads_(num_args) const char* const* arg_values, + _In_ size_t num_args, + _Outptr_ OrtProfilingEvent** out); + + /** \brief Release an opaque profiling event created via CreateProfilingEvent. + * + * \since Version 1.25. + */ + ORT_CLASS_RELEASE(ProfilingEvent); + + /** \brief Get the event category of a profiling event. + * + * \param[in] event The OrtProfilingEvent instance. + * \param[out] out Output parameter set to the event category. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.25. + */ + ORT_API2_STATUS(ProfilingEvent_GetCategory, _In_ const OrtProfilingEvent* event, + _Out_ OrtProfilingEventCategory* out); + + /** \brief Get the event name of a profiling event. + * + * \param[in] event The OrtProfilingEvent instance. + * \param[out] out Output parameter set to the event name as a null-terminated UTF-8 string. + * Do not free as it is owned by the OrtProfilingEvent instance. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.25. + */ + ORT_API2_STATUS(ProfilingEvent_GetName, _In_ const OrtProfilingEvent* event, + _Outptr_ const char** out); + + /** \brief Get the start timestamp of a profiling event in microseconds. + * + * \param[in] event The OrtProfilingEvent instance. + * \param[out] out Output parameter set to the start timestamp of the profiling event in microseconds relative to + * the profiling start time. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.25. + */ + ORT_API2_STATUS(ProfilingEvent_GetTimestampUs, _In_ const OrtProfilingEvent* event, + _Out_ int64_t* out); + + /** \brief Get the duration of a profiling event in microseconds. + * + * \param[in] event The OrtProfilingEvent instance. + * \param[out] out Output parameter set to the event duration in microseconds. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.25. + */ + ORT_API2_STATUS(ProfilingEvent_GetDurationUs, _In_ const OrtProfilingEvent* event, + _Out_ int64_t* out); + + /** \brief Get the value of an event argument by its key. + * + * The value is set to NULL if the key is not found. + * + * \param[in] event The OrtProfilingEvent instance. + * \param[in] key Null-terminated argument key to look up. + * \param[out] out Output parameter set to the argument value string, or NULL if not found. + * The value is a null-terminated UTF-8 string. + * Do not free as the string is owned by the OrtProfilingEvent instance. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * \since Version 1.25. + */ + ORT_API2_STATUS(ProfilingEvent_GetArgValue, _In_ const OrtProfilingEvent* event, _In_ const char* key, + _Outptr_result_maybenull_ const char** out); + + /** \brief Add EP profiling events to an events container. + * + * An EP profiler calls this function to report new EP profiling events (e.g., GPU kernel timings) during + * OrtEpProfilerImpl::EndProfiling(). ORT copies the EP event data during this call. The EP retains ownership of the + * OrtProfilingEvent instances and must release them via ReleaseProfilingEvent after this call returns. + * This function may be called multiple times within a single EndProfiling call to add EP events in batches. + * + * \param[in] events_container The OrtProfilingEventsContainer instance provided by ORT + * to OrtEpProfilerImpl::EndProfiling(). + * \param[in] events Array of pointers to opaque OrtProfilingEvent instances. + * \param[in] num_events Number of events in the `events` array. Must be greater than 0. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.25. + */ + ORT_API2_STATUS(ProfilingEventsContainer_AddEvents, _In_ OrtProfilingEventsContainer* events_container, + _In_reads_(num_events) const OrtProfilingEvent* const* events, + _In_ size_t num_events); +}; + +/** + * \brief The data layout type. + * + * EPs may specify a preferred data layout type. ORT's default layout type is OrtEpDataLayout_NCHW, or + * OrtEpDataLayout_Default. + * + * \since Version 1.23. + */ +typedef enum OrtEpDataLayout { + OrtEpDataLayout_NCHW = 0, + OrtEpDataLayout_NHWC, + + OrtEpDataLayout_Default = OrtEpDataLayout_NCHW, +} OrtEpDataLayout; + +/** + * \brief Node assignment policies for graph capture validation. + * + * When graph capture is enabled, ORT validates that nodes are assigned to EPs in a way that is + * compatible with graph capture. An EP can specify which validation policy ORT should apply. + * + * \since Version 1.26. + */ +typedef enum OrtGraphCaptureNodeAssignmentPolicy { + /** All nodes in the main graph must be assigned to this EP. No CPU fallback is allowed. */ + OrtGraphCaptureNodeAssignmentPolicy_ALL_NODES_ON_EP = 0, + + /** Compute nodes must be on this EP. CPU nodes are allowed for shape computation as long as + * no memory copy nodes exist. */ + OrtGraphCaptureNodeAssignmentPolicy_ALLOW_CPU_FOR_SHAPES = 1, +} OrtGraphCaptureNodeAssignmentPolicy; + +/** + * \brief The OrtEp struct provides functions to implement for an execution provider. + * \since Version 1.22. + */ +struct OrtEp { + /** \brief The ONNX Runtime API version the execution provider was compiled with. + * + * Implementation should set this to ORT_API_VERSION. + * ORT uses this to avoid calling functions that were not available when the EP was compiled. + * + * \since Version 1.22. + */ + uint32_t ort_version_supported; + + /** \brief Get the execution provider name. + * + * The returned string should be a null-terminated, UTF-8 encoded string. ORT will copy it. + * + * \param[in] this_ptr The OrtEp instance. + * \return The execution provider name. + * + * \since Version 1.22. + */ + ORT_API_T(const char*, GetName, _In_ const OrtEp* this_ptr); + + /** \brief Get information about the nodes supported by the OrtEp instance. + * + * IMPORTANT: This is not the final version of this API function. This is currently experimental but will + * be stabilized by the ONNX Runtime 1.23 release. + * + * \param[in] this_ptr The OrtEp instance. + * \param[in] graph The OrtGraph instance for which to populate node support. The OrtGraph could be a nested subgraph + * contained by a node (e.g., an If or Loop node). ONNX Runtime calls this function separately + * for each nested subgraph. + * \param[inout] graph_support_info OrtEpGraphSupportInfo instance that the implementer must fill out in order to + * specify the supported nodes. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(GetCapability, _In_ OrtEp* this_ptr, _In_ const OrtGraph* graph, + _Inout_ OrtEpGraphSupportInfo* graph_support_info); + + /** \brief Compile OrtGraph instances assigned to the OrtEp. Implementer must set a OrtNodeComputeInfo instance + * for each OrtGraph in order to define its computation function. + * + * If the session is configured to generate a pre-compiled model, the execution provider must return EPContext nodes, + * as OrtNode instances, that ONNX Runtime uses to create a pre-compiled model, known as an "EPContext model". + * An EPContext model contains EPContext nodes. Each EPContext node encapsulates the pre-compiled binary data for a + * OrtGraph compiled for a specific execution provider. For more details about the EPContext design, refer to: + * \htmlonly + * EPContext design document. + * \endhtmlonly + * + * \param[in] this_ptr The OrtEp instance. + * \param[in] graphs Array of `count` OrtGraph instances to compile. Each graph contains only the nodes for + * which the execution provider indicated support. Nested subgraphs contained by a + * node, such as an If or Loop, have separate OrtGraph instances. + * \param[in] fused_nodes Array of `count` fused nodes that will replace the compiled graphs. + * Each fused node is an OrtNode initialized with the intended fused node name and + * input/output information. + * \param[in] count The number of OrtGraph instances to compile. + * \param[out] node_compute_infos Array of `count` OrtNodeComputeInfo instances that define each OrtGraph instance's + * computation function. The implementer allocates the OrtNodeComputeInfo instances. + * ORT calls ReleaseNodeComputeInfos() to release multiple instances in a batch. + * \param[out] ep_context_nodes Output array of `count` OrtNode instances, each representing an EPContext + * node for a compiled OrtGraph. The execution provider must use + * OrtModelEditorApi::CreateNode to create the OrtNode instances. ONNX Runtime takes + * ownership of the OrtNode instances, so the execution provider must NOT call + * OrtApi::ReleaseNode. Should be ignored if the session is not configured to generate an + * EPContext model. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \note Do NOT cache the provided OrtGraph instances in any of the OrtNodeComputeInfo functions because the + * graphs are only valid for the duration of the call to Compile. Any graph/node/input/output + * names that are needed by the OrtNodeComputeInfo functions must be copied and stored by the OrtEp. + * + * \note As of version 1.24, implementation of this function is optional if the EP does not compile nodes and + * uses a kernel registry instead. + * + * \since Version 1.23. + */ + ORT_API2_STATUS(Compile, _In_ OrtEp* this_ptr, _In_ const OrtGraph** graphs, + _In_ const OrtNode** fused_nodes, _In_ size_t count, + _Out_writes_all_(count) OrtNodeComputeInfo** node_compute_infos, + _Out_writes_(count) OrtNode** ep_context_nodes); + + /** \brief Release OrtNodeComputeInfo instances. + * + * \param[in] this_ptr The OrtEp instance. + * \param[inout] node_compute_infos The OrtNodeComputeInfo instances to release. + * \param[in] num_node_compute_infos The number of OrtNodeComputeInfo instances. + * + * \note As of version 1.24, implementation of this function is optional if the EP does not compile nodes and + * uses a kernel registry instead. + * + * \since Version 1.23. + */ + ORT_API_T(void, ReleaseNodeComputeInfos, _In_ OrtEp* this_ptr, + OrtNodeComputeInfo** node_compute_infos, + _In_ size_t num_node_compute_infos); + + /** \brief Get the EP's preferred data layout. + * + * \note Implementation of this function is optional. + * If not implemented, ORT will assume that this EP prefers the data layout `OrtEpDataLayout::NCHW`. + * + * \param[in] this_ptr The OrtEp instance. + * \param[out] preferred_data_layout The EP's preferred data layout. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(GetPreferredDataLayout, _In_ OrtEp* this_ptr, _Out_ OrtEpDataLayout* preferred_data_layout); + + /** \brief Given an op with domain `domain` and type `op_type`, determine whether an associated node's data layout + * should be converted to `target_data_layout`. + * If the EP prefers a non-default data layout (see `GetPreferredDataLayout()`), this function will be called + * during layout transformation with `target_data_layout` set to the EP's preferred data layout. + * + * \note Implementation of this function is optional. + * If an EP prefers a non-default data layout, it may implement this to customize the specific op data layout + * preferences at a finer granularity. + * + * \param[in] this_ptr The OrtEp instance. + * \param[in] domain The op domain. An empty string means the ONNX domain. + * \param[in] op_type The op type. + * \param[in] target_data_layout The target data layout. + * \param[out] should_convert Whether the associated node's data layout should be converted to `target_data_layout`. + * If greater than 0, convert. + * If 0, don't convert. + * Otherwise, if less than 0, leave the decision to ORT. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(ShouldConvertDataLayoutForOp, _In_ OrtEp* this_ptr, + _In_z_ const char* domain, _In_z_ const char* op_type, + _In_ OrtEpDataLayout target_data_layout, + _Outptr_ int* should_convert); + + /** \brief Set dynamic options on this EP. + * + * Dynamic options can be set by the user at any time after session creation with `OrtApi::SetEpDynamicOptions()`. + * + * \param[in] this_ptr The OrtEp instance. + * \param[in] option_keys The dynamic option keys. + * \param[in] option_values The dynamic option values. + * \param[in] num_options The number of dynamic options. + * + * \note Implementation of this function is optional. + * An EP should only implement this if it needs to handle any dynamic options. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(SetDynamicOptions, _In_ OrtEp* this_ptr, + _In_reads_(num_options) const char* const* option_keys, + _In_reads_(num_options) const char* const* option_values, + _In_ size_t num_options); + + /** \brief Called by ORT to notify the EP of the start of a run. + * + * \param[in] this_ptr The OrtEp instance. + * \param[in] run_options The run options for this run. + * + * \note Implementation of this function is optional. + * When graph capture/replay is enabled and a graph has already been captured, ORT skips + * normal execution and calls ReplayGraph() directly, so this callback is not invoked for replay runs. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(OnRunStart, _In_ OrtEp* this_ptr, _In_ const OrtRunOptions* run_options); + + /** \brief Called by ORT to notify the EP of the end of a run. + * + * \param[in] this_ptr The OrtEp instance. + * \param[in] run_options The run options for this run. + * \param[in] sync_stream Whether any associated stream should be synchronized during this call. + * Only applicable if there is such a stream. + * + * \note Implementation of this function is optional. + * When graph capture/replay is enabled and a graph has already been captured, ORT skips + * normal execution and calls ReplayGraph() directly, so this callback is not invoked for replay runs. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(OnRunEnd, _In_ OrtEp* this_ptr, _In_ const OrtRunOptions* run_options, _In_ bool sync_stream); + + /** \brief Create an OrtAllocator for the given OrtMemoryInfo for an OrtSession. + * + * The OrtMemoryInfo instance will match one of the values set in the OrtEpDevice using EpDevice_AddAllocatorInfo. + * Any allocator specific options should be read from the session options. + * + * If nullptr OrtEpFactory::CreateAllocator will be used. + * + * \param[in] this_ptr The OrtEpFactory instance. + * \param[in] memory_info The OrtMemoryInfo to create the allocator for. May be nullptr. + * \param[out] allocator The created OrtAllocator instance. Set to nullptr if the default CPU allocator is used. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(CreateAllocator, _In_ OrtEp* this_ptr, + _In_ const OrtMemoryInfo* memory_info, + _Outptr_result_maybenull_ OrtAllocator** allocator); + + /** \brief Create a synchronization stream for the given memory device for an OrtSession. + * + * This is used to create a synchronization stream for the execution provider and is used to synchronize + * operations on the device during model execution. + * Any stream specific options should be read from the session options. + * + * If nullptr OrtEpFactory::CreateSyncStreamForDevice will be used. + * + * \param[in] this_ptr The OrtEpFactory instance. + * \param[in] memory_device The OrtMemoryDevice to create the synchronization stream for. + * \param[out] stream The created OrtSyncStreamImpl instance. nullptr if the execution provider is not stream aware. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(CreateSyncStreamForDevice, _In_ OrtEp* this_ptr, + _In_ const OrtMemoryDevice* memory_device, + _Outptr_ OrtSyncStreamImpl** stream); + + /** \brief Get a string with details about the EP stack used to produce a compiled model. + * + * This function gets a compatibility information string that contains details about the execution provider + * used to compile a given model. This string can later be used with ValidateCompiledModelCompatibilityInfo + * to determine if a compiled model is compatible with the EP. + * + * The returned string should be a null-terminated, UTF-8 encoded string. ORT will copy it. + * + * \param[in] this_ptr The OrtEp instance. + * \param[in] graph The OrtGraph instance for which to generate compatibility information. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API_T(const char*, GetCompiledModelCompatibilityInfo, _In_ OrtEp* this_ptr, + _In_ const OrtGraph* graph); + + /** \brief Gets the execution provider's kernel registry, if any. + * + * A kernel registry contains kernel creation information for operator kernels supported by an EP. + * + * \param[in] this_ptr The OrtEp instance. + * \param[out] kernel_registry Output parameter set to the EP's kernel registry, which must remain valid throughout + * the lifetime of the EP. Can be NULL if the EP doesn't use a kernel registry. + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \note Implementation of this function is optional. If set to NULL, ORT assumes the EP compiles nodes. + * + * \since Version 1.24. + */ + ORT_API2_STATUS(GetKernelRegistry, _In_ OrtEp* this_ptr, + _Outptr_result_maybenull_ const OrtKernelRegistry** kernel_registry); + + /** \brief Gets whether the execution provider supports concurrent run calls made on the session. + * + * \param[in] this_ptr The OrtEp instance. + * \param[out] is_supported Whether concurrent runs are supported. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \note Implementation of this function is optional and it may be set to NULL. + * If not implemented, ORT assumes that concurrent runs are supported. + * + * \since Version 1.24. + */ + ORT_API2_STATUS(IsConcurrentRunSupported, _In_ OrtEp* this_ptr, _Outptr_ bool* is_supported); + + /** \brief Called by ORT to block until the device has completed all preceding requested tasks. + * + * Currently this is primarily used by the IOBinding object to ensure that all inputs have been copied + * to the device before execution begins. + * + * \param[in] this_ptr The OrtEp instance. + * + * \note Implementation of this function is optional. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.25. + */ + ORT_API2_STATUS(Sync, _In_ OrtEp* this_ptr); + + /** \brief Return a new profiler for the execution provider. + * + * If the EP supports profiling, it should create and return an OrtEpProfilerImpl instance. + * ORT takes ownership of each non-NULL instance returned and will call OrtEpProfilerImpl::Release when + * it is no longer needed. + * + * ORT may call this function multiple times over the lifetime of a single OrtEp instance, for example + * during EP registration and again per run if run-level profiling is enabled. Each call is independent and + * the EP must return a new profiler instance (or NULL if profiling is not supported). + * + * \param[in] this_ptr The OrtEp instance. + * \param[out] profiler Output parameter set to a new OrtEpProfilerImpl instance created by the EP. + * Set to NULL if the EP does not support profiling. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \note Implementation of this function is optional. If set to NULL, ORT assumes the EP does not + * support profiling. + * + * \since Version 1.25. + */ + ORT_API2_STATUS(CreateProfiler, _In_ OrtEp* this_ptr, + _Outptr_result_maybenull_ OrtEpProfilerImpl** profiler); + + /** \brief Indicate whether the graph capturing mode (e.g., CUDA graph) is enabled for the provider. + * + * Graph capture allows an EP to record a sequence of device (e.g., GPU) operations during an initial run and replay + * them on subsequent runs, bypassing per-kernel CPU launch overhead. + * + * Applications enable graph capture via EP-specific provider options (e.g., `enable_cuda_graph=1` + * for the CUDA EP). An EP should return true from this function if it has been configured to enable + * graph capture/replay. + * + * **ORT graph capture/replay summary:** + * During OrtSession initialization, ORT calls OrtEp::IsGraphCaptureEnabled() on each EP in the order specified during + * provider registration with the session. If an EP returns true, ORT validates that the graph is suitable for + * graph capture, and if so, caches the EP for graph capture during the next run. The graph validation ensures + * that there are no control flow nodes and that node-to-EP assignments are compatible with the policy specified + * by the EP via OrtEp::GetGraphCaptureNodeAssignmentPolicy(). + * Note that an OrtSession only supports graph capture for one EP (i.e., the first EP to claim support). + * + * During the first call to OrtApi::Run() for the OrtSession, ORT performs multiple internal runs of the model + * until the EP indicates that the graph has been captured by returning `true` from `OrtEp::IsGraphCaptured()`. + * If the EP is unable to capture the graph within 8 runs, the call to OrtApi::Run() returns an error OrtStatus. + * Each internal run invokes `OrtEp::OnRunStart()`, normal execution, and `OrtEp::OnRunEnd()`. EPs should use + * these run callbacks to track the number of necessary warm-up runs and begin/end graph capture when ready. + * + * After successful graph capture, subsequent calls to OrtApi::Run() skip normal execution and ORT instead calls + * `OrtEp::ReplayGraph()` directly. + * + * Applications can capture and replay multiple graphs (e.g., one per distinct input shape) by setting the + * `"gpu_graph_id"` run config entry via `OrtApi::AddRunConfigEntry()` to different integer values. ORT passes + * the value as the `graph_annotation_id` parameter to `OrtEp::IsGraphCaptured()` and `OrtEp::ReplayGraph()`. + * + * \param[in] this_ptr The OrtEp instance. + * \return true if graph capture mode is enabled, false otherwise. + * + * \note Implementation of this function is optional. If set to NULL, ORT assumes graph capture is not enabled. + * \note If this function returns true, `OrtEp::IsGraphCaptured` and `OrtEp::ReplayGraph` must also be implemented. + * If either is NULL, ORT will log a warning and ignore this EP for graph capture. + * + * \since Version 1.26. + */ + ORT_API_T(bool, IsGraphCaptureEnabled, _In_ const OrtEp* this_ptr); + + /** \brief Indicate whether a graph has been captured and instantiated. + * + * ORT calls this before each `Session::Run()`. If true, ORT calls `ReplayGraph()` instead of + * normal execution. After a run where this returns false, ORT automatically retries until it + * returns true (handling warm-up runs transparently). + * + * \param[in] this_ptr The OrtEp instance. + * \param[in] graph_annotation_id Identifies which captured graph to query. + * Applications can set this value via `OrtApi::AddRunConfigEntry()` with the key `"gpu_graph_id"`. + * The default value is 0 when the run config entry is not set. + * Setting different IDs allows the EP to capture and manage multiple graphs (e.g., one per + * distinct input shape). A value of -1 means graph capture/replay should be skipped for this run. + * \return true if the graph has been captured, false otherwise. + * + * \note This function must be implemented if `OrtEp::IsGraphCaptureEnabled` is implemented and may return true. + * + * \since Version 1.26. + */ + ORT_API_T(bool, IsGraphCaptured, _In_ const OrtEp* this_ptr, _In_ int graph_annotation_id); + + /** \brief Run the instantiated (captured) graph. + * + * Called by ORT instead of normal execution when `IsGraphCaptured()` returns true. + * + * \param[in] this_ptr The OrtEp instance. + * \param[in] graph_annotation_id Identifies which captured graph to replay. + * Applications can set this value via `OrtApi::AddRunConfigEntry()` with the key `"gpu_graph_id"`. + * The default value is 0 when the run config entry is not set. + * A value of -1 means graph replay should be skipped for this run. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \note This function must be implemented if `OrtEp::IsGraphCaptureEnabled` is implemented and may return true. + * + * \since Version 1.26. + */ + ORT_API2_STATUS(ReplayGraph, _In_ OrtEp* this_ptr, _In_ int graph_annotation_id); + + /** \brief Get the node assignment validation policy for graph capture. + * + * When graph capture is enabled, ORT validates that nodes are assigned to EPs in a way that is + * compatible with graph capture. This function tells ORT which validation policy to apply. + * + * \param[in] this_ptr The OrtEp instance. + * \return The node assignment policy for graph capture. + * + * \note Implementation of this function is optional. If set to NULL, ORT uses + * OrtGraphCaptureNodeAssignmentPolicy_ALL_NODES_ON_EP (strictest validation). + * + * \since Version 1.26. + */ + ORT_API_T(OrtGraphCaptureNodeAssignmentPolicy, GetGraphCaptureNodeAssignmentPolicy, + _In_ const OrtEp* this_ptr); + + /** \brief Query the available device resource for partitioning budget. + * + * Called by ORT during graph partitioning when no explicit resource budget threshold + * has been configured via session options. The EP should query its device for the + * currently available resource (e.g., free GPU memory) and return it as an OrtResourceCount. + * + * If the EP does not support resource querying, set this function pointer to NULL. + * ORT will skip threshold-based budget enforcement in that case. + * + * \param[in] this_ptr The OrtEp instance. + * \param[out] available The available device resource. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \note Implementation of this function is optional. If set to NULL, no automatic + * resource threshold is established and budget enforcement requires an explicit + * threshold from session options. + * + * \since Version 1.26. + */ + ORT_API2_STATUS(GetAvailableResource, _In_ const OrtEp* this_ptr, _Out_ OrtResourceCount* available); + + /** \brief Called by ORT when session initialization is complete. + * + * This provides an opportunity for execution providers to optionally synchronize and + * clean up temporary resources to reduce memory usage and ensure the first inference run is fast. + * + * \param[in] this_ptr The OrtEp instance. + * + * \note Implementation of this function is optional. If set to NULL, ORT assumes no + * post-initialization work is needed and treats it as a no-op success. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.27. + */ + ORT_API2_STATUS(OnSessionInitializationEnd, _In_ OrtEp* this_ptr); + + /** \brief Get the EP's default memory device. + * + * The EP's default memory device identifies the hardware the EP operates on. ORT uses it to: + * - Determine if data copies are needed between EPs (inserting memcpy nodes at EP boundaries) + * - Determine if the EP is CPU-based (which affects synchronization and data transfer decisions) + * - Bind execution streams to the correct device + * + * If the implementation allows an EP to be created with multiple EpDevices this should return the OrtMemoryDevice + * that ORT should consider as default for this EP instance. + * + * An OrtMemoryDevice is obtained from an OrtMemoryInfo via `OrtEpApi::MemoryInfo_GetMemoryDevice()`. + * Typically, an EP creates OrtMemoryInfo instances and registers them with its OrtEpDevice(s) via + * `OrtEpApi::EpDevice_AddAllocatorInfo()`. The OrtMemoryDevice returned here must correspond to an + * OrtMemoryInfo registered as an `OrtDeviceAllocator` entry (either `OrtDeviceMemoryType_DEFAULT` or + * `OrtDeviceMemoryType_HOST_ACCESSIBLE`). An OrtMemoryDevice from an `OrtReadOnlyAllocator` entry is + * not accepted as the EP's default/identity device. + * + * The returned pointer must remain valid for the lifetime of the OrtEp instance + * (typically by storing the parent OrtMemoryInfo as a member of the EP). + * + * If this function is not implemented (NULL), or if it sets `device` to NULL, ORT infers + * the default memory device from the first OrtEpDevice's `OrtDeviceAllocator` entry with + * `OrtDeviceMemoryType_DEFAULT` registered via `EpDevice_AddAllocatorInfo`. EPs created against + * multiple OrtEpDevices whose default memory devices differ should implement this function to + * disambiguate; otherwise the first OrtEpDevice's default memory device is used and the others + * are ignored for identity purposes. If no such allocator entry is registered, the EP defaults + * to a CPU memory device. + * + * \param[in] this_ptr The OrtEp instance. + * \param[out] device Set to the EP's default OrtMemoryDevice, or NULL to use the default behavior (described above). + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \note Implementation of this function is optional. If set to NULL (not implemented), ORT + * infers the default memory device using the default behavior described above. + * + * \since Version 1.27. + */ + ORT_API2_STATUS(GetDefaultMemoryDevice, _In_ const OrtEp* this_ptr, + _Outptr_result_maybenull_ const OrtMemoryDevice** device); + + /** \brief Release a previously captured graph and its associated resources. + * + * Called when the caller no longer needs the captured graph for the given annotation ID. + * This allows the EP to free buffers and other resources tied to this graph. + * + * \param[in] this_ptr The EP instance. + * \param[in] graph_annotation_id The annotation ID of the graph to release. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \note Implementation of this function is optional. If set to NULL, ORT assumes + * no captured graph release is needed and treats it as a no-op success. + * + * \note Thread safety: For EPs that support concurrent Run() calls, this method may be + * called concurrently with Run(). The EP is responsible for ensuring thread safety + * of its own state in that case. For non-concurrent EPs, the session serializes + * calls via its internal mutex. + * + * \since Version 1.27. + */ + ORT_API2_STATUS(ReleaseCapturedGraph, _In_ OrtEp* this_ptr, _In_ int graph_annotation_id); +}; + +/** \brief The function signature that ORT will call to create OrtEpFactory instances. + * + * This must be available in a function called 'CreateEpFactories' in the execution provider library. + * + * \param[in] registered_name The name the execution library is registered with by RegisterExecutionProviderLibrary + * \param[in] ort_api_base The OrtApiBase instance that is used by the factory to get the OrtApi instance for the + * version of ORT that the library was compiled against. + * \param[in] default_logger The default ORT logger that can be used for logging outside of an inference session. + * \param[in,out] factories The implementation should create and add OrtEpFactory instances to this + * pre-allocated array. + * i.e. usage is `factories[0] = new MyEpFactory();` + * \param[in] max_factories The maximum number of OrtEpFactory instances that can be added to `factories`. + * Current default is to allow 4 factories. This can be increased in the future if needed. + * \param[out] num_factories The number of OrtEpFactory instances created by the factory and added to `factories`. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.22. + */ +typedef OrtStatus* (*CreateEpApiFactoriesFn)(_In_ const char* registered_name, _In_ const OrtApiBase* ort_api_base, + _In_ const OrtLogger* default_logger, + _Inout_ OrtEpFactory** factories, _In_ size_t max_factories, + _Out_ size_t* num_factories); + +/** \brief The function signature that ORT will call to release an OrtEpFactory instance. + * + * This must be available in a function called 'ReleaseEpFactory' in the execution provider library. + * + * \param[in] factory The OrtEpFactory instance to release. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.22. + */ +typedef OrtStatus* (*ReleaseEpApiFactoryFn)(_In_ OrtEpFactory* factory); + +/** + * \brief The OrtEpFactory provides functions to create and manage execution providers. + * \since Version 1.22. + */ +struct OrtEpFactory { + /** \brief The ONNX Runtime version the execution provider was compiled with. + * + * Implementation should set to ORT_API_VERSION. + * ORT will use this to ensure it does not call functions that were not available when the library was compiled. + * + * \since Version 1.22. + */ + uint32_t ort_version_supported; + + /** \brief Get the name of the execution provider that the factory creates. + * + * The returned string should be a null-terminated, UTF-8 encoded string. ORT will copy it. + * + * \param[in] this_ptr The OrtEpFactory instance. + * \return The name of the execution provider the factory creates. + * + * \since Version 1.22. + */ + ORT_API_T(const char*, GetName, const OrtEpFactory* this_ptr); + + /** \brief Get the name of vendor who owns the execution provider that the factory creates. + * + * The returned string should be a null-terminated, UTF-8 encoded string. ORT will copy it. + * + * \param[in] this_ptr The OrtEpFactory instance. + * \return vendor The vendor name of the execution provider the factory creates. + * + * \since Version 1.22. + */ + ORT_API_T(const char*, GetVendor, const OrtEpFactory* this_ptr); // return EP vendor + + /** \brief Get information from the execution provider about OrtHardwareDevice support. + * + * \param[in] this_ptr The OrtEpFactory instance. + * Non-const as the factory is passed through to the CreateEp call via the OrtEpDevice. + * \param[in] devices The OrtHardwareDevice instances that are available. + * \param[in] num_devices The number of OrtHardwareDevice instances. + * \param[out] ep_devices OrtEpDevice instances for each OrtHardwareDevice that the EP can use. + * The implementation should call OrtEpApi::CreateEpDevice to create, and add the OrtEpDevice + * instances to this pre-allocated array. ORT will take ownership of the values returned. + * i.e. usage is `ep_devices[0] = ;` + * \param[in] max_ep_devices The maximum number of OrtEpDevices that can be added to ep_devices. + * Current default is 8. This can be increased if needed. + * \param[out] num_ep_devices The number of EP devices added to ep_devices. + * \return true if the factory can create an execution provider that uses `device`. + * + * \since Version 1.22. + */ + ORT_API2_STATUS(GetSupportedDevices, _In_ OrtEpFactory* this_ptr, + _In_reads_(num_devices) const OrtHardwareDevice* const* devices, + _In_ size_t num_devices, + _Inout_ OrtEpDevice** ep_devices, + _In_ size_t max_ep_devices, + _Out_ size_t* num_ep_devices); + + /** \brief Function to create an OrtEp instance for use in a Session. + * + * ORT will call ReleaseEp to release the instance when it is no longer needed. + * + * \param[in] this_ptr The OrtEpFactory instance. + * \param[in] devices The OrtHardwareDevice instances that the execution provider was selected to use. + * May be a subset of the OrtHardwareDevice instances that the execution provider's factory + * set as supported in the call to OrtEpFactory::GetSupportedDevices. + * \param[in] ep_metadata_pairs Execution provider metadata that was provided to OrtEpApi::CreateEpDevice, for each + * device. + * \param[in] num_devices The number of devices the execution provider was selected for. + * \param[in] session_options The OrtSessionOptions instance that contains the configuration options for the + * session. This will include ep_options from GetSupportedDevices as well as any + * user provided overrides. + * Execution provider options will have been added with a prefix of 'ep.[ep name].'. + * The OrtSessionOptions instance will NOT be valid after this call and should not be + * stored for later use. + * \param[in] logger The OrtLogger instance for the session that the execution provider should use for logging. + * \param[out] ep The OrtEp instance created by the factory. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.22. + */ + ORT_API2_STATUS(CreateEp, _In_ OrtEpFactory* this_ptr, + _In_reads_(num_devices) const OrtHardwareDevice* const* devices, + _In_reads_(num_devices) const OrtKeyValuePairs* const* ep_metadata_pairs, + _In_ size_t num_devices, + _In_ const OrtSessionOptions* session_options, + _In_ const OrtLogger* logger, _Outptr_ OrtEp** ep); + + /** \brief Release the OrtEp instance. + * + * \param[in] this_ptr The OrtEpFactory instance. + * \param[in] ep The OrtEp instance to release. + * + * \since Version 1.22. + */ + ORT_API_T(void, ReleaseEp, OrtEpFactory* this_ptr, struct OrtEp* ep); + + /** \brief Get the vendor id who owns the execution provider that the factory creates. + * + * This is typically the PCI vendor ID. See https://pcisig.com/membership/member-companies + * + * \param[in] this_ptr The OrtEpFactory instance. + * \return vendor_id The vendor ID of the execution provider the factory creates. + * + * \since Version 1.23. + */ + ORT_API_T(uint32_t, GetVendorId, const OrtEpFactory* this_ptr); + + /** \brief Get the version of the execution provider that the factory creates. + * + * The version string should adhere to the Semantic Versioning 2.0 specification + * (https://github.com/semver/semver/blob/v2.0.0/semver.md). + * + * The returned string should be a null-terminated, UTF-8 encoded string. ORT will copy it. + * + * \param[in] this_ptr The OrtEpFactory instance. + * \return The execution provider version string. + * + * \since Version 1.23. + */ + ORT_API_T(const char*, GetVersion, _In_ const OrtEpFactory* this_ptr); + + /** \brief Validate the compatibility of a compiled model with the execution provider factory for one or more devices. + * + * Given a compatibility info string produced during model compilation, the EP factory should determine whether the + * compiled model is compatible with the EP factory when targeting the provided hardware devices. All devices provided + * must belong to the same execution provider instance that this factory creates. + * + * The EP factory implementation should consider the set of devices (e.g., multi-adapter or multi-GPU scenarios) when + * evaluating compatibility and set `model_compatibility` accordingly. + * + * \param[in] this_ptr The OrtEpFactory instance. + * \param[in] devices Array of OrtHardwareDevice pointers that the EP would run on. All must map to this EP. + * \param[in] num_devices Number of entries in `devices`. + * \param[in] compatibility_info The compatibility information string produced when the model was compiled. + * \param[out] model_compatibility OrtCompiledModelCompatibility value describing the compatibility of the model with the EP. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(ValidateCompiledModelCompatibilityInfo, _In_ OrtEpFactory* this_ptr, + _In_reads_(num_devices) const OrtHardwareDevice* const* devices, + _In_ size_t num_devices, + _In_ const char* compatibility_info, + _Out_ OrtCompiledModelCompatibility* model_compatibility); + + /** \brief Create an OrtAllocator that can be shared across sessions for the given OrtMemoryInfo. + * + * The factory that creates the EP is responsible for providing the allocators required by the EP. + * The OrtMemoryInfo instance will match one of the values set in the OrtEpDevice using EpDevice_AddAllocatorInfo. + * + * \param[in] this_ptr The OrtEpFactory instance. + * \param[in] memory_info The OrtMemoryInfo to create the allocator for. May be nullptr. + * \param[in] allocator_options Optional key-value pairs for allocator options, can be nullptr. + * \param[out] allocator The created OrtAllocator instance. Set to nullptr if the default CPU allocator is used. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(CreateAllocator, _In_ OrtEpFactory* this_ptr, + _In_ const OrtMemoryInfo* memory_info, + _In_opt_ const OrtKeyValuePairs* allocator_options, + _Outptr_result_maybenull_ OrtAllocator** allocator); + + /** \brief Release an OrtAllocator created by the factory. + * + * \since Version 1.23. + */ + ORT_API_T(void, ReleaseAllocator, _In_ OrtEpFactory* this_ptr, _In_ OrtAllocator* allocator); + + /** \brief Create an OrtDataTransferImpl instance for the factory. + * + * This is used to create an IDataTransfer implementation that can be used to copy data between devices + * that the execution provider supports. + * + * \param[in] this_ptr The OrtEpFactory instance. + * \param[out] data_transfer The created OrtDataTransferImpl instance. Set to nullptr if not required. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(CreateDataTransfer, _In_ OrtEpFactory* this_ptr, + _Outptr_result_maybenull_ OrtDataTransferImpl** data_transfer); + + /** \brief Check if execution providers created by the factory are stream aware. + * + * \param[in] this_ptr The OrtEpFactory instance. + * \return True if the factory creates execution providers that are stream aware and it implements CreateSyncStreamForDevice. + * + * \since Version 1.23. + */ + ORT_API_T(bool, IsStreamAware, _In_ const OrtEpFactory* this_ptr); + + /** \brief Create a synchronization stream for the given memory device. + * + * This is used to create a synchronization stream for the memory device that can be used for operations outside of + * a session. + * + * \param[in] this_ptr The OrtEpFactory instance. + * \param[in] memory_device The OrtMemoryDevice to create the synchronization stream for. + * \param[in] stream_options Options for stream creation. May be nullptr. + * \param[out] stream The created OrtSyncStreamImpl instance. nullptr if the execution provider is not stream aware. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(CreateSyncStreamForDevice, _In_ OrtEpFactory* this_ptr, + _In_ const OrtMemoryDevice* memory_device, + _In_opt_ const OrtKeyValuePairs* stream_options, + _Outptr_ OrtSyncStreamImpl** stream); + + /** \brief Check for known incompatibility reasons between a hardware device and this execution provider. + * + * This function allows an execution provider to check if a specific hardware device is compatible + * with the execution provider. The EP can set specific incompatibility reasons via the + * OrtDeviceEpIncompatibilityDetails parameter using OrtEpApi::DeviceEpIncompatibilityDetails_SetDetails. + * + * \param[in] this_ptr The OrtEpFactory instance. + * \param[in] hw The hardware device to check for incompatibility. + * \param[in,out] details Pre-allocated incompatibility details object created and initialized by ORT. + * The EP can use OrtEpApi::DeviceEpIncompatibilityDetails_SetDetails to set + * incompatibility information. If the device is compatible, the EP can + * leave the object unchanged (it defaults to no incompatibility). + * + * \note Implementation of this function is optional. + * If not implemented, ORT will assume the device is compatible with this EP. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24. + */ + ORT_API2_STATUS(GetHardwareDeviceIncompatibilityDetails, _In_ OrtEpFactory* this_ptr, + _In_ const OrtHardwareDevice* hw, + _Inout_ OrtDeviceEpIncompatibilityDetails* details); + + /** \brief Create an OrtExternalResourceImporterImpl for external resource import. + * + * This is used to create an external resource importer that enables zero-copy import of + * external GPU memory (e.g., D3D12 shared resources) and synchronization primitives + * (e.g., D3D12 timeline fences). + * + * EPs that support external resource import (via CUDA, HIP, Vulkan, or D3D12 APIs) can + * implement this to allow applications to share GPU resources without copies. + * + * \param[in] this_ptr The OrtEpFactory instance. + * \param[in] ep_device The OrtEpDevice to create the external resource importer for. + * \param[out] out_importer The created OrtExternalResourceImporterImpl instance. + * Set to nullptr if external resource import is not supported. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \note Implementation of this function is optional. + * An EP factory should only implement this if it supports external resource import. + * If not implemented or not supported, return ORT_NOT_IMPLEMENTED or set out_importer to nullptr. + * + * \since Version 1.24. + */ + ORT_API2_STATUS(CreateExternalResourceImporterForDevice, _In_ OrtEpFactory* this_ptr, + _In_ const OrtEpDevice* ep_device, + _Outptr_result_maybenull_ OrtExternalResourceImporterImpl** out_importer); + + /** \brief Returns the number of OrtCustomOpDomains that this factory provides. + * + * \param[in] this_ptr The OrtEpFactory instance. + * \param[out] num_domains Output parameter set to the number of provided OrtCustomOpDomain instances. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24. + */ + ORT_API2_STATUS(GetNumCustomOpDomains, _In_ OrtEpFactory* this_ptr, _Out_ size_t* num_domains); + + /** \brief Gets the EP-specific OrtCustomOpDomains. + * + * This function is used when running inference on a model that contains EP-specific custom operations. + * + * Workflow: + * 1. The EP factory implements this function to supply a list of OrtCustomOpDomain instances. + * 2. The application either 1) calls SessionOptionsAppendExecutionProvider_V2() with an OrtEpDevice containing + * the plugin EP's factory or 2) enables auto ep selection. + * 3. 1) SessionOptionsAppendExecutionProvider_V2() appends the provided OrtCustomOpDomains to the + * session options or 2) ORT registers the OrtCustomOpDomains provided by the EP devices + * that could be potentially selected. + * + * As a result, any session created from these session options will have these custom op domains registered + * in ORT, ensuring that the custom ops are properly recognized and validated when the model is loaded. + * + * Plugin EPs can provide two types of custom ops: + * 1. A full OrtCustomOp with a concrete kernel implementation + * - A Plugin EP can supply an OrtCustomOp and a corresponding CustomKernel::Compute() implementation. + * - In GetCapability(), it calls EpGraphSupportInfo_AddSingleNode() to inform ORT + * that the custom node should NOT be fused or compiled. Instead, ORT should invoke + * the custom node's Compute() function at runtime. + * + * 2. A "placeholder" OrtCustomOp with an empty kernel implementation + * - A compile-based Plugin EP can supply an OrtCustomOp whose CustomKernel::Compute() + * does nothing. The purpose is to satisfy model validation during model loading by + * registering the custom op as a valid operator in the session. + * - In GetCapability(), the EP should call EpGraphSupportInfo_AddNodesToFuse() to + * notify ORT that this custom node should be fused and compiled by the EP. + * - In Compile(), the EP executes its compiled bits to perform inference for + * the fused custom node. + * + * Note: The OrtCustomOpDomain instances must be valid while any session is using them. + EP factory has the responsibility to release OrtCustomOpDomain instances it creates. It happens + * automatically if using the C++ Ort::CustomOpDomain class. + * + * \param[in] this_ptr The OrtEpFactory instance. + * \param[out] domains Array of `num_domains` elements pre-allocated by ORT that should be filled with + OrtCustomOpDomain instances created by the EP. The `num_domains` is the value returned by + GetNumCustomOpDomains(). + * \param[in] num_domains The size of the `domains` array pre-allocated by ORT. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24. + */ + ORT_API2_STATUS(GetCustomOpDomains, _In_ OrtEpFactory* this_ptr, + _Out_writes_all_(num_domains) OrtCustomOpDomain** domains, _In_ size_t num_domains); + + /** \brief Initialize graphics interop for the EP factory. + * + * This function sets up graphics interop context that enables synchronization between + * external graphics API workloads (D3D12, Vulkan) and ONNX Runtime inference. + * + * The factory stores the graphics context configuration and uses it when creating + * synchronization streams via CreateSyncStreamForDevice. This approach + * is more graceful than passing the command queue directly during stream creation. + * + * The implementation is EP-specific. EPs may create a specialized interop context using + * platform-specific APIs to enable GPU-GPU synchronization. + * + * Key design points: + * - Single init function with all required params (avoids multiple init signatures) + * - Factory stores context and uses it in stream creation + * - Paired with DeinitGraphicsInterop for cleanup + * + * \param[in] this_ptr The OrtEpFactory instance. + * \param[in] ep_device The OrtEpDevice to initialize graphics interop for. + * \param[in] config Configuration specifying the graphics API and required handles. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \note Implementation of this function is optional. + * EPs that don't support graphics interop should set this to nullptr or return ORT_NOT_IMPLEMENTED. + * + * \since Version 1.25. + */ + ORT_API2_STATUS(InitGraphicsInterop, _In_ OrtEpFactory* this_ptr, + _In_ const OrtEpDevice* ep_device, + _In_ const OrtGraphicsInteropConfig* config); + + /** \brief Deinitialize graphics interop for the EP factory. + * + * This function cleans up any graphics interop context that was set up by InitGraphicsInterop. + * Should be called when graphics interop is no longer needed. + * + * \param[in] this_ptr The OrtEpFactory instance. + * \param[in] ep_device The OrtEpDevice to deinitialize graphics interop for. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \note Implementation of this function is optional. + * EPs that don't support graphics interop should set this to nullptr or return ORT_NOT_IMPLEMENTED. + * + * \since Version 1.25. + */ + ORT_API2_STATUS(DeinitGraphicsInterop, _In_ OrtEpFactory* this_ptr, + _In_ const OrtEpDevice* ep_device); + + /** \brief Select the best model variant candidate from metadata. + * + * Evaluates each candidate's metadata against the given hardware device and optional session options, + * and returns the index of the best match. + * + * Each candidate is an OrtKeyValuePairs representing one model variant. The KVP uses indexed keys + * so that the EP can inspect each model's metadata independently. A variant always has num_models >= 1. + * + * Required and optional keys: + * - "num_models" — number of models in this variant (>= 1) (required) + * - "\.ep_compatibility_info" — compatibility string for model i (required per model) + * - "\.role" — role/purpose of model i (e.g., "prefill", "decode") (optional) + * - "\.future_meaningful_info" — additional EP-meaningful metadata for model i (optional) + * + * where \ is a zero-based index (e.g., "0.ep_compatibility_info", "1.ep_compatibility_info"). + * + * The implementer should loop from 0 to num_models - 1 and validate each "\.ep_compatibility_info" entry. + * An advanced implementation may additionally consider "role" or other metadata when ranking candidates. + * + * **Why this function exists:** + * + * The existing ValidateCompiledModelCompatibilityInfo() alone is not sufficient for some EPs to determine the best + * compatible model when there are multiple candidates. For example, an EP may support multiple compilation modes + * (e.g., "speed optimized" vs "memory optimized") that produce different compatibility strings. The EP can implement + * this function to evaluate the candidate metadata and select the best compatible variant based on its own criteria, + * the target device, and the session options. + * + * If all candidates are unsupported, this function succeeds and sets `selected_index` to SIZE_MAX. + * + * \note The implementer should validate each "\.ep_compatibility_info" in the candidate (e.g., by calling + * ValidateCompiledModelCompatibilityInfo for each one) before determining the best match. + * + * \param[in] this_ptr The OrtEpFactory instance. + * \param[in] device The target hardware device that the EP would run on. Must map to this EP. + * \param[in] candidates Array of OrtKeyValuePairs pointers (one per model variant). + * \param[in] num_candidates Number of candidates (i.e., number of model variants to evaluate). + * \param[in] session_options Optional session options to consider when selecting the best candidate. + * May be nullptr if no session-level preferences are relevant. + * \param[out] selected_index Selected candidate index, or SIZE_MAX if all unsupported. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.28. + */ + ORT_API2_STATUS(SelectBestModelCandidate, _In_ OrtEpFactory* this_ptr, + _In_ const OrtHardwareDevice* device, + _In_reads_(num_candidates) const OrtKeyValuePairs* const* candidates, + _In_ size_t num_candidates, + _In_opt_ const OrtSessionOptions* session_options, + _Out_ size_t* selected_index); +}; + +#ifdef __cplusplus +} +#endif diff --git a/go/onnxruntime/onnxruntime_error_code.h b/go/onnxruntime/onnxruntime_error_code.h new file mode 100644 index 0000000000000..ab5c4b3a93ad6 --- /dev/null +++ b/go/onnxruntime/onnxruntime_error_code.h @@ -0,0 +1,92 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +/** \addtogroup Global + * ONNX Runtime C API + * @{ + */ + +#ifdef __cplusplus +extern "C" { +#endif + +/** \brief Error codes reported by ONNX Runtime. + * + * The error code associated with an ::OrtStatus. + */ +typedef enum OrtErrorCode { + /** + * Success. No error occurred. + */ + ORT_OK, + /** + * Generic failure that does not map to a more specific error code. Consult the error message for details. + */ + ORT_FAIL, + /** + * A caller-supplied argument was invalid (e.g. NULL pointer, out-of-range value, mismatched shape/rank, or bad + * configuration). + */ + ORT_INVALID_ARGUMENT, + /** + * A required file (such as a model file) does not exist. + */ + ORT_NO_SUCHFILE, + /** + * Legacy/unused but retained for ABI compatibility. Historically returned when a model could not be found by name in + * the ONNX Runtime Server (removed in 2022). + */ + ORT_NO_MODEL, + /** + * A hardware accelerator or backend engine reported a failure (e.g. a device crash or other device-level error). + */ + ORT_ENGINE_ERROR, + /** + * A generic runtime exception was caught. The error message is the primary source of detail. + */ + ORT_RUNTIME_EXCEPTION, + /** + * Protobuf parsing or serialization failed. + */ + ORT_INVALID_PROTOBUF, + /** + * Invalid session state for the requested operation. Despite the name, this code does not mean "success, model + * loaded"; it is returned when the session is in the wrong state for the requested call (e.g. a model is already + * loaded, the session is already initialized, or no model has been loaded yet). The name is historical and is + * retained for ABI compatibility; consult the error message for the specific condition. + */ + ORT_MODEL_LOADED, + /** + * The requested functionality is not implemented in this build. + */ + ORT_NOT_IMPLEMENTED, + /** + * The model graph is structurally invalid (e.g. recursive function definitions, invalid tensor dimensions, or + * malformed nodes). + */ + ORT_INVALID_GRAPH, + /** + * An execution provider reported a generic failure. + */ + ORT_EP_FAIL, + /** + * Model loading or session initialization was canceled at the caller's request. + */ + ORT_MODEL_LOAD_CANCELED, + /** + * The model requires compilation by an execution provider, but compilation was disabled via session options. + */ + ORT_MODEL_REQUIRES_COMPILATION, + /** + * A requested resource could not be found. + */ + ORT_NOT_FOUND, +} OrtErrorCode; + +#ifdef __cplusplus +} +#endif + +/// @} diff --git a/go/onnxruntime/options.go b/go/onnxruntime/options.go new file mode 100644 index 0000000000000..73e833eff7edc --- /dev/null +++ b/go/onnxruntime/options.go @@ -0,0 +1,227 @@ +package onnxruntime + +/* +#include "cshim.h" +#include +*/ +import "C" +import ( + "sort" + "unsafe" +) + +// SessionOptions configures how a Session is created. +type SessionOptions struct { + handle *C.OrtSessionOptions +} + +// NewSessionOptions creates a new SessionOptions with default values. +func NewSessionOptions() (*SessionOptions, error) { + if err := checkInit(); err != nil { + return nil, err + } + var opts *C.OrtSessionOptions + if err := checkStatus(C.ort_CreateSessionOptions(&opts)); err != nil { + return nil, wrapErr("create session options", err) + } + return &SessionOptions{handle: opts}, nil +} + +// SetIntraOpNumThreads sets the number of threads used within individual ops. +// 0 means use the default. +func (o *SessionOptions) SetIntraOpNumThreads(n int) error { + return wrapErr("set intra op threads", checkStatus(C.ort_SetIntraOpNumThreads(o.handle, C.int(n)))) +} + +// SetInterOpNumThreads sets the number of threads used to run independent ops in parallel. +// 0 means use the default. +func (o *SessionOptions) SetInterOpNumThreads(n int) error { + return wrapErr("set inter op threads", checkStatus(C.ort_SetInterOpNumThreads(o.handle, C.int(n)))) +} + +// SetGraphOptimizationLevel sets the level of graph optimizations applied. +func (o *SessionOptions) SetGraphOptimizationLevel(level GraphOptimizationLevel) error { + return wrapErr("set graph optimization level", + checkStatus(C.ort_SetSessionGraphOptimizationLevel(o.handle, C.GraphOptimizationLevel(level)))) +} + +// AddConfigEntry sets an arbitrary session configuration key-value pair. +func (o *SessionOptions) AddConfigEntry(key, value string) error { + cKey := C.CString(key) + defer C.free(unsafe.Pointer(cKey)) + cVal := C.CString(value) + defer C.free(unsafe.Pointer(cVal)) + return wrapErr("add config entry", checkStatus(C.ort_AddSessionConfigEntry(o.handle, cKey, cVal))) +} + +// AppendExecutionProvider adds an execution provider. CUDA and TensorRT are +// routed through their dedicated V2 APIs; all others use the generic string +// key-value API. +func (o *SessionOptions) AppendExecutionProvider(name string, options map[string]string) error { + switch name { + case "CUDA", "CUDAExecutionProvider": + return o.appendCUDA(options) + case "TensorRT", "TensorrtExecutionProvider": + return o.appendTensorRT(options) + default: + return o.appendGenericProvider(name, options) + } +} + +// Clone creates a copy of the session options. +func (o *SessionOptions) Clone() (*SessionOptions, error) { + var out *C.OrtSessionOptions + if err := checkStatus(C.ort_CloneSessionOptions(o.handle, &out)); err != nil { + return nil, wrapErr("clone session options", err) + } + return &SessionOptions{handle: out}, nil +} + +// DisableMemPattern disables memory pattern optimization. +func (o *SessionOptions) DisableMemPattern() error { + return wrapErr("disable mem pattern", checkStatus(C.ort_DisableMemPattern(o.handle))) +} + +// EnableMemPattern enables memory pattern optimization (default). +func (o *SessionOptions) EnableMemPattern() error { + return wrapErr("enable mem pattern", checkStatus(C.ort_EnableMemPattern(o.handle))) +} + +// EnableCpuMemArena enables the CPU memory arena. +func (o *SessionOptions) EnableCpuMemArena() error { + return wrapErr("enable cpu mem arena", checkStatus(C.ort_EnableCpuMemArena(o.handle))) +} + +// DisableCpuMemArena disables the CPU memory arena. +func (o *SessionOptions) DisableCpuMemArena() error { + return wrapErr("disable cpu mem arena", checkStatus(C.ort_DisableCpuMemArena(o.handle))) +} + +// EnableProfiling enables profiling with the given file prefix. +func (o *SessionOptions) EnableProfiling(profileFilePrefix string) error { + cPrefix := C.CString(profileFilePrefix) + defer C.free(unsafe.Pointer(cPrefix)) + return wrapErr("enable profiling", checkStatus(C.ort_EnableProfiling(o.handle, cPrefix))) +} + +// DisableProfiling disables profiling. +func (o *SessionOptions) DisableProfiling() error { + return wrapErr("disable profiling", checkStatus(C.ort_DisableProfiling(o.handle))) +} + +// AddFreeDimensionOverride fixes a dynamic dimension by its denotation string. +func (o *SessionOptions) AddFreeDimensionOverride(dimDenotation string, dimValue int64) error { + cDim := C.CString(dimDenotation) + defer C.free(unsafe.Pointer(cDim)) + return wrapErr("add free dimension override", + checkStatus(C.ort_AddFreeDimensionOverride(o.handle, cDim, C.int64_t(dimValue)))) +} + +// AddFreeDimensionOverrideByName fixes a dynamic dimension by its name. +func (o *SessionOptions) AddFreeDimensionOverrideByName(dimName string, dimValue int64) error { + cDim := C.CString(dimName) + defer C.free(unsafe.Pointer(cDim)) + return wrapErr("add free dimension override by name", + checkStatus(C.ort_AddFreeDimensionOverrideByName(o.handle, cDim, C.int64_t(dimValue)))) +} + +// SetExecutionMode sets sequential or parallel execution mode. +func (o *SessionOptions) SetExecutionMode(mode ExecutionMode) error { + return wrapErr("set execution mode", + checkStatus(C.ort_SetSessionExecutionMode(o.handle, C.ExecutionMode(mode)))) +} + +// AddInitializer overrides a model initializer with the given tensor value. +func (o *SessionOptions) AddInitializer(name string, value *Tensor) error { + cName := C.CString(name) + defer C.free(unsafe.Pointer(cName)) + return wrapErr("add initializer", checkStatus(C.ort_AddInitializer(o.handle, cName, value.value))) +} + +// Close releases the session options. It is idempotent. +func (o *SessionOptions) Close() error { + if o.handle != nil { + C.ort_ReleaseSessionOptions(o.handle) + o.handle = nil + } + return nil +} + +func (o *SessionOptions) appendCUDA(opts map[string]string) error { + var cudaOpts *C.OrtCUDAProviderOptionsV2 + if err := checkStatus(C.ort_CreateCUDAProviderOptions(&cudaOpts)); err != nil { + return wrapErr("create cuda options", err) + } + defer C.ort_ReleaseCUDAProviderOptions(cudaOpts) + + if len(opts) > 0 { + keys, values := sortedKV(opts) + defer freeStringArrays(keys, values) + if err := checkStatus(C.ort_UpdateCUDAProviderOptions( + cudaOpts, &keys[0], &values[0], C.size_t(len(opts)))); err != nil { + return wrapErr("update cuda options", err) + } + } + + return wrapErr("append cuda provider", + checkStatus(C.ort_SessionOptionsAppendExecutionProvider_CUDA_V2(o.handle, cudaOpts))) +} + +func (o *SessionOptions) appendTensorRT(opts map[string]string) error { + var trtOpts *C.OrtTensorRTProviderOptionsV2 + if err := checkStatus(C.ort_CreateTensorRTProviderOptions(&trtOpts)); err != nil { + return wrapErr("create tensorrt options", err) + } + defer C.ort_ReleaseTensorRTProviderOptions(trtOpts) + + if len(opts) > 0 { + keys, values := sortedKV(opts) + defer freeStringArrays(keys, values) + if err := checkStatus(C.ort_UpdateTensorRTProviderOptions( + trtOpts, &keys[0], &values[0], C.size_t(len(opts)))); err != nil { + return wrapErr("update tensorrt options", err) + } + } + + return wrapErr("append tensorrt provider", + checkStatus(C.ort_SessionOptionsAppendExecutionProvider_TensorRT_V2(o.handle, trtOpts))) +} + +func (o *SessionOptions) appendGenericProvider(name string, opts map[string]string) error { + cName := C.CString(name) + defer C.free(unsafe.Pointer(cName)) + + if len(opts) == 0 { + return wrapErr("append provider", + checkStatus(C.ort_SessionOptionsAppendExecutionProvider(o.handle, cName, nil, nil, 0))) + } + + keys, values := sortedKV(opts) + defer freeStringArrays(keys, values) + return wrapErr("append provider", + checkStatus(C.ort_SessionOptionsAppendExecutionProvider( + o.handle, cName, &keys[0], &values[0], C.size_t(len(opts))))) +} + +func sortedKV(m map[string]string) ([]*C.char, []*C.char) { + sorted := make([]string, 0, len(m)) + for k := range m { + sorted = append(sorted, k) + } + sort.Strings(sorted) + + keys := make([]*C.char, len(sorted)) + values := make([]*C.char, len(sorted)) + for i, k := range sorted { + keys[i] = C.CString(k) + values[i] = C.CString(m[k]) + } + return keys, values +} + +func freeStringArrays(keys, values []*C.char) { + for i := range keys { + C.free(unsafe.Pointer(keys[i])) + C.free(unsafe.Pointer(values[i])) + } +} diff --git a/go/onnxruntime/options_test.go b/go/onnxruntime/options_test.go new file mode 100644 index 0000000000000..fcc1611a0c25e --- /dev/null +++ b/go/onnxruntime/options_test.go @@ -0,0 +1,135 @@ +package onnxruntime + +import ( + "testing" +) + +func TestCloneSessionOptions(t *testing.T) { + opts, err := NewSessionOptions() + if err != nil { + t.Fatal(err) + } + defer opts.Close() + + opts.SetIntraOpNumThreads(2) + + clone, err := opts.Clone() + if err != nil { + t.Fatal(err) + } + defer clone.Close() + + sess, err := NewSession(testdataPath("add_f32.onnx"), clone) + if err != nil { + t.Fatal(err) + } + sess.Close() +} + +func TestSessionOptionsMemory(t *testing.T) { + opts, err := NewSessionOptions() + if err != nil { + t.Fatal(err) + } + defer opts.Close() + + if err := opts.DisableMemPattern(); err != nil { + t.Fatal(err) + } + if err := opts.EnableMemPattern(); err != nil { + t.Fatal(err) + } + if err := opts.DisableCpuMemArena(); err != nil { + t.Fatal(err) + } + if err := opts.EnableCpuMemArena(); err != nil { + t.Fatal(err) + } +} + +func TestSessionOptionsExecutionMode(t *testing.T) { + opts, err := NewSessionOptions() + if err != nil { + t.Fatal(err) + } + defer opts.Close() + + if err := opts.SetExecutionMode(ExecutionModeParallel); err != nil { + t.Fatal(err) + } + if err := opts.SetExecutionMode(ExecutionModeSequential); err != nil { + t.Fatal(err) + } +} + +func TestSessionOptionsProfiling(t *testing.T) { + opts, err := NewSessionOptions() + if err != nil { + t.Fatal(err) + } + defer opts.Close() + + if err := opts.EnableProfiling("/tmp/ort_test_profile"); err != nil { + t.Fatal(err) + } + + sess, err := NewSession(testdataPath("add_f32.onnx"), opts) + if err != nil { + t.Fatal(err) + } + + path, err := sess.EndProfiling() + if err != nil { + t.Fatal(err) + } + if path == "" { + t.Error("expected non-empty profiling path") + } + t.Logf("profiling output: %s", path) + sess.Close() +} + +func TestSessionOptionsFreeDimension(t *testing.T) { + opts, err := NewSessionOptions() + if err != nil { + t.Fatal(err) + } + defer opts.Close() + + if err := opts.AddFreeDimensionOverrideByName("batch", 4); err != nil { + t.Fatal(err) + } + + sess, err := NewSession(testdataPath("matmul_dynamic.onnx"), opts) + if err != nil { + t.Fatal(err) + } + defer sess.Close() + + inputs := sess.Inputs() + for _, in := range inputs { + for _, d := range in.Shape { + if d == -1 { + t.Errorf("expected no dynamic dims after override, but input %s has shape %v", in.Name, in.Shape) + } + } + } +} + +func TestTensorIsTensor(t *testing.T) { + tensor, err := CreateTensor[float32]([]int64{2}, []float32{1, 2}) + if err != nil { + t.Fatal(err) + } + defer tensor.Close() + + if !tensor.IsTensor() { + t.Error("expected IsTensor() = true") + } + if tensor.IsSequence() { + t.Error("expected IsSequence() = false") + } + if tensor.IsMap() { + t.Error("expected IsMap() = false") + } +} diff --git a/go/onnxruntime/ort.go b/go/onnxruntime/ort.go new file mode 100644 index 0000000000000..ef02338a89ba0 --- /dev/null +++ b/go/onnxruntime/ort.go @@ -0,0 +1,170 @@ +package onnxruntime + +/* +#include "cshim.h" +#include +*/ +import "C" +import ( + "fmt" + "os" + "path/filepath" + "sync" + "sync/atomic" + "unsafe" +) + +var ( + mu sync.Mutex + libPath string + initialized bool + shutdown bool + env *C.OrtEnv + cpuMemInfo *C.OrtMemoryInfo + sessionCount atomic.Int64 +) + +// SetSharedLibraryPath sets the path to the ONNX Runtime shared library. +// Must be called before Init. Ignored after a successful Init. +func SetSharedLibraryPath(path string) { + mu.Lock() + defer mu.Unlock() + if !initialized { + libPath = path + } +} + +// Init initializes the ONNX Runtime environment. It is retryable on failure +// and idempotent after success. +func Init() error { + mu.Lock() + defer mu.Unlock() + + if shutdown { + return errShutdown + } + if initialized { + return nil + } + + path := resolveLibPath() + fn, err := loadLibrary(path) + if err != nil { + return wrapErr("load library", err) + } + + rc := C.ort_init_api(fn) + if rc == 1 { + return wrapErr("init api", fmt.Errorf("OrtGetApiBase returned NULL")) + } + if rc == 2 { + return wrapErr("init api", fmt.Errorf("GetApi(%d) returned NULL; ORT library may be too old", C.ORT_GO_API_VERSION)) + } + + cLogID := C.CString("onnxruntime-go") + defer C.free(unsafe.Pointer(cLogID)) + + var ortEnv *C.OrtEnv + if err := checkStatus(C.ort_CreateEnv(C.ORT_LOGGING_LEVEL_WARNING, cLogID, &ortEnv)); err != nil { + return wrapErr("create env", err) + } + + var memInfo *C.OrtMemoryInfo + if err := checkStatus(C.ort_CreateCpuMemoryInfo(C.OrtDeviceAllocator, C.OrtMemTypeDefault, &memInfo)); err != nil { + C.ort_ReleaseEnv(ortEnv) + return wrapErr("create cpu memory info", err) + } + + env = ortEnv + cpuMemInfo = memInfo + initialized = true + return nil +} + +// IsInitialized reports whether Init has completed successfully. +func IsInitialized() bool { + mu.Lock() + defer mu.Unlock() + return initialized +} + +// Shutdown releases the ONNX Runtime environment. It returns an error if +// any sessions are still open. After Shutdown, Init cannot be called again. +func Shutdown() error { + mu.Lock() + defer mu.Unlock() + + if shutdown { + return nil + } + if !initialized { + shutdown = true + return nil + } + + if n := sessionCount.Load(); n > 0 { + return fmt.Errorf("ort: cannot shut down: %d session(s) still open", n) + } + + C.ort_ReleaseMemoryInfo(cpuMemInfo) + cpuMemInfo = nil + C.ort_ReleaseEnv(env) + env = nil + initialized = false + shutdown = true + return nil +} + +// AvailableProviders returns the execution providers available in the loaded +// ORT library. +func AvailableProviders() ([]string, error) { + mu.Lock() + if !initialized { + mu.Unlock() + return nil, errNotInitialized + } + mu.Unlock() + + var cProviders **C.char + var count C.int + if err := checkStatus(C.ort_GetAvailableProviders(&cProviders, &count)); err != nil { + return nil, wrapErr("get available providers", err) + } + defer C.ort_ReleaseAvailableProviders(cProviders, count) + + n := int(count) + providers := make([]string, n) + ptrs := unsafe.Slice((**C.char)(unsafe.Pointer(cProviders)), n) + for i := 0; i < n; i++ { + providers[i] = C.GoString(ptrs[i]) + } + return providers, nil +} + +func resolveLibPath() string { + if libPath != "" { + return resolveDir(libPath) + } + if envPath := os.Getenv("ORT_LIB_PATH"); envPath != "" { + return resolveDir(envPath) + } + return platformLibraryName() +} + +func resolveDir(path string) string { + info, err := os.Stat(path) + if err == nil && info.IsDir() { + return filepath.Join(path, platformLibraryName()) + } + return path +} + +func checkInit() error { + if !initialized { + if shutdown { + return errShutdown + } + return errNotInitialized + } + return nil +} diff --git a/go/onnxruntime/ort_test.go b/go/onnxruntime/ort_test.go new file mode 100644 index 0000000000000..d246ea6d90ce6 --- /dev/null +++ b/go/onnxruntime/ort_test.go @@ -0,0 +1,96 @@ +package onnxruntime + +import ( + "errors" + "os" + "testing" +) + +func TestMain(m *testing.M) { + libPath := os.Getenv("ORT_LIB_PATH") + if libPath == "" { + libPath = ".ort-lib/lib" + if _, err := os.Stat(libPath); err != nil { + println("skipping tests: set ORT_LIB_PATH to the directory containing libonnxruntime.so") + os.Exit(0) + } + } + SetSharedLibraryPath(libPath) + if err := Init(); err != nil { + println("failed to init ORT:", err.Error()) + os.Exit(1) + } + code := m.Run() + Shutdown() + os.Exit(code) +} + +func TestInitIdempotent(t *testing.T) { + if err := Init(); err != nil { + t.Fatalf("second Init should succeed: %v", err) + } +} + +func TestIsInitialized(t *testing.T) { + if !IsInitialized() { + t.Fatal("expected initialized") + } +} + +func TestAvailableProviders(t *testing.T) { + providers, err := AvailableProviders() + if err != nil { + t.Fatal(err) + } + if len(providers) == 0 { + t.Fatal("expected at least one provider") + } + found := false + for _, p := range providers { + if p == "CPUExecutionProvider" { + found = true + } + } + if !found { + t.Errorf("expected CPUExecutionProvider, got %v", providers) + } +} + +func TestShutdownWithOpenSession(t *testing.T) { + sess, err := NewSession(testdataPath("add_f32.onnx"), nil) + if err != nil { + t.Fatal(err) + } + + err = Shutdown() + if err == nil { + t.Fatal("expected error when shutting down with open session") + } + sess.Close() +} + +func TestInitBadPathRetryable(t *testing.T) { + // We can't easily test this because Init() is already successful and sticky. + // Instead, verify that calling Init() after success is idempotent (no-op), + // which is the complementary guarantee: once initialized, bad paths don't + // regress. The retryable-on-failure path is exercised by the Init error + // handling logic using sync.Mutex (not sync.Once). + SetSharedLibraryPath("/nonexistent/path") + if err := Init(); err != nil { + t.Fatalf("Init should be idempotent after success, got: %v", err) + } +} + +func TestOrtErrorAs(t *testing.T) { + _, err := NewSession("nonexistent_model.onnx", nil) + if err == nil { + t.Fatal("expected error for nonexistent model") + } + var ortErr *OrtError + if !errors.As(err, &ortErr) { + t.Fatalf("expected *OrtError, got %T: %v", err, err) + } + if ortErr.Code != ErrorCodeNoSuchFile { + t.Errorf("expected NoSuchFile, got %s", ortErr.Code) + } +} diff --git a/go/onnxruntime/qwen3_test.go b/go/onnxruntime/qwen3_test.go new file mode 100644 index 0000000000000..ffdcb7c9c4c95 --- /dev/null +++ b/go/onnxruntime/qwen3_test.go @@ -0,0 +1,192 @@ +package onnxruntime + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + "testing" +) + +func findQwen3Model() string { + if p := os.Getenv("QWEN3_ONNX_PATH"); p != "" { + return p + } + home, _ := os.UserHomeDir() + glob := filepath.Join(home, ".cache/huggingface/hub/models--onnx-community--Qwen3-Embedding-0.6B-ONNX/snapshots/*/onnx/model_int8.onnx") + matches, _ := filepath.Glob(glob) + if len(matches) > 0 { + return matches[0] + } + return "" +} + +func TestQwen3EmbeddingIntrospection(t *testing.T) { + modelPath := findQwen3Model() + if modelPath == "" { + t.Skip("Qwen3-Embedding model not found; set QWEN3_ONNX_PATH or download via huggingface-cli") + } + + sess, err := NewSession(modelPath, nil) + if err != nil { + t.Fatal(err) + } + defer sess.Close() + + inputs := sess.Inputs() + if len(inputs) != 59 { + t.Fatalf("expected 59 inputs, got %d", len(inputs)) + } + + if inputs[0].Name != "input_ids" || inputs[0].DataType != TensorElementDataTypeInt64 { + t.Errorf("input[0]: expected input_ids int64, got %s %s", inputs[0].Name, inputs[0].DataType) + } + if inputs[1].Name != "attention_mask" || inputs[1].DataType != TensorElementDataTypeInt64 { + t.Errorf("input[1]: expected attention_mask int64, got %s %s", inputs[1].Name, inputs[1].DataType) + } + if inputs[2].Name != "position_ids" || inputs[2].DataType != TensorElementDataTypeInt64 { + t.Errorf("input[2]: expected position_ids int64, got %s %s", inputs[2].Name, inputs[2].DataType) + } + + for i := 3; i < 59; i++ { + in := inputs[i] + if !strings.HasPrefix(in.Name, "past_key_values.") { + t.Errorf("input[%d]: expected past_key_values.*, got %s", i, in.Name) + } + if in.DataType != TensorElementDataTypeFloat32 { + t.Errorf("input %s: expected Float32, got %s", in.Name, in.DataType) + } + } + + outputs := sess.Outputs() + if len(outputs) != 57 { + t.Fatalf("expected 57 outputs, got %d", len(outputs)) + } + if outputs[0].Name != "last_hidden_state" || outputs[0].DataType != TensorElementDataTypeFloat32 { + t.Errorf("output[0]: expected last_hidden_state float32, got %s %s", outputs[0].Name, outputs[0].DataType) + } +} + +func TestQwen3EmbeddingInference(t *testing.T) { + modelPath := findQwen3Model() + if modelPath == "" { + t.Skip("Qwen3-Embedding model not found; set QWEN3_ONNX_PATH or download via huggingface-cli") + } + + opts, err := NewSessionOptions() + if err != nil { + t.Fatal(err) + } + defer opts.Close() + opts.SetIntraOpNumThreads(4) + opts.SetGraphOptimizationLevel(GraphOptimizationLevelAll) + + sess, err := NewSession(modelPath, opts) + if err != nil { + t.Fatal(err) + } + defer sess.Close() + + seqLen := int64(5) + tokenIDs := []int64{151644, 8948, 198, 2610, 525} // <|im_start|> system \n You are + attentionMask := make([]int64, seqLen) + positionIDs := make([]int64, seqLen) + for i := int64(0); i < seqLen; i++ { + attentionMask[i] = 1 + positionIDs[i] = i + } + + inputMap := make(map[string]*Tensor) + var tensorsToClose []*Tensor + + inputIDs, err := CreateTensor[int64]([]int64{1, seqLen}, tokenIDs) + if err != nil { + t.Fatal(err) + } + tensorsToClose = append(tensorsToClose, inputIDs) + inputMap["input_ids"] = inputIDs + + mask, err := CreateTensor[int64]([]int64{1, seqLen}, attentionMask) + if err != nil { + t.Fatal(err) + } + tensorsToClose = append(tensorsToClose, mask) + inputMap["attention_mask"] = mask + + pos, err := CreateTensor[int64]([]int64{1, seqLen}, positionIDs) + if err != nil { + t.Fatal(err) + } + tensorsToClose = append(tensorsToClose, pos) + inputMap["position_ids"] = pos + + for i := 0; i < 28; i++ { + for _, role := range []string{"key", "value"} { + name := fmt.Sprintf("past_key_values.%d.%s", i, role) + kv, err := CreateTensor[float32]([]int64{1, 8, 0, 128}, []float32{}) + if err != nil { + t.Fatalf("create %s: %v", name, err) + } + tensorsToClose = append(tensorsToClose, kv) + inputMap[name] = kv + } + } + + defer func() { + for _, tensor := range tensorsToClose { + tensor.Close() + } + }() + + results, err := sess.Run(context.Background(), inputMap, []string{"last_hidden_state"}) + if err != nil { + t.Fatal(err) + } + defer func() { + for _, r := range results { + r.Close() + } + }() + + out, ok := results["last_hidden_state"] + if !ok { + t.Fatal("last_hidden_state not in output") + } + + shape := out.Shape() + if len(shape) != 3 { + t.Fatalf("expected 3D output, got %dD: %v", len(shape), shape) + } + if shape[0] != 1 { + t.Errorf("expected batch=1, got %d", shape[0]) + } + if shape[1] != seqLen { + t.Errorf("expected seq=%d, got %d", seqLen, shape[1]) + } + if shape[2] != 1024 { + t.Errorf("expected hidden=1024, got %d", shape[2]) + } + + data, err := TensorData[float32](out) + if err != nil { + t.Fatal(err) + } + if len(data) != int(seqLen)*1024 { + t.Errorf("expected %d elements, got %d", seqLen*1024, len(data)) + } + + allZero := true + for _, v := range data[:100] { + if v != 0 { + allZero = false + break + } + } + if allZero { + t.Error("output is all zeros — model may not have run correctly") + } + + t.Logf("Qwen3-Embedding inference OK: output shape %v, first values: [%.4f, %.4f, %.4f, ...]", + shape, data[0], data[1], data[2]) +} diff --git a/go/onnxruntime/run_options.go b/go/onnxruntime/run_options.go new file mode 100644 index 0000000000000..c0582975b0aee --- /dev/null +++ b/go/onnxruntime/run_options.go @@ -0,0 +1,63 @@ +package onnxruntime + +/* +#include "cshim.h" +#include +*/ +import "C" +import "unsafe" + +type RunOptions struct { + handle *C.OrtRunOptions +} + +func NewRunOptions() (*RunOptions, error) { + if err := checkInit(); err != nil { + return nil, err + } + var opts *C.OrtRunOptions + if err := checkStatus(C.ort_CreateRunOptions(&opts)); err != nil { + return nil, wrapErr("create run options", err) + } + return &RunOptions{handle: opts}, nil +} + +func (o *RunOptions) SetLogVerbosityLevel(level int) error { + return wrapErr("set run log verbosity level", + checkStatus(C.ort_RunOptionsSetRunLogVerbosityLevel(o.handle, C.int(level)))) +} + +func (o *RunOptions) SetLogSeverityLevel(level int) error { + return wrapErr("set run log severity level", + checkStatus(C.ort_RunOptionsSetRunLogSeverityLevel(o.handle, C.int(level)))) +} + +func (o *RunOptions) SetTag(tag string) error { + cTag := C.CString(tag) + defer C.free(unsafe.Pointer(cTag)) + return wrapErr("set run tag", checkStatus(C.ort_RunOptionsSetRunTag(o.handle, cTag))) +} + +func (o *RunOptions) SetTerminate() error { + return wrapErr("set terminate", checkStatus(C.ort_RunOptionsSetTerminate(o.handle))) +} + +func (o *RunOptions) UnsetTerminate() error { + return wrapErr("unset terminate", checkStatus(C.ort_RunOptionsUnsetTerminate(o.handle))) +} + +func (o *RunOptions) AddConfigEntry(key, value string) error { + cKey := C.CString(key) + defer C.free(unsafe.Pointer(cKey)) + cVal := C.CString(value) + defer C.free(unsafe.Pointer(cVal)) + return wrapErr("add run config entry", checkStatus(C.ort_AddRunConfigEntry(o.handle, cKey, cVal))) +} + +func (o *RunOptions) Close() error { + if o.handle != nil { + C.ort_ReleaseRunOptions(o.handle) + o.handle = nil + } + return nil +} diff --git a/go/onnxruntime/run_options_test.go b/go/onnxruntime/run_options_test.go new file mode 100644 index 0000000000000..42de03fb5e340 --- /dev/null +++ b/go/onnxruntime/run_options_test.go @@ -0,0 +1,64 @@ +package onnxruntime + +import ( + "context" + "testing" +) + +func TestRunOptions(t *testing.T) { + opts, err := NewRunOptions() + if err != nil { + t.Fatal(err) + } + defer opts.Close() + + if err := opts.SetLogVerbosityLevel(1); err != nil { + t.Errorf("SetLogVerbosityLevel: %v", err) + } + if err := opts.SetLogSeverityLevel(2); err != nil { + t.Errorf("SetLogSeverityLevel: %v", err) + } + if err := opts.SetTag("test-tag"); err != nil { + t.Errorf("SetTag: %v", err) + } + if err := opts.AddConfigEntry("test.key", "test.value"); err != nil { + t.Errorf("AddConfigEntry: %v", err) + } +} + +func TestRunWithOptions(t *testing.T) { + sess, err := NewSession(testdataPath("add_f32.onnx"), nil) + if err != nil { + t.Fatal(err) + } + defer sess.Close() + + opts, err := NewRunOptions() + if err != nil { + t.Fatal(err) + } + defer opts.Close() + + a, _ := CreateTensor[float32]([]int64{2, 3}, []float32{1, 2, 3, 4, 5, 6}) + b, _ := CreateTensor[float32]([]int64{2, 3}, []float32{10, 20, 30, 40, 50, 60}) + defer a.Close() + defer b.Close() + + results, err := sess.RunWithOptions(context.Background(), opts, map[string]*Tensor{ + "A": a, "B": b, + }, []string{"C"}) + if err != nil { + t.Fatal(err) + } + defer func() { + for _, r := range results { + r.Close() + } + }() + + out := results["C"] + data, _ := TensorData[float32](out) + if data[0] != 11 { + t.Errorf("expected 11, got %f", data[0]) + } +} diff --git a/go/onnxruntime/session.go b/go/onnxruntime/session.go new file mode 100644 index 0000000000000..96030e069cad0 --- /dev/null +++ b/go/onnxruntime/session.go @@ -0,0 +1,497 @@ +package onnxruntime + +/* +#include "cshim.h" +#include +*/ +import "C" +import ( + "context" + "fmt" + "sync" + "unsafe" +) + +// IOInfo describes a single input or output of a model. +type IOInfo struct { + Name string + DataType TensorElementDataType + Shape []int64 // dynamic dimensions are -1 +} + +// Session wraps an ORT inference session. It is safe for concurrent use: +// multiple goroutines may call Run simultaneously. +type Session struct { + mu sync.RWMutex + handle *C.OrtSession + inputs []IOInfo + outputs []IOInfo + closed bool +} + +// NewSession creates a session from an ONNX model file. +// If opts is nil, default options are used. +func NewSession(modelPath string, opts *SessionOptions) (*Session, error) { + if err := checkInit(); err != nil { + return nil, err + } + + ownOpts, opts, err := ensureOpts(opts) + if err != nil { + return nil, err + } + if ownOpts { + defer func() { _ = opts.Close() }() + } + + cPath := C.CString(modelPath) + defer C.free(unsafe.Pointer(cPath)) + + var handle *C.OrtSession + if err := checkStatus(C.ort_CreateSession(env, cPath, opts.handle, &handle)); err != nil { + return nil, wrapErr("create session", err) + } + + return finalizeSession(handle) +} + +// NewSessionFromBytes creates a session from an in-memory ONNX model. +// If opts is nil, default options are used. +func NewSessionFromBytes(model []byte, opts *SessionOptions) (*Session, error) { + if err := checkInit(); err != nil { + return nil, err + } + if len(model) == 0 { + return nil, fmt.Errorf("ort: create session: empty model data") + } + + ownOpts, opts, err := ensureOpts(opts) + if err != nil { + return nil, err + } + if ownOpts { + defer func() { _ = opts.Close() }() + } + + var handle *C.OrtSession + if err := checkStatus(C.ort_CreateSessionFromArray( + env, unsafe.Pointer(&model[0]), C.size_t(len(model)), opts.handle, &handle)); err != nil { + return nil, wrapErr("create session from bytes", err) + } + + return finalizeSession(handle) +} + +// Inputs returns metadata for all model inputs. +func (s *Session) Inputs() []IOInfo { + return copyIOInfo(s.inputs) +} + +// Outputs returns metadata for all model outputs. +func (s *Session) Outputs() []IOInfo { + return copyIOInfo(s.outputs) +} + +// Run executes inference. inputs maps input names to tensors. outputNames +// lists which outputs to request; if nil, all outputs are returned. +// The returned tensors must be closed by the caller. +// +// Run is safe for concurrent use on the same session. +func (s *Session) Run(ctx context.Context, inputs map[string]*Tensor, outputNames []string) (map[string]*Tensor, error) { + s.mu.RLock() + defer s.mu.RUnlock() + + if s.closed { + return nil, fmt.Errorf("ort: run: session is closed") + } + + if ctx.Err() != nil { + return nil, ctx.Err() + } + + if outputNames == nil { + outputNames = make([]string, len(s.outputs)) + for i, o := range s.outputs { + outputNames[i] = o.Name + } + } + + nInputs := len(inputs) + nOutputs := len(outputNames) + + cInputNames := make([]*C.char, nInputs) + cInputValues := make([]*C.OrtValue, nInputs) + i := 0 + for name, tensor := range inputs { + cInputNames[i] = C.CString(name) + cInputValues[i] = tensor.value + i++ + } + defer func() { + for _, cn := range cInputNames { + C.free(unsafe.Pointer(cn)) + } + }() + + cOutputNames := make([]*C.char, nOutputs) + for i, name := range outputNames { + cOutputNames[i] = C.CString(name) + } + defer func() { + for _, cn := range cOutputNames { + C.free(unsafe.Pointer(cn)) + } + }() + + cOutputValues := make([]*C.OrtValue, nOutputs) + + var runOpts *C.OrtRunOptions + var stopWatcher chan struct{} + + if ctx.Done() != nil { + if err := checkStatus(C.ort_CreateRunOptions(&runOpts)); err != nil { + return nil, wrapErr("create run options", err) + } + defer C.ort_ReleaseRunOptions(runOpts) + + stopWatcher = make(chan struct{}) + watcherDone := make(chan struct{}) + done := ctx.Done() + go func() { + defer close(watcherDone) + select { + case <-done: + C.ort_RunOptionsSetTerminate(runOpts) + case <-stopWatcher: + } + }() + defer func() { + close(stopWatcher) + <-watcherDone + }() + } + + var inNamesPtr **C.char + var inValuesPtr **C.OrtValue + if nInputs > 0 { + inNamesPtr = &cInputNames[0] + inValuesPtr = &cInputValues[0] + } + + var outNamesPtr **C.char + if nOutputs > 0 { + outNamesPtr = &cOutputNames[0] + } + + err := checkStatus(C.ort_Run( + s.handle, runOpts, + inNamesPtr, + inValuesPtr, C.size_t(nInputs), + outNamesPtr, C.size_t(nOutputs), + &cOutputValues[0], + )) + + if err != nil { + for _, v := range cOutputValues { + if v != nil { + C.ort_ReleaseValue(v) + } + } + if ctx.Err() != nil { + return nil, ctx.Err() + } + return nil, wrapErr("run", err) + } + + result := make(map[string]*Tensor, nOutputs) + for i, name := range outputNames { + t, err := wrapOutputTensor(cOutputValues[i]) + if err != nil { + for _, v := range cOutputValues[i+1:] { + if v != nil { + C.ort_ReleaseValue(v) + } + } + for _, existing := range result { + _ = existing.Close() + } + return nil, wrapErr("wrap output "+name, err) + } + result[name] = t + } + + return result, nil +} + +// RunWithOptions executes inference with explicit run options. +func (s *Session) RunWithOptions(ctx context.Context, opts *RunOptions, inputs map[string]*Tensor, outputNames []string) (map[string]*Tensor, error) { + s.mu.RLock() + defer s.mu.RUnlock() + + if s.closed { + return nil, fmt.Errorf("ort: run: session is closed") + } + + if ctx.Err() != nil { + return nil, ctx.Err() + } + + if outputNames == nil { + outputNames = make([]string, len(s.outputs)) + for i, o := range s.outputs { + outputNames[i] = o.Name + } + } + + nInputs := len(inputs) + nOutputs := len(outputNames) + + cInputNames := make([]*C.char, nInputs) + cInputValues := make([]*C.OrtValue, nInputs) + i := 0 + for name, tensor := range inputs { + cInputNames[i] = C.CString(name) + cInputValues[i] = tensor.value + i++ + } + defer func() { + for _, cn := range cInputNames { + C.free(unsafe.Pointer(cn)) + } + }() + + cOutputNames := make([]*C.char, nOutputs) + for i, name := range outputNames { + cOutputNames[i] = C.CString(name) + } + defer func() { + for _, cn := range cOutputNames { + C.free(unsafe.Pointer(cn)) + } + }() + + cOutputValues := make([]*C.OrtValue, nOutputs) + + var runOpts *C.OrtRunOptions + if opts != nil { + runOpts = opts.handle + } + + var inNamesPtr **C.char + var inValuesPtr **C.OrtValue + if nInputs > 0 { + inNamesPtr = &cInputNames[0] + inValuesPtr = &cInputValues[0] + } + + var outNamesPtr **C.char + if nOutputs > 0 { + outNamesPtr = &cOutputNames[0] + } + + err := checkStatus(C.ort_Run( + s.handle, runOpts, + inNamesPtr, + inValuesPtr, C.size_t(nInputs), + outNamesPtr, C.size_t(nOutputs), + &cOutputValues[0], + )) + + if err != nil { + for _, v := range cOutputValues { + if v != nil { + C.ort_ReleaseValue(v) + } + } + if ctx.Err() != nil { + return nil, ctx.Err() + } + return nil, wrapErr("run", err) + } + + result := make(map[string]*Tensor, nOutputs) + for i, name := range outputNames { + t, err := wrapOutputTensor(cOutputValues[i]) + if err != nil { + for _, v := range cOutputValues[i+1:] { + if v != nil { + C.ort_ReleaseValue(v) + } + } + for _, existing := range result { + _ = existing.Close() + } + return nil, wrapErr("wrap output "+name, err) + } + result[name] = t + } + + return result, nil +} + +// EndProfiling ends profiling and returns the profile file path. +func (s *Session) EndProfiling() (string, error) { + s.mu.RLock() + defer s.mu.RUnlock() + + if s.closed { + return "", fmt.Errorf("ort: end profiling: session is closed") + } + + var allocator *C.OrtAllocator + if err := checkStatus(C.ort_GetAllocatorWithDefaultOptions(&allocator)); err != nil { + return "", wrapErr("get allocator", err) + } + + var cPath *C.char + if err := checkStatus(C.ort_SessionEndProfiling(s.handle, allocator, &cPath)); err != nil { + return "", wrapErr("end profiling", err) + } + path := C.GoString(cPath) + C.ort_AllocatorFree(allocator, unsafe.Pointer(cPath)) + return path, nil +} + +// Close releases the session. It is idempotent. Close waits for any +// in-flight Run calls to complete. +func (s *Session) Close() error { + s.mu.Lock() + defer s.mu.Unlock() + + if s.closed { + return nil + } + s.closed = true + C.ort_ReleaseSession(s.handle) + s.handle = nil + sessionCount.Add(-1) + return nil +} + +func ensureOpts(opts *SessionOptions) (bool, *SessionOptions, error) { + if opts != nil { + return false, opts, nil + } + o, err := NewSessionOptions() + if err != nil { + return false, nil, err + } + return true, o, nil +} + +func finalizeSession(handle *C.OrtSession) (*Session, error) { + inputs, err := introspectIO(handle, true) + if err != nil { + C.ort_ReleaseSession(handle) + return nil, err + } + + outputs, err := introspectIO(handle, false) + if err != nil { + C.ort_ReleaseSession(handle) + return nil, err + } + + sessionCount.Add(1) + return &Session{ + handle: handle, + inputs: inputs, + outputs: outputs, + }, nil +} + +func introspectIO(session *C.OrtSession, isInput bool) ([]IOInfo, error) { + var count C.size_t + var err error + if isInput { + err = checkStatus(C.ort_SessionGetInputCount(session, &count)) + } else { + err = checkStatus(C.ort_SessionGetOutputCount(session, &count)) + } + if err != nil { + return nil, wrapErr("get io count", err) + } + + var allocator *C.OrtAllocator + if err := checkStatus(C.ort_GetAllocatorWithDefaultOptions(&allocator)); err != nil { + return nil, wrapErr("get allocator", err) + } + + infos := make([]IOInfo, int(count)) + for i := C.size_t(0); i < count; i++ { + var cName *C.char + if isInput { + err = checkStatus(C.ort_SessionGetInputName(session, i, allocator, &cName)) + } else { + err = checkStatus(C.ort_SessionGetOutputName(session, i, allocator, &cName)) + } + if err != nil { + return nil, wrapErr("get io name", err) + } + infos[i].Name = C.GoString(cName) + C.ort_AllocatorFree(allocator, unsafe.Pointer(cName)) + + var typeInfo *C.OrtTypeInfo + if isInput { + err = checkStatus(C.ort_SessionGetInputTypeInfo(session, i, &typeInfo)) + } else { + err = checkStatus(C.ort_SessionGetOutputTypeInfo(session, i, &typeInfo)) + } + if err != nil { + return nil, wrapErr("get io type info", err) + } + + var onnxType C.enum_ONNXType + if err := checkStatus(C.ort_GetOnnxTypeFromTypeInfo(typeInfo, &onnxType)); err != nil { + C.ort_ReleaseTypeInfo(typeInfo) + return nil, wrapErr("get onnx type", err) + } + + if onnxType == C.ONNX_TYPE_TENSOR { + var tensorInfo *C.OrtTensorTypeAndShapeInfo + if err := checkStatus(C.ort_CastTypeInfoToTensorInfo(typeInfo, &tensorInfo)); err != nil { + C.ort_ReleaseTypeInfo(typeInfo) + return nil, wrapErr("cast to tensor info", err) + } + + var cDtype C.enum_ONNXTensorElementDataType + if err := checkStatus(C.ort_GetTensorElementType(tensorInfo, &cDtype)); err != nil { + C.ort_ReleaseTypeInfo(typeInfo) + return nil, wrapErr("get element type", err) + } + infos[i].DataType = TensorElementDataType(cDtype) + + var ndims C.size_t + if err := checkStatus(C.ort_GetDimensionsCount(tensorInfo, &ndims)); err != nil { + C.ort_ReleaseTypeInfo(typeInfo) + return nil, wrapErr("get dims count", err) + } + + shape := make([]int64, int(ndims)) + if ndims > 0 { + if err := checkStatus(C.ort_GetDimensions(tensorInfo, + (*C.int64_t)(unsafe.Pointer(&shape[0])), ndims)); err != nil { + C.ort_ReleaseTypeInfo(typeInfo) + return nil, wrapErr("get dims", err) + } + } + infos[i].Shape = shape + } + + C.ort_ReleaseTypeInfo(typeInfo) + } + + return infos, nil +} + +func copyIOInfo(infos []IOInfo) []IOInfo { + out := make([]IOInfo, len(infos)) + for i, info := range infos { + out[i] = IOInfo{ + Name: info.Name, + DataType: info.DataType, + Shape: copyShape(info.Shape), + } + } + return out +} diff --git a/go/onnxruntime/session_test.go b/go/onnxruntime/session_test.go new file mode 100644 index 0000000000000..b6379f6537d07 --- /dev/null +++ b/go/onnxruntime/session_test.go @@ -0,0 +1,331 @@ +package onnxruntime + +import ( + "context" + "math" + "os" + "path/filepath" + "sync" + "testing" +) + +func testdataPath(name string) string { + return filepath.Join("..", "testdata", name) +} + +func TestNewSessionAddModel(t *testing.T) { + sess, err := NewSession(testdataPath("add_f32.onnx"), nil) + if err != nil { + t.Fatal(err) + } + defer sess.Close() + + inputs := sess.Inputs() + if len(inputs) != 2 { + t.Fatalf("expected 2 inputs, got %d", len(inputs)) + } + for _, in := range inputs { + if in.DataType != TensorElementDataTypeFloat32 { + t.Errorf("input %s: expected Float32, got %s", in.Name, in.DataType) + } + if len(in.Shape) != 2 || in.Shape[0] != 2 || in.Shape[1] != 3 { + t.Errorf("input %s: expected shape [2,3], got %v", in.Name, in.Shape) + } + } + + outputs := sess.Outputs() + if len(outputs) != 1 { + t.Fatalf("expected 1 output, got %d", len(outputs)) + } + if outputs[0].DataType != TensorElementDataTypeFloat32 { + t.Errorf("output dtype: expected Float32, got %s", outputs[0].DataType) + } +} + +func TestRunAddModel(t *testing.T) { + sess, err := NewSession(testdataPath("add_f32.onnx"), nil) + if err != nil { + t.Fatal(err) + } + defer sess.Close() + + a := []float32{1, 2, 3, 4, 5, 6} + b := []float32{10, 20, 30, 40, 50, 60} + expected := []float32{11, 22, 33, 44, 55, 66} + + tensorA, err := CreateTensor[float32]([]int64{2, 3}, a) + if err != nil { + t.Fatal(err) + } + defer tensorA.Close() + + tensorB, err := CreateTensor[float32]([]int64{2, 3}, b) + if err != nil { + t.Fatal(err) + } + defer tensorB.Close() + + results, err := sess.Run(context.Background(), map[string]*Tensor{ + "A": tensorA, + "B": tensorB, + }, nil) + if err != nil { + t.Fatal(err) + } + defer func() { + for _, r := range results { + r.Close() + } + }() + + out, ok := results["C"] + if !ok { + t.Fatal("output C not found") + } + + data, err := TensorData[float32](out) + if err != nil { + t.Fatal(err) + } + + for i, v := range data { + if math.Abs(float64(v-expected[i])) > 1e-6 { + t.Errorf("output[%d]: expected %f, got %f", i, expected[i], v) + } + } +} + +func TestNewSessionFromBytes(t *testing.T) { + data, err := os.ReadFile(testdataPath("add_f32.onnx")) + if err != nil { + t.Fatal(err) + } + sess, err := NewSessionFromBytes(data, nil) + if err != nil { + t.Fatal(err) + } + defer sess.Close() + + if len(sess.Inputs()) != 2 { + t.Fatalf("expected 2 inputs, got %d", len(sess.Inputs())) + } +} + +func TestDynamicShapeModel(t *testing.T) { + sess, err := NewSession(testdataPath("matmul_dynamic.onnx"), nil) + if err != nil { + t.Fatal(err) + } + defer sess.Close() + + inputs := sess.Inputs() + aInfo := inputs[0] + hasDynamic := false + for _, d := range aInfo.Shape { + if d == -1 { + hasDynamic = true + } + } + if !hasDynamic { + t.Errorf("expected dynamic dim (-1) in input shape, got %v", aInfo.Shape) + } + + a := []float32{1, 0, 0, 1, 0, 1, 1, 0} + b := []float32{1, 2, 3, 4, 5, 6, 7, 8} + + tensorA, err := CreateTensor[float32]([]int64{2, 4}, a) + if err != nil { + t.Fatal(err) + } + defer tensorA.Close() + + tensorB, err := CreateTensor[float32]([]int64{4, 2}, b) + if err != nil { + t.Fatal(err) + } + defer tensorB.Close() + + results, err := sess.Run(context.Background(), map[string]*Tensor{ + "A": tensorA, + "B": tensorB, + }, nil) + if err != nil { + t.Fatal(err) + } + defer func() { + for _, r := range results { + r.Close() + } + }() + + out := results["C"] + if out.Shape()[0] != 2 || out.Shape()[1] != 2 { + t.Errorf("expected output shape [2,2], got %v", out.Shape()) + } +} + +func TestZeroLengthDimension(t *testing.T) { + sess, err := NewSession(testdataPath("kvconcat.onnx"), nil) + if err != nil { + t.Fatal(err) + } + defer sess.Close() + + past, err := CreateTensor[float32]([]int64{1, 2, 0, 4}, []float32{}) + if err != nil { + t.Fatal(err) + } + defer past.Close() + + newVal, err := CreateTensor[float32]([]int64{1, 2, 1, 4}, []float32{ + 1, 2, 3, 4, 5, 6, 7, 8, + }) + if err != nil { + t.Fatal(err) + } + defer newVal.Close() + + results, err := sess.Run(context.Background(), map[string]*Tensor{ + "past": past, + "new_val": newVal, + }, nil) + if err != nil { + t.Fatal(err) + } + defer func() { + for _, r := range results { + r.Close() + } + }() + + out := results["out"] + if out == nil { + t.Fatalf("output 'out' not found in results (keys: %v)", mapKeys(results)) + } + if out.Shape()[2] != 1 { + t.Errorf("expected concat output seq dim = 1 (0+1), got %v", out.Shape()) + } +} + +func TestSubsetOutputNames(t *testing.T) { + sess, err := NewSession(testdataPath("add_f32.onnx"), nil) + if err != nil { + t.Fatal(err) + } + defer sess.Close() + + a, _ := CreateTensor[float32]([]int64{2, 3}, []float32{1, 2, 3, 4, 5, 6}) + b, _ := CreateTensor[float32]([]int64{2, 3}, []float32{1, 1, 1, 1, 1, 1}) + defer a.Close() + defer b.Close() + + results, err := sess.Run(context.Background(), map[string]*Tensor{ + "A": a, "B": b, + }, []string{"C"}) + if err != nil { + t.Fatal(err) + } + defer func() { + for _, r := range results { + r.Close() + } + }() + + if len(results) != 1 { + t.Errorf("expected 1 output, got %d", len(results)) + } +} + +func TestConcurrentRun(t *testing.T) { + sess, err := NewSession(testdataPath("add_f32.onnx"), nil) + if err != nil { + t.Fatal(err) + } + defer sess.Close() + + var wg sync.WaitGroup + errs := make(chan error, 16*50) + + for g := 0; g < 16; g++ { + wg.Add(1) + go func() { + defer wg.Done() + for i := 0; i < 50; i++ { + a, _ := CreateTensor[float32]([]int64{2, 3}, []float32{1, 2, 3, 4, 5, 6}) + b, _ := CreateTensor[float32]([]int64{2, 3}, []float32{1, 1, 1, 1, 1, 1}) + + results, err := sess.Run(context.Background(), map[string]*Tensor{ + "A": a, "B": b, + }, nil) + a.Close() + b.Close() + if err != nil { + errs <- err + continue + } + for _, r := range results { + r.Close() + } + } + }() + } + + wg.Wait() + close(errs) + for err := range errs { + t.Error(err) + } +} + +func TestCancelledContext(t *testing.T) { + sess, err := NewSession(testdataPath("add_f32.onnx"), nil) + if err != nil { + t.Fatal(err) + } + defer sess.Close() + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + a, _ := CreateTensor[float32]([]int64{2, 3}, []float32{1, 2, 3, 4, 5, 6}) + b, _ := CreateTensor[float32]([]int64{2, 3}, []float32{1, 1, 1, 1, 1, 1}) + defer a.Close() + defer b.Close() + + _, err = sess.Run(ctx, map[string]*Tensor{"A": a, "B": b}, nil) + if err == nil { + t.Fatal("expected error from cancelled context") + } +} + +func TestSessionOptions(t *testing.T) { + opts, err := NewSessionOptions() + if err != nil { + t.Fatal(err) + } + defer opts.Close() + + if err := opts.SetIntraOpNumThreads(1); err != nil { + t.Fatal(err) + } + if err := opts.SetInterOpNumThreads(1); err != nil { + t.Fatal(err) + } + if err := opts.SetGraphOptimizationLevel(GraphOptimizationLevelAll); err != nil { + t.Fatal(err) + } + + sess, err := NewSession(testdataPath("add_f32.onnx"), opts) + if err != nil { + t.Fatal(err) + } + sess.Close() +} + +func mapKeys(m map[string]*Tensor) []string { + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + return keys +} diff --git a/go/onnxruntime/string_tensor.go b/go/onnxruntime/string_tensor.go new file mode 100644 index 0000000000000..2c78980c518b0 --- /dev/null +++ b/go/onnxruntime/string_tensor.go @@ -0,0 +1,116 @@ +package onnxruntime + +/* +#include "cshim.h" +#include +*/ +import "C" +import ( + "fmt" + "unsafe" +) + +// CreateStringTensor creates a tensor containing string data. +func CreateStringTensor(shape []int64, data []string) (*Tensor, error) { + if err := checkInit(); err != nil { + return nil, err + } + + count := shapeElementCount(shape) + if count < 0 { + return nil, fmt.Errorf("ort: create string tensor: shape contains negative dimension") + } + if int64(len(data)) != count { + return nil, fmt.Errorf("ort: create string tensor: data length %d does not match shape element count %d", len(data), count) + } + + var allocator *C.OrtAllocator + if err := checkStatus(C.ort_GetAllocatorWithDefaultOptions(&allocator)); err != nil { + return nil, wrapErr("get allocator", err) + } + + var cShape *C.int64_t + if len(shape) > 0 { + cShape = (*C.int64_t)(unsafe.Pointer(&shape[0])) + } + + var value *C.OrtValue + if err := checkStatus(C.ort_CreateTensorAsOrtValue( + allocator, cShape, C.size_t(len(shape)), + C.ONNXTensorElementDataType(TensorElementDataTypeString), &value)); err != nil { + return nil, wrapErr("create string tensor", err) + } + + if count > 0 { + cStrings := make([]*C.char, len(data)) + for i, s := range data { + cStrings[i] = C.CString(s) + } + err := checkStatus(C.ort_FillStringTensor(value, &cStrings[0], C.size_t(len(data)))) + for _, cs := range cStrings { + C.free(unsafe.Pointer(cs)) + } + if err != nil { + C.ort_ReleaseValue(value) + return nil, wrapErr("fill string tensor", err) + } + } + + return &Tensor{ + value: value, + dtype: TensorElementDataTypeString, + shape: copyShape(shape), + owned: true, + }, nil +} + +// StringData reads string data from a tensor. Returns an error if the tensor +// is not a string tensor. +func (t *Tensor) StringData() ([]string, error) { + if t.closed { + return nil, fmt.Errorf("ort: string data: tensor is closed") + } + if t.dtype != TensorElementDataTypeString { + return nil, fmt.Errorf("ort: string data: tensor is %s, not String", t.dtype) + } + + count := shapeElementCount(t.shape) + if count == 0 { + return nil, nil + } + + var totalLen C.size_t + if err := checkStatus(C.ort_GetStringTensorDataLength(t.value, &totalLen)); err != nil { + return nil, wrapErr("get string tensor length", err) + } + + n := int(count) + buf := make([]byte, int(totalLen)) + offsets := make([]C.size_t, n) + + var bufPtr unsafe.Pointer + if totalLen > 0 { + bufPtr = unsafe.Pointer(&buf[0]) + } else { + bufPtr = unsafe.Pointer(&buf) + } + + if err := checkStatus(C.ort_GetStringTensorContent( + t.value, bufPtr, totalLen, &offsets[0], C.size_t(n))); err != nil { + return nil, wrapErr("get string tensor content", err) + } + + result := make([]string, n) + for i := 0; i < n; i++ { + start := int(offsets[i]) + var end int + if i+1 < n { + end = int(offsets[i+1]) + } else { + end = int(totalLen) + } + result[i] = string(buf[start:end]) + } + + return result, nil +} diff --git a/go/onnxruntime/string_tensor_test.go b/go/onnxruntime/string_tensor_test.go new file mode 100644 index 0000000000000..000f1b5867fb9 --- /dev/null +++ b/go/onnxruntime/string_tensor_test.go @@ -0,0 +1,84 @@ +package onnxruntime + +import ( + "testing" +) + +func TestCreateStringTensor(t *testing.T) { + data := []string{"hello", "world", "foo"} + tensor, err := CreateStringTensor([]int64{3}, data) + if err != nil { + t.Fatal(err) + } + defer tensor.Close() + + if tensor.DataType() != TensorElementDataTypeString { + t.Errorf("expected String, got %s", tensor.DataType()) + } + if tensor.ElementCount() != 3 { + t.Errorf("expected 3 elements, got %d", tensor.ElementCount()) + } + + got, err := tensor.StringData() + if err != nil { + t.Fatal(err) + } + for i, v := range got { + if v != data[i] { + t.Errorf("[%d]: expected %q, got %q", i, data[i], v) + } + } +} + +func TestCreateStringTensorEmpty(t *testing.T) { + tensor, err := CreateStringTensor([]int64{0}, []string{}) + if err != nil { + t.Fatal(err) + } + defer tensor.Close() + + got, err := tensor.StringData() + if err != nil { + t.Fatal(err) + } + if got != nil { + t.Errorf("expected nil for empty string tensor, got %v", got) + } +} + +func TestCreateStringTensor2D(t *testing.T) { + data := []string{"a", "bb", "ccc", "dddd", "eeeee", "ffffff"} + tensor, err := CreateStringTensor([]int64{2, 3}, data) + if err != nil { + t.Fatal(err) + } + defer tensor.Close() + + shape := tensor.Shape() + if shape[0] != 2 || shape[1] != 3 { + t.Errorf("expected shape [2,3], got %v", shape) + } + + got, err := tensor.StringData() + if err != nil { + t.Fatal(err) + } + for i, v := range got { + if v != data[i] { + t.Errorf("[%d]: expected %q, got %q", i, data[i], v) + } + } +} + +func TestStringDataOnNonStringTensor(t *testing.T) { + tensor, err := CreateTensor[float32]([]int64{2}, []float32{1, 2}) + if err != nil { + t.Fatal(err) + } + defer tensor.Close() + + _, err = tensor.StringData() + if err == nil { + t.Fatal("expected error calling StringData on float tensor") + } +} diff --git a/go/onnxruntime/tensor.go b/go/onnxruntime/tensor.go new file mode 100644 index 0000000000000..f72ea7387969c --- /dev/null +++ b/go/onnxruntime/tensor.go @@ -0,0 +1,331 @@ +package onnxruntime + +/* +#include "cshim.h" +*/ +import "C" +import ( + "fmt" + "runtime" + "unsafe" +) + +// Tensor wraps an OrtValue containing tensor data. +type Tensor struct { + value *C.OrtValue + dtype TensorElementDataType + shape []int64 + pinner runtime.Pinner + keep any + owned bool + closed bool +} + +// CreateTensor creates a tensor backed by the provided data slice. +// The data must not be freed or resized until the tensor is closed. +// The shape must have non-negative dimensions whose product equals len(data). +func CreateTensor[T TensorElement](shape []int64, data []T) (*Tensor, error) { + if err := checkInit(); err != nil { + return nil, err + } + + dtype := dtypeOf[T]() + count := shapeElementCount(shape) + if count < 0 { + return nil, fmt.Errorf("ort: create tensor: shape contains negative dimension") + } + if int64(len(data)) != count { + return nil, fmt.Errorf("ort: create tensor: data length %d does not match shape element count %d", len(data), count) + } + + if count == 0 { + return createEmptyTensor(dtype, shape) + } + + t := &Tensor{dtype: dtype, shape: copyShape(shape), owned: false} + t.pinner.Pin(&data[0]) + t.keep = data + + var cShape *C.int64_t + if len(shape) > 0 { + cShape = (*C.int64_t)(unsafe.Pointer(&shape[0])) + } + dataSize := C.size_t(len(data)) * C.size_t(elemSize(dtype)) + dataPtr := unsafe.Pointer(&data[0]) + + var value *C.OrtValue + if err := checkStatus(C.ort_CreateTensorWithDataAsOrtValue( + cpuMemInfo, dataPtr, dataSize, cShape, C.size_t(len(shape)), + C.ONNXTensorElementDataType(dtype), &value)); err != nil { + t.pinner.Unpin() + return nil, wrapErr("create tensor", err) + } + + t.value = value + return t, nil +} + +// NewTensorFromBytes creates a tensor from raw bytes for types without a +// native Go mapping (Float16, BFloat16) or for fully dynamic pipelines. +// len(data) must equal the product of shape multiplied by the element size of dtype. +func NewTensorFromBytes(dtype TensorElementDataType, shape []int64, data []byte) (*Tensor, error) { + if err := checkInit(); err != nil { + return nil, err + } + + count := shapeElementCount(shape) + if count < 0 { + return nil, fmt.Errorf("ort: create tensor: shape contains negative dimension") + } + + es := elemSize(dtype) + if es == 0 { + return nil, fmt.Errorf("ort: create tensor: unsupported dtype %s", dtype) + } + + expected := int(count) * es + if len(data) != expected { + return nil, fmt.Errorf("ort: create tensor: data length %d does not match expected %d bytes", len(data), expected) + } + + if count == 0 { + return createEmptyTensor(dtype, shape) + } + + t := &Tensor{dtype: dtype, shape: copyShape(shape), owned: false} + t.pinner.Pin(&data[0]) + t.keep = data + + var cShape *C.int64_t + if len(shape) > 0 { + cShape = (*C.int64_t)(unsafe.Pointer(&shape[0])) + } + dataSize := C.size_t(len(data)) + dataPtr := unsafe.Pointer(&data[0]) + + var value *C.OrtValue + if err := checkStatus(C.ort_CreateTensorWithDataAsOrtValue( + cpuMemInfo, dataPtr, dataSize, cShape, C.size_t(len(shape)), + C.ONNXTensorElementDataType(dtype), &value)); err != nil { + t.pinner.Unpin() + return nil, wrapErr("create tensor", err) + } + + t.value = value + return t, nil +} + +// TensorData returns a typed view of the tensor's data buffer. The returned +// slice is valid until the tensor is closed. Returns an error if T's dtype +// does not match the tensor's element type. +func TensorData[T TensorElement](t *Tensor) ([]T, error) { + if t.closed { + return nil, fmt.Errorf("ort: tensor data: tensor is closed") + } + + expected := dtypeOf[T]() + if t.dtype != expected { + return nil, fmt.Errorf("ort: tensor data: type mismatch: tensor is %s, requested %s", t.dtype, expected) + } + + count := shapeElementCount(t.shape) + if count == 0 { + return nil, nil + } + + var dataPtr unsafe.Pointer + if err := checkStatus(C.ort_GetTensorMutableData(t.value, &dataPtr)); err != nil { + return nil, wrapErr("tensor data", err) + } + + return unsafe.Slice((*T)(dataPtr), count), nil +} + +// Bytes returns a raw byte view of the tensor's data buffer. +func (t *Tensor) Bytes() ([]byte, error) { + if t.closed { + return nil, fmt.Errorf("ort: tensor bytes: tensor is closed") + } + + count := shapeElementCount(t.shape) + if count == 0 { + return nil, nil + } + + var dataPtr unsafe.Pointer + if err := checkStatus(C.ort_GetTensorMutableData(t.value, &dataPtr)); err != nil { + return nil, wrapErr("tensor bytes", err) + } + + nbytes := int(count) * elemSize(t.dtype) + return unsafe.Slice((*byte)(dataPtr), nbytes), nil +} + +// Shape returns a copy of the tensor's shape. +func (t *Tensor) Shape() []int64 { + return copyShape(t.shape) +} + +// DataType returns the tensor's element data type. +func (t *Tensor) DataType() TensorElementDataType { + return t.dtype +} + +// ElementCount returns the total number of elements. +func (t *Tensor) ElementCount() int64 { + return shapeElementCount(t.shape) +} + +// ValueType returns the ONNX type of the underlying OrtValue. +func (t *Tensor) ValueType() (int, error) { + if t.closed { + return 0, fmt.Errorf("ort: value type: tensor is closed") + } + var onnxType C.enum_ONNXType + if err := checkStatus(C.ort_GetValueType(t.value, &onnxType)); err != nil { + return 0, wrapErr("get value type", err) + } + return int(onnxType), nil +} + +// IsTensor reports whether this value is a tensor (as opposed to sequence/map). +func (t *Tensor) IsTensor() bool { + vt, err := t.ValueType() + return err == nil && vt == 1 // ONNX_TYPE_TENSOR +} + +// IsSequence reports whether this value is a sequence. +func (t *Tensor) IsSequence() bool { + vt, err := t.ValueType() + return err == nil && vt == 3 // ONNX_TYPE_SEQUENCE +} + +// IsMap reports whether this value is a map. +func (t *Tensor) IsMap() bool { + vt, err := t.ValueType() + return err == nil && vt == 4 // ONNX_TYPE_MAP +} + +// SequenceLen returns the number of elements in a sequence value. +func (t *Tensor) SequenceLen() (int, error) { + if t.closed { + return 0, fmt.Errorf("ort: sequence len: value is closed") + } + var count C.size_t + if err := checkStatus(C.ort_GetValueCount(t.value, &count)); err != nil { + return 0, wrapErr("get value count", err) + } + return int(count), nil +} + +// SequenceAt returns the element at the given index from a sequence value. +// The returned tensor must be closed by the caller. +func (t *Tensor) SequenceAt(index int) (*Tensor, error) { + if t.closed { + return nil, fmt.Errorf("ort: sequence at: value is closed") + } + var allocator *C.OrtAllocator + if err := checkStatus(C.ort_GetAllocatorWithDefaultOptions(&allocator)); err != nil { + return nil, wrapErr("get allocator", err) + } + var value *C.OrtValue + if err := checkStatus(C.ort_GetValue(t.value, C.int(index), allocator, &value)); err != nil { + return nil, wrapErr("get value", err) + } + return wrapOutputTensor(value) +} + +// Close releases the tensor's resources. It is idempotent. +func (t *Tensor) Close() error { + if t.closed { + return nil + } + t.closed = true + if t.value != nil { + C.ort_ReleaseValue(t.value) + t.value = nil + } + if !t.owned { + t.pinner.Unpin() + } + t.keep = nil + return nil +} + +func createEmptyTensor(dtype TensorElementDataType, shape []int64) (*Tensor, error) { + var allocator *C.OrtAllocator + if err := checkStatus(C.ort_GetAllocatorWithDefaultOptions(&allocator)); err != nil { + return nil, wrapErr("get allocator", err) + } + + cShape := (*C.int64_t)(nil) + if len(shape) > 0 { + cShape = (*C.int64_t)(unsafe.Pointer(&shape[0])) + } + + var value *C.OrtValue + if err := checkStatus(C.ort_CreateTensorAsOrtValue( + allocator, cShape, C.size_t(len(shape)), + C.ONNXTensorElementDataType(dtype), &value)); err != nil { + return nil, wrapErr("create empty tensor", err) + } + + return &Tensor{ + value: value, + dtype: dtype, + shape: copyShape(shape), + owned: true, + }, nil +} + +func wrapOutputTensor(value *C.OrtValue) (*Tensor, error) { + var info *C.OrtTensorTypeAndShapeInfo + if err := checkStatus(C.ort_GetTensorTypeAndShape(value, &info)); err != nil { + return nil, wrapErr("get output tensor type", err) + } + defer C.ort_ReleaseTensorTypeAndShapeInfo(info) + + var cDtype C.enum_ONNXTensorElementDataType + if err := checkStatus(C.ort_GetTensorElementType(info, &cDtype)); err != nil { + return nil, wrapErr("get output element type", err) + } + + var ndims C.size_t + if err := checkStatus(C.ort_GetDimensionsCount(info, &ndims)); err != nil { + return nil, wrapErr("get output dims count", err) + } + + shape := make([]int64, int(ndims)) + if ndims > 0 { + if err := checkStatus(C.ort_GetDimensions(info, (*C.int64_t)(unsafe.Pointer(&shape[0])), ndims)); err != nil { + return nil, wrapErr("get output dims", err) + } + } + + return &Tensor{ + value: value, + dtype: TensorElementDataType(cDtype), + shape: shape, + owned: true, + }, nil +} + +func shapeElementCount(shape []int64) int64 { + if len(shape) == 0 { + return 1 + } + count := int64(1) + for _, d := range shape { + if d < 0 { + return -1 + } + count *= d + } + return count +} + +func copyShape(s []int64) []int64 { + out := make([]int64, len(s)) + copy(out, s) + return out +} diff --git a/go/onnxruntime/tensor_test.go b/go/onnxruntime/tensor_test.go new file mode 100644 index 0000000000000..30c1c071d34e9 --- /dev/null +++ b/go/onnxruntime/tensor_test.go @@ -0,0 +1,183 @@ +package onnxruntime + +import ( + "testing" +) + +func TestCreateTensorFloat32(t *testing.T) { + data := []float32{1, 2, 3, 4, 5, 6} + tensor, err := CreateTensor[float32]([]int64{2, 3}, data) + if err != nil { + t.Fatal(err) + } + defer tensor.Close() + + if tensor.DataType() != TensorElementDataTypeFloat32 { + t.Errorf("expected Float32, got %s", tensor.DataType()) + } + if tensor.ElementCount() != 6 { + t.Errorf("expected 6 elements, got %d", tensor.ElementCount()) + } + + got, err := TensorData[float32](tensor) + if err != nil { + t.Fatal(err) + } + for i, v := range got { + if v != data[i] { + t.Errorf("[%d]: expected %f, got %f", i, data[i], v) + } + } +} + +func TestCreateTensorInt64(t *testing.T) { + data := []int64{10, 20, 30} + tensor, err := CreateTensor[int64]([]int64{3}, data) + if err != nil { + t.Fatal(err) + } + defer tensor.Close() + + if tensor.DataType() != TensorElementDataTypeInt64 { + t.Errorf("expected Int64, got %s", tensor.DataType()) + } + + got, err := TensorData[int64](tensor) + if err != nil { + t.Fatal(err) + } + for i, v := range got { + if v != data[i] { + t.Errorf("[%d]: expected %d, got %d", i, data[i], v) + } + } +} + +func TestCreateTensorBool(t *testing.T) { + data := []bool{true, false, true} + tensor, err := CreateTensor[bool]([]int64{3}, data) + if err != nil { + t.Fatal(err) + } + defer tensor.Close() + + if tensor.DataType() != TensorElementDataTypeBool { + t.Errorf("expected Bool, got %s", tensor.DataType()) + } +} + +func TestCreateTensorZeroLength(t *testing.T) { + tensor, err := CreateTensor[float32]([]int64{1, 2, 0, 4}, []float32{}) + if err != nil { + t.Fatal(err) + } + defer tensor.Close() + + if tensor.ElementCount() != 0 { + t.Errorf("expected 0 elements, got %d", tensor.ElementCount()) + } + + shape := tensor.Shape() + if len(shape) != 4 || shape[2] != 0 { + t.Errorf("expected shape [1,2,0,4], got %v", shape) + } +} + +func TestTensorDataTypeMismatch(t *testing.T) { + tensor, err := CreateTensor[float32]([]int64{2}, []float32{1, 2}) + if err != nil { + t.Fatal(err) + } + defer tensor.Close() + + _, err = TensorData[int64](tensor) + if err == nil { + t.Fatal("expected type mismatch error") + } +} + +func TestTensorShapeMismatch(t *testing.T) { + _, err := CreateTensor[float32]([]int64{2, 3}, []float32{1, 2, 3}) + if err == nil { + t.Fatal("expected shape/data length mismatch error") + } +} + +func TestTensorNegativeDim(t *testing.T) { + _, err := CreateTensor[float32]([]int64{-1, 3}, []float32{1, 2, 3}) + if err == nil { + t.Fatal("expected negative dimension error") + } +} + +func TestTensorUseAfterClose(t *testing.T) { + tensor, err := CreateTensor[float32]([]int64{2}, []float32{1, 2}) + if err != nil { + t.Fatal(err) + } + tensor.Close() + + _, err = TensorData[float32](tensor) + if err == nil { + t.Fatal("expected error on use after close") + } +} + +func TestTensorDoubleClose(t *testing.T) { + tensor, err := CreateTensor[float32]([]int64{2}, []float32{1, 2}) + if err != nil { + t.Fatal(err) + } + if err := tensor.Close(); err != nil { + t.Fatal(err) + } + if err := tensor.Close(); err != nil { + t.Fatal("double close should not error") + } +} + +func TestNewTensorFromBytes(t *testing.T) { + data := []byte{0, 0, 0x80, 0x3f, 0, 0, 0, 0x40} + tensor, err := NewTensorFromBytes(TensorElementDataTypeFloat32, []int64{2}, data) + if err != nil { + t.Fatal(err) + } + defer tensor.Close() + + got, err := TensorData[float32](tensor) + if err != nil { + t.Fatal(err) + } + if got[0] != 1.0 || got[1] != 2.0 { + t.Errorf("expected [1.0, 2.0], got %v", got) + } +} + +func TestTensorBytes(t *testing.T) { + data := []float32{1.0, 2.0} + tensor, err := CreateTensor[float32]([]int64{2}, data) + if err != nil { + t.Fatal(err) + } + defer tensor.Close() + + b, err := tensor.Bytes() + if err != nil { + t.Fatal(err) + } + if len(b) != 8 { + t.Errorf("expected 8 bytes, got %d", len(b)) + } +} + +func TestScalarTensor(t *testing.T) { + tensor, err := CreateTensor[float32]([]int64{}, []float32{42.0}) + if err != nil { + t.Fatal(err) + } + defer tensor.Close() + + if tensor.ElementCount() != 1 { + t.Errorf("expected 1 element, got %d", tensor.ElementCount()) + } +} diff --git a/go/onnxruntime/types.go b/go/onnxruntime/types.go new file mode 100644 index 0000000000000..d27a59ec2c527 --- /dev/null +++ b/go/onnxruntime/types.go @@ -0,0 +1,129 @@ +package onnxruntime + +// TensorElementDataType represents the data type of tensor elements. +type TensorElementDataType int + +const ( + TensorElementDataTypeUndefined TensorElementDataType = 0 + TensorElementDataTypeFloat32 TensorElementDataType = 1 + TensorElementDataTypeUint8 TensorElementDataType = 2 + TensorElementDataTypeInt8 TensorElementDataType = 3 + TensorElementDataTypeUint16 TensorElementDataType = 4 + TensorElementDataTypeInt16 TensorElementDataType = 5 + TensorElementDataTypeInt32 TensorElementDataType = 6 + TensorElementDataTypeInt64 TensorElementDataType = 7 + TensorElementDataTypeString TensorElementDataType = 8 + TensorElementDataTypeBool TensorElementDataType = 9 + TensorElementDataTypeFloat16 TensorElementDataType = 10 + TensorElementDataTypeFloat64 TensorElementDataType = 11 + TensorElementDataTypeUint32 TensorElementDataType = 12 + TensorElementDataTypeUint64 TensorElementDataType = 13 + TensorElementDataTypeBFloat16 TensorElementDataType = 16 +) + +func (t TensorElementDataType) String() string { + switch t { + case TensorElementDataTypeUndefined: + return "Undefined" + case TensorElementDataTypeFloat32: + return "Float32" + case TensorElementDataTypeUint8: + return "Uint8" + case TensorElementDataTypeInt8: + return "Int8" + case TensorElementDataTypeUint16: + return "Uint16" + case TensorElementDataTypeInt16: + return "Int16" + case TensorElementDataTypeInt32: + return "Int32" + case TensorElementDataTypeInt64: + return "Int64" + case TensorElementDataTypeString: + return "String" + case TensorElementDataTypeBool: + return "Bool" + case TensorElementDataTypeFloat16: + return "Float16" + case TensorElementDataTypeFloat64: + return "Float64" + case TensorElementDataTypeUint32: + return "Uint32" + case TensorElementDataTypeUint64: + return "Uint64" + case TensorElementDataTypeBFloat16: + return "BFloat16" + default: + return "Unknown" + } +} + +// GraphOptimizationLevel controls the level of graph optimizations applied. +type GraphOptimizationLevel int + +const ( + GraphOptimizationLevelDisableAll GraphOptimizationLevel = 0 + GraphOptimizationLevelBasic GraphOptimizationLevel = 1 + GraphOptimizationLevelExtended GraphOptimizationLevel = 2 + GraphOptimizationLevelAll GraphOptimizationLevel = 99 +) + +// ExecutionMode controls sequential or parallel execution of independent ops. +type ExecutionMode int + +const ( + ExecutionModeSequential ExecutionMode = 0 + ExecutionModeParallel ExecutionMode = 1 +) + +// TensorElement enumerates Go types with a direct ONNX tensor element mapping. +type TensorElement interface { + bool | int8 | uint8 | int16 | uint16 | int32 | uint32 | + int64 | uint64 | float32 | float64 +} + +func dtypeOf[T TensorElement]() TensorElementDataType { + var zero T + switch any(zero).(type) { + case float32: + return TensorElementDataTypeFloat32 + case uint8: + return TensorElementDataTypeUint8 + case int8: + return TensorElementDataTypeInt8 + case uint16: + return TensorElementDataTypeUint16 + case int16: + return TensorElementDataTypeInt16 + case int32: + return TensorElementDataTypeInt32 + case int64: + return TensorElementDataTypeInt64 + case bool: + return TensorElementDataTypeBool + case float64: + return TensorElementDataTypeFloat64 + case uint32: + return TensorElementDataTypeUint32 + case uint64: + return TensorElementDataTypeUint64 + default: + return TensorElementDataTypeUndefined + } +} + +func elemSize(dt TensorElementDataType) int { + switch dt { + case TensorElementDataTypeBool, TensorElementDataTypeInt8, TensorElementDataTypeUint8: + return 1 + case TensorElementDataTypeInt16, TensorElementDataTypeUint16, + TensorElementDataTypeFloat16, TensorElementDataTypeBFloat16: + return 2 + case TensorElementDataTypeInt32, TensorElementDataTypeUint32, TensorElementDataTypeFloat32: + return 4 + case TensorElementDataTypeInt64, TensorElementDataTypeUint64, TensorElementDataTypeFloat64: + return 8 + default: + return 0 + } +} diff --git a/go/testdata/add_f32.onnx b/go/testdata/add_f32.onnx new file mode 100644 index 0000000000000000000000000000000000000000..68b53b4ba15799228bf569073ec374bc323d6126 GIT binary patch literal 100 zcmdbXH<^Oi2;qOiW3MPcKR=$cPdKN(u3C@o+E-ad0tlFaa?$ VNYDvgFbOE=j4tTJ!o?uK3jnbO3d8^a literal 0 HcmV?d00001 diff --git a/go/testdata/gen_models.py b/go/testdata/gen_models.py new file mode 100644 index 0000000000000..365e0e8dc6082 --- /dev/null +++ b/go/testdata/gen_models.py @@ -0,0 +1,82 @@ +"""Generate minimal ONNX test models for Go bindings testing.""" + +import os +import onnx +from onnx import TensorProto, helper + +OUT_DIR = os.path.dirname(os.path.abspath(__file__)) + + +def make_add_f32(): + """Add op: A[2,3] + B[2,3] = C[2,3], all float32.""" + A = helper.make_tensor_value_info("A", TensorProto.FLOAT, [2, 3]) + B = helper.make_tensor_value_info("B", TensorProto.FLOAT, [2, 3]) + C = helper.make_tensor_value_info("C", TensorProto.FLOAT, [2, 3]) + + node = helper.make_node("Add", inputs=["A", "B"], outputs=["C"]) + graph = helper.make_graph([node], "add_graph", [A, B], [C]) + model = helper.make_model( + graph, + opset_imports=[helper.make_opsetid("", 13)], + ) + model.ir_version = 8 + onnx.checker.check_model(model) + path = os.path.join(OUT_DIR, "add_f32.onnx") + onnx.save(model, path) + return path + + +def make_matmul_dynamic(): + """MatMul with dynamic batch: A[batch,4] @ B[4,2] = C[batch,2].""" + A = helper.make_tensor_value_info("A", TensorProto.FLOAT, ["batch", 4]) + B = helper.make_tensor_value_info("B", TensorProto.FLOAT, [4, 2]) + C = helper.make_tensor_value_info("C", TensorProto.FLOAT, ["batch", 2]) + + node = helper.make_node("MatMul", inputs=["A", "B"], outputs=["C"]) + graph = helper.make_graph([node], "matmul_graph", [A, B], [C]) + model = helper.make_model( + graph, + opset_imports=[helper.make_opsetid("", 13)], + ) + model.ir_version = 8 + onnx.checker.check_model(model) + path = os.path.join(OUT_DIR, "matmul_dynamic.onnx") + onnx.save(model, path) + return path + + +def make_kvconcat(): + """Concat on axis=2: past[1,2,seq,4] + new_val[1,2,1,4] = out[1,2,seq+1,4]. + + Tests zero-length dim path when seq=0. + """ + past = helper.make_tensor_value_info( + "past", TensorProto.FLOAT, [1, 2, "seq", 4] + ) + new_val = helper.make_tensor_value_info( + "new_val", TensorProto.FLOAT, [1, 2, 1, 4] + ) + out = helper.make_tensor_value_info( + "out", TensorProto.FLOAT, [1, 2, "seq_plus_1", 4] + ) + + node = helper.make_node( + "Concat", inputs=["past", "new_val"], outputs=["out"], axis=2 + ) + graph = helper.make_graph([node], "kvconcat_graph", [past, new_val], [out]) + model = helper.make_model( + graph, + opset_imports=[helper.make_opsetid("", 13)], + ) + model.ir_version = 8 + onnx.checker.check_model(model) + path = os.path.join(OUT_DIR, "kvconcat.onnx") + onnx.save(model, path) + return path + + +if __name__ == "__main__": + for gen in [make_add_f32, make_matmul_dynamic, make_kvconcat]: + path = gen() + size = os.path.getsize(path) + print(f"{os.path.basename(path):25s} {size:4d} bytes") diff --git a/go/testdata/kvconcat.onnx b/go/testdata/kvconcat.onnx new file mode 100644 index 0000000000000000000000000000000000000000..c7176550ae14bc813205568a1277bff8bff1c512 GIT binary patch literal 181 zcmd;G$(2)3ooYLZWLy%ccEL;o%yZ}gMB7Xn? literal 0 HcmV?d00001 diff --git a/go/testdata/matmul_dynamic.onnx b/go/testdata/matmul_dynamic.onnx new file mode 100644 index 0000000000000000000000000000000000000000..db7698298e3705b7d2ecdc3c8155ba9f8c091445 GIT binary patch literal 116 zcmdVssK>bXH>XO)T*(%@N|sO)SYR&52JhN-W5Tk^m|Y66O-*U=-rz sVi#gfN-Rmv;9}xni4q3ssN5NwawaDhE(QT!0KRAu+yDRo literal 0 HcmV?d00001 From fed64eea2ea27a46e1b79b448ff0cd7e80b4e4df Mon Sep 17 00:00:00 2001 From: Danny Date: Wed, 8 Jul 2026 14:15:20 +0700 Subject: [PATCH 2/4] [Go] Fix crash bugs, add test coverage, bump API version to 27 Fix TOCTOU race in NewIOBinding (use-after-free on concurrent Close), empty outputNames panic in Run/RunWithOptions, and metadata use-after- close segfault. Add 15 tests covering all identified coverage gaps. Remove dead code. Bump minimum ORT version to 1.27.0 for getter APIs. --- go/onnxruntime/cshim.c | 10 ++ go/onnxruntime/cshim.h | 9 +- go/onnxruntime/errors.go | 9 +- go/onnxruntime/iobinding.go | 10 +- go/onnxruntime/iobinding_test.go | 117 +++++++++++++++ go/onnxruntime/metadata.go | 19 +++ go/onnxruntime/metadata_test.go | 38 +++++ go/onnxruntime/options.go | 18 +++ go/onnxruntime/options_test.go | 64 ++++++++ go/onnxruntime/run_options_test.go | 30 ++++ go/onnxruntime/session.go | 156 ++++++-------------- go/onnxruntime/session_test.go | 63 ++++++++ go/onnxruntime/string_tensor_test.go | 7 + go/onnxruntime/tensor_test.go | 209 +++++++++++++++++++++++++++ 14 files changed, 635 insertions(+), 124 deletions(-) diff --git a/go/onnxruntime/cshim.c b/go/onnxruntime/cshim.c index 82354e59a5321..d0d2e8545c297 100644 --- a/go/onnxruntime/cshim.c +++ b/go/onnxruntime/cshim.c @@ -457,6 +457,16 @@ OrtStatusPtr ort_CreateMemoryInfo(const char *name, enum OrtAllocatorType type, return g_api->CreateMemoryInfo(name, type, id, mem_type, out); } +// Session options getters (since 1.27) + +OrtStatusPtr ort_GetMemPatternEnabled(const OrtSessionOptions *opts, int *out) { + return g_api->GetMemPatternEnabled(opts, out); +} + +OrtStatusPtr ort_GetSessionExecutionMode(const OrtSessionOptions *opts, ExecutionMode *out) { + return g_api->GetSessionExecutionMode(opts, out); +} + // Error handling OrtErrorCode ort_GetErrorCode(const OrtStatus *status) { diff --git a/go/onnxruntime/cshim.h b/go/onnxruntime/cshim.h index 41935d1bc008c..4b52e9413416c 100644 --- a/go/onnxruntime/cshim.h +++ b/go/onnxruntime/cshim.h @@ -4,8 +4,9 @@ #include "onnxruntime_c_api.h" // Minimum ORT API version required by these bindings. -// Covers all functions used, including UpdateCUDAProviderOptions (1.16). -#define ORT_GO_API_VERSION 17 +// Requires ORT >= 1.27.0. Covers all functions used, including +// GetMemPatternEnabled and GetSessionExecutionMode (1.27). +#define ORT_GO_API_VERSION 27 // Initialize the global OrtApi pointer from a resolved OrtGetApiBase function. // Returns 0 on success, 1 if apiBase is NULL, 2 if GetApi returns NULL. @@ -170,6 +171,10 @@ OrtStatusPtr ort_AddInitializer(OrtSessionOptions *opts, const char *name, const // Session profiling OrtStatusPtr ort_SessionEndProfiling(OrtSession *session, OrtAllocator *allocator, char **out); +// Session options getters (since 1.27) +OrtStatusPtr ort_GetMemPatternEnabled(const OrtSessionOptions *opts, int *out); +OrtStatusPtr ort_GetSessionExecutionMode(const OrtSessionOptions *opts, ExecutionMode *out); + // Value type OrtStatusPtr ort_GetValueType(const OrtValue *value, enum ONNXType *out); OrtStatusPtr ort_GetValueCount(const OrtValue *value, size_t *out); diff --git a/go/onnxruntime/errors.go b/go/onnxruntime/errors.go index e5d71c0e72b42..f28c34f99fc72 100644 --- a/go/onnxruntime/errors.go +++ b/go/onnxruntime/errors.go @@ -4,10 +4,7 @@ package onnxruntime #include "cshim.h" */ import "C" -import ( - "fmt" - "unsafe" -) +import "fmt" // ErrorCode mirrors OrtErrorCode from the C API. type ErrorCode int @@ -87,7 +84,3 @@ func wrapErr(context string, err error) error { var errNotInitialized = fmt.Errorf("ort: not initialized; call Init() first") var errShutdown = fmt.Errorf("ort: already shut down") - -func statusToErr(status unsafe.Pointer) error { - return checkStatus((*C.OrtStatus)(status)) -} diff --git a/go/onnxruntime/iobinding.go b/go/onnxruntime/iobinding.go index a13358420cccd..feb3dc4018ae3 100644 --- a/go/onnxruntime/iobinding.go +++ b/go/onnxruntime/iobinding.go @@ -5,7 +5,10 @@ package onnxruntime #include */ import "C" -import "unsafe" +import ( + "fmt" + "unsafe" +) type AllocatorType int @@ -74,6 +77,11 @@ func NewIOBinding(session *Session) (*IOBinding, error) { if err := checkInit(); err != nil { return nil, err } + session.mu.RLock() + defer session.mu.RUnlock() + if session.closed { + return nil, fmt.Errorf("ort: create io binding: session is closed") + } var binding *C.OrtIoBinding if err := checkStatus(C.ort_CreateIoBinding(session.handle, &binding)); err != nil { return nil, wrapErr("create io binding", err) diff --git a/go/onnxruntime/iobinding_test.go b/go/onnxruntime/iobinding_test.go index 0c5755d02c06d..ab97872fe674e 100644 --- a/go/onnxruntime/iobinding_test.go +++ b/go/onnxruntime/iobinding_test.go @@ -1,6 +1,7 @@ package onnxruntime import ( + "math" "testing" ) @@ -99,3 +100,119 @@ func TestIOBindingClear(t *testing.T) { binding.ClearInputs() binding.ClearOutputs() } + +func TestIOBindingOnClosedSession(t *testing.T) { + sess, err := NewSession(testdataPath("add_f32.onnx"), nil) + if err != nil { + t.Fatal(err) + } + sess.Close() + + _, err = NewIOBinding(sess) + if err == nil { + t.Fatal("expected error creating IOBinding on closed session") + } +} + +func TestIOBindingBindOutput(t *testing.T) { + sess, err := NewSession(testdataPath("add_f32.onnx"), nil) + if err != nil { + t.Fatal(err) + } + defer sess.Close() + + binding, err := NewIOBinding(sess) + if err != nil { + t.Fatal(err) + } + defer binding.Close() + + a, _ := CreateTensor[float32]([]int64{2, 3}, []float32{1, 2, 3, 4, 5, 6}) + b, _ := CreateTensor[float32]([]int64{2, 3}, []float32{10, 20, 30, 40, 50, 60}) + defer a.Close() + defer b.Close() + + if err := binding.BindInput("A", a); err != nil { + t.Fatal(err) + } + if err := binding.BindInput("B", b); err != nil { + t.Fatal(err) + } + + outTensor, err := CreateTensor[float32]([]int64{2, 3}, []float32{0, 0, 0, 0, 0, 0}) + if err != nil { + t.Fatal(err) + } + defer outTensor.Close() + + if err := binding.BindOutput("C", outTensor); err != nil { + t.Fatal(err) + } + + if err := binding.Run(nil); err != nil { + t.Fatal(err) + } + + data, err := TensorData[float32](outTensor) + if err != nil { + t.Fatal(err) + } + + expected := []float32{11, 22, 33, 44, 55, 66} + for i, v := range data { + if math.Abs(float64(v-expected[i])) > 1e-6 { + t.Errorf("output[%d]: expected %f, got %f", i, expected[i], v) + } + } +} + +func TestIOBindingOutputNames(t *testing.T) { + sess, err := NewSession(testdataPath("add_f32.onnx"), nil) + if err != nil { + t.Fatal(err) + } + defer sess.Close() + + binding, err := NewIOBinding(sess) + if err != nil { + t.Fatal(err) + } + defer binding.Close() + + a, _ := CreateTensor[float32]([]int64{2, 3}, []float32{1, 2, 3, 4, 5, 6}) + b, _ := CreateTensor[float32]([]int64{2, 3}, []float32{10, 20, 30, 40, 50, 60}) + defer a.Close() + defer b.Close() + + if err := binding.BindInput("A", a); err != nil { + t.Fatal(err) + } + if err := binding.BindInput("B", b); err != nil { + t.Fatal(err) + } + + memInfo, err := NewCPUMemoryInfo() + if err != nil { + t.Fatal(err) + } + defer memInfo.Close() + + if err := binding.BindOutputToDevice("C", memInfo); err != nil { + t.Fatal(err) + } + + if err := binding.Run(nil); err != nil { + t.Fatal(err) + } + + names, err := binding.OutputNames() + if err != nil { + t.Fatal(err) + } + if len(names) != 1 { + t.Fatalf("expected 1 output name, got %d", len(names)) + } + if names[0] != "C" { + t.Errorf("expected output name 'C', got %q", names[0]) + } +} diff --git a/go/onnxruntime/metadata.go b/go/onnxruntime/metadata.go index 24a325bcde707..3d73b4068fd86 100644 --- a/go/onnxruntime/metadata.go +++ b/go/onnxruntime/metadata.go @@ -61,6 +61,9 @@ func (m *ModelMetadata) GraphDescription() (string, error) { } func (m *ModelMetadata) Version() (int64, error) { + if err := m.checkClosed(); err != nil { + return 0, err + } var version C.int64_t if err := checkStatus(C.ort_ModelMetadataGetVersion(m.handle, &version)); err != nil { return 0, wrapErr("metadata version", err) @@ -69,6 +72,9 @@ func (m *ModelMetadata) Version() (int64, error) { } func (m *ModelMetadata) CustomMetadataKeys() ([]string, error) { + if err := m.checkClosed(); err != nil { + return nil, err + } alloc, err := defaultAllocator() if err != nil { return nil, wrapErr("metadata custom keys", err) @@ -97,6 +103,9 @@ func (m *ModelMetadata) CustomMetadataKeys() ([]string, error) { } func (m *ModelMetadata) LookupCustomMetadata(key string) (string, error) { + if err := m.checkClosed(); err != nil { + return "", err + } alloc, err := defaultAllocator() if err != nil { return "", wrapErr("metadata lookup", err) @@ -129,7 +138,17 @@ func (m *ModelMetadata) Close() error { return nil } +func (m *ModelMetadata) checkClosed() error { + if m.closed { + return fmt.Errorf("ort: metadata: already closed") + } + return nil +} + func (m *ModelMetadata) getString(fn func(*C.OrtAllocator, **C.char) error, context string) (string, error) { + if err := m.checkClosed(); err != nil { + return "", err + } alloc, err := defaultAllocator() if err != nil { return "", wrapErr("metadata "+context, err) diff --git a/go/onnxruntime/metadata_test.go b/go/onnxruntime/metadata_test.go index b62b49da0de31..f3deecc681084 100644 --- a/go/onnxruntime/metadata_test.go +++ b/go/onnxruntime/metadata_test.go @@ -68,3 +68,41 @@ func TestModelMetadataDoubleClose(t *testing.T) { meta.Close() meta.Close() } + +func TestModelMetadataUseAfterClose(t *testing.T) { + sess, err := NewSession(testdataPath("add_f32.onnx"), nil) + if err != nil { + t.Fatal(err) + } + defer sess.Close() + + meta, err := sess.ModelMetadata() + if err != nil { + t.Fatal(err) + } + meta.Close() + + _, err = meta.ProducerName() + if err == nil { + t.Fatal("expected error calling ProducerName after Close") + } +} + +func TestModelMetadataGraphDescription(t *testing.T) { + sess, err := NewSession(testdataPath("add_f32.onnx"), nil) + if err != nil { + t.Fatal(err) + } + defer sess.Close() + + meta, err := sess.ModelMetadata() + if err != nil { + t.Fatal(err) + } + defer meta.Close() + + _, err = meta.GraphDescription() + if err != nil { + t.Errorf("GraphDescription: %v", err) + } +} diff --git a/go/onnxruntime/options.go b/go/onnxruntime/options.go index 73e833eff7edc..b4673cb0e8ff5 100644 --- a/go/onnxruntime/options.go +++ b/go/onnxruntime/options.go @@ -131,6 +131,24 @@ func (o *SessionOptions) SetExecutionMode(mode ExecutionMode) error { checkStatus(C.ort_SetSessionExecutionMode(o.handle, C.ExecutionMode(mode)))) } +// GetExecutionMode returns the current execution mode. Requires ORT >= 1.27. +func (o *SessionOptions) GetExecutionMode() (ExecutionMode, error) { + var mode C.ExecutionMode + if err := checkStatus(C.ort_GetSessionExecutionMode(o.handle, &mode)); err != nil { + return 0, wrapErr("get execution mode", err) + } + return ExecutionMode(mode), nil +} + +// IsMemPatternEnabled reports whether memory pattern optimization is enabled. Requires ORT >= 1.27. +func (o *SessionOptions) IsMemPatternEnabled() (bool, error) { + var out C.int + if err := checkStatus(C.ort_GetMemPatternEnabled(o.handle, &out)); err != nil { + return false, wrapErr("get mem pattern enabled", err) + } + return out != 0, nil +} + // AddInitializer overrides a model initializer with the given tensor value. func (o *SessionOptions) AddInitializer(name string, value *Tensor) error { cName := C.CString(name) diff --git a/go/onnxruntime/options_test.go b/go/onnxruntime/options_test.go index fcc1611a0c25e..422bfad486166 100644 --- a/go/onnxruntime/options_test.go +++ b/go/onnxruntime/options_test.go @@ -62,6 +62,57 @@ func TestSessionOptionsExecutionMode(t *testing.T) { } } +func TestSessionOptionsGetExecutionMode(t *testing.T) { + opts, err := NewSessionOptions() + if err != nil { + t.Fatal(err) + } + defer opts.Close() + + opts.SetExecutionMode(ExecutionModeParallel) + mode, err := opts.GetExecutionMode() + if err != nil { + t.Fatal(err) + } + if mode != ExecutionModeParallel { + t.Errorf("expected Parallel, got %d", mode) + } + + opts.SetExecutionMode(ExecutionModeSequential) + mode, err = opts.GetExecutionMode() + if err != nil { + t.Fatal(err) + } + if mode != ExecutionModeSequential { + t.Errorf("expected Sequential, got %d", mode) + } +} + +func TestSessionOptionsIsMemPatternEnabled(t *testing.T) { + opts, err := NewSessionOptions() + if err != nil { + t.Fatal(err) + } + defer opts.Close() + + enabled, err := opts.IsMemPatternEnabled() + if err != nil { + t.Fatal(err) + } + if !enabled { + t.Error("expected mem pattern enabled by default") + } + + opts.DisableMemPattern() + enabled, err = opts.IsMemPatternEnabled() + if err != nil { + t.Fatal(err) + } + if enabled { + t.Error("expected mem pattern disabled after DisableMemPattern") + } +} + func TestSessionOptionsProfiling(t *testing.T) { opts, err := NewSessionOptions() if err != nil { @@ -116,6 +167,19 @@ func TestSessionOptionsFreeDimension(t *testing.T) { } } +func TestAppendExecutionProviderUnknown(t *testing.T) { + opts, err := NewSessionOptions() + if err != nil { + t.Fatal(err) + } + defer opts.Close() + + err = opts.AppendExecutionProvider("NoSuchProvider_XYZ_999", nil) + if err == nil { + t.Fatal("expected error for unknown execution provider") + } +} + func TestTensorIsTensor(t *testing.T) { tensor, err := CreateTensor[float32]([]int64{2}, []float32{1, 2}) if err != nil { diff --git a/go/onnxruntime/run_options_test.go b/go/onnxruntime/run_options_test.go index 42de03fb5e340..f93c2d561fac5 100644 --- a/go/onnxruntime/run_options_test.go +++ b/go/onnxruntime/run_options_test.go @@ -62,3 +62,33 @@ func TestRunWithOptions(t *testing.T) { t.Errorf("expected 11, got %f", data[0]) } } + +func TestRunOptionsTerminate(t *testing.T) { + sess, err := NewSession(testdataPath("add_f32.onnx"), nil) + if err != nil { + t.Fatal(err) + } + defer sess.Close() + + opts, err := NewRunOptions() + if err != nil { + t.Fatal(err) + } + defer opts.Close() + + if err := opts.SetTerminate(); err != nil { + t.Fatal(err) + } + + a, _ := CreateTensor[float32]([]int64{2, 3}, []float32{1, 2, 3, 4, 5, 6}) + b, _ := CreateTensor[float32]([]int64{2, 3}, []float32{10, 20, 30, 40, 50, 60}) + defer a.Close() + defer b.Close() + + _, err = sess.RunWithOptions(context.Background(), opts, map[string]*Tensor{ + "A": a, "B": b, + }, []string{"C"}) + if err == nil { + t.Fatal("expected error from terminated run options") + } +} diff --git a/go/onnxruntime/session.go b/go/onnxruntime/session.go index 96030e069cad0..92181d3db211e 100644 --- a/go/onnxruntime/session.go +++ b/go/onnxruntime/session.go @@ -116,111 +116,11 @@ func (s *Session) Run(ctx context.Context, inputs map[string]*Tensor, outputName } } - nInputs := len(inputs) - nOutputs := len(outputNames) - - cInputNames := make([]*C.char, nInputs) - cInputValues := make([]*C.OrtValue, nInputs) - i := 0 - for name, tensor := range inputs { - cInputNames[i] = C.CString(name) - cInputValues[i] = tensor.value - i++ + if len(outputNames) == 0 { + return nil, fmt.Errorf("ort: run: no output names specified") } - defer func() { - for _, cn := range cInputNames { - C.free(unsafe.Pointer(cn)) - } - }() - - cOutputNames := make([]*C.char, nOutputs) - for i, name := range outputNames { - cOutputNames[i] = C.CString(name) - } - defer func() { - for _, cn := range cOutputNames { - C.free(unsafe.Pointer(cn)) - } - }() - - cOutputValues := make([]*C.OrtValue, nOutputs) - - var runOpts *C.OrtRunOptions - var stopWatcher chan struct{} - - if ctx.Done() != nil { - if err := checkStatus(C.ort_CreateRunOptions(&runOpts)); err != nil { - return nil, wrapErr("create run options", err) - } - defer C.ort_ReleaseRunOptions(runOpts) - - stopWatcher = make(chan struct{}) - watcherDone := make(chan struct{}) - done := ctx.Done() - go func() { - defer close(watcherDone) - select { - case <-done: - C.ort_RunOptionsSetTerminate(runOpts) - case <-stopWatcher: - } - }() - defer func() { - close(stopWatcher) - <-watcherDone - }() - } - - var inNamesPtr **C.char - var inValuesPtr **C.OrtValue - if nInputs > 0 { - inNamesPtr = &cInputNames[0] - inValuesPtr = &cInputValues[0] - } - - var outNamesPtr **C.char - if nOutputs > 0 { - outNamesPtr = &cOutputNames[0] - } - - err := checkStatus(C.ort_Run( - s.handle, runOpts, - inNamesPtr, - inValuesPtr, C.size_t(nInputs), - outNamesPtr, C.size_t(nOutputs), - &cOutputValues[0], - )) - if err != nil { - for _, v := range cOutputValues { - if v != nil { - C.ort_ReleaseValue(v) - } - } - if ctx.Err() != nil { - return nil, ctx.Err() - } - return nil, wrapErr("run", err) - } - - result := make(map[string]*Tensor, nOutputs) - for i, name := range outputNames { - t, err := wrapOutputTensor(cOutputValues[i]) - if err != nil { - for _, v := range cOutputValues[i+1:] { - if v != nil { - C.ort_ReleaseValue(v) - } - } - for _, existing := range result { - _ = existing.Close() - } - return nil, wrapErr("wrap output "+name, err) - } - result[name] = t - } - - return result, nil + return s.runInner(ctx, inputs, outputNames, nil) } // RunWithOptions executes inference with explicit run options. @@ -243,9 +143,21 @@ func (s *Session) RunWithOptions(ctx context.Context, opts *RunOptions, inputs m } } + if len(outputNames) == 0 { + return nil, fmt.Errorf("ort: run: no output names specified") + } + + var runOpts *C.OrtRunOptions + if opts != nil { + runOpts = opts.handle + } + + return s.runInner(ctx, inputs, outputNames, runOpts) +} + +func (s *Session) runInner(ctx context.Context, inputs map[string]*Tensor, outputNames []string, runOpts *C.OrtRunOptions) (map[string]*Tensor, error) { nInputs := len(inputs) nOutputs := len(outputNames) - cInputNames := make([]*C.char, nInputs) cInputValues := make([]*C.OrtValue, nInputs) i := 0 @@ -272,9 +184,32 @@ func (s *Session) RunWithOptions(ctx context.Context, opts *RunOptions, inputs m cOutputValues := make([]*C.OrtValue, nOutputs) - var runOpts *C.OrtRunOptions - if opts != nil { - runOpts = opts.handle + ownRunOpts := false + if runOpts == nil && ctx.Done() != nil { + var err error + if err = checkStatus(C.ort_CreateRunOptions(&runOpts)); err != nil { + return nil, wrapErr("create run options", err) + } + ownRunOpts = true + + stopWatcher := make(chan struct{}) + watcherDone := make(chan struct{}) + done := ctx.Done() + go func() { + defer close(watcherDone) + select { + case <-done: + C.ort_RunOptionsSetTerminate(runOpts) + case <-stopWatcher: + } + }() + defer func() { + close(stopWatcher) + <-watcherDone + }() + } + if ownRunOpts { + defer C.ort_ReleaseRunOptions(runOpts) } var inNamesPtr **C.char @@ -284,16 +219,11 @@ func (s *Session) RunWithOptions(ctx context.Context, opts *RunOptions, inputs m inValuesPtr = &cInputValues[0] } - var outNamesPtr **C.char - if nOutputs > 0 { - outNamesPtr = &cOutputNames[0] - } - err := checkStatus(C.ort_Run( s.handle, runOpts, inNamesPtr, inValuesPtr, C.size_t(nInputs), - outNamesPtr, C.size_t(nOutputs), + &cOutputNames[0], C.size_t(nOutputs), &cOutputValues[0], )) diff --git a/go/onnxruntime/session_test.go b/go/onnxruntime/session_test.go index b6379f6537d07..b038b0c896400 100644 --- a/go/onnxruntime/session_test.go +++ b/go/onnxruntime/session_test.go @@ -322,6 +322,69 @@ func TestSessionOptions(t *testing.T) { sess.Close() } +func TestRunEmptyOutputNames(t *testing.T) { + sess, err := NewSession(testdataPath("add_f32.onnx"), nil) + if err != nil { + t.Fatal(err) + } + defer sess.Close() + + a, _ := CreateTensor[float32]([]int64{2, 3}, []float32{1, 2, 3, 4, 5, 6}) + b, _ := CreateTensor[float32]([]int64{2, 3}, []float32{1, 1, 1, 1, 1, 1}) + defer a.Close() + defer b.Close() + + _, err = sess.Run(context.Background(), map[string]*Tensor{ + "A": a, "B": b, + }, []string{}) + if err == nil { + t.Fatal("expected error for empty output names") + } +} + +func TestRunAfterClose(t *testing.T) { + sess, err := NewSession(testdataPath("add_f32.onnx"), nil) + if err != nil { + t.Fatal(err) + } + sess.Close() + + a, _ := CreateTensor[float32]([]int64{2, 3}, []float32{1, 2, 3, 4, 5, 6}) + b, _ := CreateTensor[float32]([]int64{2, 3}, []float32{1, 1, 1, 1, 1, 1}) + defer a.Close() + defer b.Close() + + _, err = sess.Run(context.Background(), map[string]*Tensor{ + "A": a, "B": b, + }, nil) + if err == nil { + t.Fatal("expected error running on closed session") + } + if got := err.Error(); got != "ort: run: session is closed" { + t.Errorf("unexpected error message: %s", got) + } +} + +func TestNewSessionFromBytesCorrupt(t *testing.T) { + garbage := []byte("this is not a valid onnx model at all") + _, err := NewSessionFromBytes(garbage, nil) + if err == nil { + t.Fatal("expected error for corrupt model bytes") + } +} + +func TestNewSessionFromBytesEmpty(t *testing.T) { + _, err := NewSessionFromBytes(nil, nil) + if err == nil { + t.Fatal("expected error for nil model bytes") + } + + _, err = NewSessionFromBytes([]byte{}, nil) + if err == nil { + t.Fatal("expected error for empty model bytes") + } +} + func mapKeys(m map[string]*Tensor) []string { keys := make([]string, 0, len(m)) for k := range m { diff --git a/go/onnxruntime/string_tensor_test.go b/go/onnxruntime/string_tensor_test.go index 000f1b5867fb9..cef1f42118348 100644 --- a/go/onnxruntime/string_tensor_test.go +++ b/go/onnxruntime/string_tensor_test.go @@ -82,3 +82,10 @@ func TestStringDataOnNonStringTensor(t *testing.T) { t.Fatal("expected error calling StringData on float tensor") } } + +func TestCreateStringTensorShapeMismatch(t *testing.T) { + _, err := CreateStringTensor([]int64{2, 3}, []string{"a", "b"}) + if err == nil { + t.Fatal("expected error for shape/data mismatch") + } +} diff --git a/go/onnxruntime/tensor_test.go b/go/onnxruntime/tensor_test.go index 30c1c071d34e9..fed2815c80d3b 100644 --- a/go/onnxruntime/tensor_test.go +++ b/go/onnxruntime/tensor_test.go @@ -181,3 +181,212 @@ func TestScalarTensor(t *testing.T) { t.Errorf("expected 1 element, got %d", tensor.ElementCount()) } } + +func TestCreateTensorAllTypes(t *testing.T) { + t.Run("uint8", func(t *testing.T) { + data := []uint8{1, 2, 3} + tensor, err := CreateTensor[uint8]([]int64{3}, data) + if err != nil { + t.Fatal(err) + } + defer tensor.Close() + if tensor.DataType() != TensorElementDataTypeUint8 { + t.Errorf("expected Uint8, got %s", tensor.DataType()) + } + got, err := TensorData[uint8](tensor) + if err != nil { + t.Fatal(err) + } + for i, v := range got { + if v != data[i] { + t.Errorf("[%d]: expected %d, got %d", i, data[i], v) + } + } + }) + + t.Run("int8", func(t *testing.T) { + data := []int8{-1, 0, 1} + tensor, err := CreateTensor[int8]([]int64{3}, data) + if err != nil { + t.Fatal(err) + } + defer tensor.Close() + if tensor.DataType() != TensorElementDataTypeInt8 { + t.Errorf("expected Int8, got %s", tensor.DataType()) + } + got, err := TensorData[int8](tensor) + if err != nil { + t.Fatal(err) + } + for i, v := range got { + if v != data[i] { + t.Errorf("[%d]: expected %d, got %d", i, data[i], v) + } + } + }) + + t.Run("int16", func(t *testing.T) { + data := []int16{-100, 0, 100} + tensor, err := CreateTensor[int16]([]int64{3}, data) + if err != nil { + t.Fatal(err) + } + defer tensor.Close() + if tensor.DataType() != TensorElementDataTypeInt16 { + t.Errorf("expected Int16, got %s", tensor.DataType()) + } + got, err := TensorData[int16](tensor) + if err != nil { + t.Fatal(err) + } + for i, v := range got { + if v != data[i] { + t.Errorf("[%d]: expected %d, got %d", i, data[i], v) + } + } + }) + + t.Run("uint16", func(t *testing.T) { + data := []uint16{0, 1000, 65535} + tensor, err := CreateTensor[uint16]([]int64{3}, data) + if err != nil { + t.Fatal(err) + } + defer tensor.Close() + if tensor.DataType() != TensorElementDataTypeUint16 { + t.Errorf("expected Uint16, got %s", tensor.DataType()) + } + got, err := TensorData[uint16](tensor) + if err != nil { + t.Fatal(err) + } + for i, v := range got { + if v != data[i] { + t.Errorf("[%d]: expected %d, got %d", i, data[i], v) + } + } + }) + + t.Run("int32", func(t *testing.T) { + data := []int32{-1, 0, 2147483647} + tensor, err := CreateTensor[int32]([]int64{3}, data) + if err != nil { + t.Fatal(err) + } + defer tensor.Close() + if tensor.DataType() != TensorElementDataTypeInt32 { + t.Errorf("expected Int32, got %s", tensor.DataType()) + } + got, err := TensorData[int32](tensor) + if err != nil { + t.Fatal(err) + } + for i, v := range got { + if v != data[i] { + t.Errorf("[%d]: expected %d, got %d", i, data[i], v) + } + } + }) + + t.Run("uint32", func(t *testing.T) { + data := []uint32{0, 42, 4294967295} + tensor, err := CreateTensor[uint32]([]int64{3}, data) + if err != nil { + t.Fatal(err) + } + defer tensor.Close() + if tensor.DataType() != TensorElementDataTypeUint32 { + t.Errorf("expected Uint32, got %s", tensor.DataType()) + } + got, err := TensorData[uint32](tensor) + if err != nil { + t.Fatal(err) + } + for i, v := range got { + if v != data[i] { + t.Errorf("[%d]: expected %d, got %d", i, data[i], v) + } + } + }) + + t.Run("uint64", func(t *testing.T) { + data := []uint64{0, 42, 18446744073709551615} + tensor, err := CreateTensor[uint64]([]int64{3}, data) + if err != nil { + t.Fatal(err) + } + defer tensor.Close() + if tensor.DataType() != TensorElementDataTypeUint64 { + t.Errorf("expected Uint64, got %s", tensor.DataType()) + } + got, err := TensorData[uint64](tensor) + if err != nil { + t.Fatal(err) + } + for i, v := range got { + if v != data[i] { + t.Errorf("[%d]: expected %d, got %d", i, data[i], v) + } + } + }) + + t.Run("float64", func(t *testing.T) { + data := []float64{1.5, -2.5, 3.14} + tensor, err := CreateTensor[float64]([]int64{3}, data) + if err != nil { + t.Fatal(err) + } + defer tensor.Close() + if tensor.DataType() != TensorElementDataTypeFloat64 { + t.Errorf("expected Float64, got %s", tensor.DataType()) + } + got, err := TensorData[float64](tensor) + if err != nil { + t.Fatal(err) + } + for i, v := range got { + if v != data[i] { + t.Errorf("[%d]: expected %f, got %f", i, data[i], v) + } + } + }) +} + +func TestNewTensorFromBytesUnsupportedDtype(t *testing.T) { + _, err := NewTensorFromBytes(TensorElementDataTypeString, []int64{2}, []byte{0, 0, 0, 0}) + if err == nil { + t.Fatal("expected error for String dtype in NewTensorFromBytes") + } +} + +func TestNewTensorFromBytesFloat16(t *testing.T) { + // IEEE 754 half-precision: 1.0 = 0x3C00, 2.0 = 0x4000 (little-endian) + data := []byte{0x00, 0x3C, 0x00, 0x40} + tensor, err := NewTensorFromBytes(TensorElementDataTypeFloat16, []int64{2}, data) + if err != nil { + t.Fatal(err) + } + defer tensor.Close() + + if tensor.DataType() != TensorElementDataTypeFloat16 { + t.Errorf("expected Float16, got %s", tensor.DataType()) + } + + shape := tensor.Shape() + if len(shape) != 1 || shape[0] != 2 { + t.Errorf("expected shape [2], got %v", shape) + } + + b, err := tensor.Bytes() + if err != nil { + t.Fatal(err) + } + if len(b) != len(data) { + t.Fatalf("expected %d bytes, got %d", len(data), len(b)) + } + for i, v := range b { + if v != data[i] { + t.Errorf("byte[%d]: expected 0x%02X, got 0x%02X", i, data[i], v) + } + } +} From 5688439d8df197ea862ccbd6872b0b9b2286c45c Mon Sep 17 00:00:00 2001 From: Danny Date: Wed, 8 Jul 2026 16:38:46 +0700 Subject: [PATCH 3/4] [Go] Support older ORT versions with API version fallback Try API version 27 first, fall back to 17 for ORT 1.17-1.26. GetExecutionMode and IsMemPatternEnabled return a clear error on older libraries instead of crashing. Minimum ORT is now 1.17. --- go/onnxruntime/cshim.c | 22 +++++++++++++++++++--- go/onnxruntime/cshim.h | 15 +++++++++------ go/onnxruntime/options.go | 7 +++++++ go/onnxruntime/ort.go | 7 +++++-- 4 files changed, 40 insertions(+), 11 deletions(-) diff --git a/go/onnxruntime/cshim.c b/go/onnxruntime/cshim.c index d0d2e8545c297..a9a2b13e8d194 100644 --- a/go/onnxruntime/cshim.c +++ b/go/onnxruntime/cshim.c @@ -1,12 +1,24 @@ #include "cshim.h" static const OrtApi *g_api = NULL; +static int g_api_version = 0; -int ort_init_api(void *get_api_base_fn) { +int ort_init_api(void *get_api_base_fn, int *actual_version) { const OrtApiBase *base = ((const OrtApiBase *(*)(void))get_api_base_fn)(); if (base == NULL) return 1; - g_api = base->GetApi(ORT_GO_API_VERSION); - return g_api == NULL ? 2 : 0; + g_api = base->GetApi(ORT_GO_API_VERSION_MAX); + if (g_api != NULL) { + g_api_version = ORT_GO_API_VERSION_MAX; + *actual_version = g_api_version; + return 0; + } + g_api = base->GetApi(ORT_GO_API_VERSION_MIN); + if (g_api != NULL) { + g_api_version = ORT_GO_API_VERSION_MIN; + *actual_version = g_api_version; + return 0; + } + return 2; } // Environment @@ -459,6 +471,10 @@ OrtStatusPtr ort_CreateMemoryInfo(const char *name, enum OrtAllocatorType type, // Session options getters (since 1.27) +int ort_api_version(void) { + return g_api_version; +} + OrtStatusPtr ort_GetMemPatternEnabled(const OrtSessionOptions *opts, int *out) { return g_api->GetMemPatternEnabled(opts, out); } diff --git a/go/onnxruntime/cshim.h b/go/onnxruntime/cshim.h index 4b52e9413416c..295386bea2d2d 100644 --- a/go/onnxruntime/cshim.h +++ b/go/onnxruntime/cshim.h @@ -3,14 +3,16 @@ #include "onnxruntime_c_api.h" -// Minimum ORT API version required by these bindings. -// Requires ORT >= 1.27.0. Covers all functions used, including -// GetMemPatternEnabled and GetSessionExecutionMode (1.27). -#define ORT_GO_API_VERSION 27 +// Preferred API version (ORT >= 1.27.0, includes getter APIs). +// Falls back to ORT_GO_API_VERSION_MIN for older libraries. +#define ORT_GO_API_VERSION_MAX 27 +#define ORT_GO_API_VERSION_MIN 17 // Initialize the global OrtApi pointer from a resolved OrtGetApiBase function. -// Returns 0 on success, 1 if apiBase is NULL, 2 if GetApi returns NULL. -int ort_init_api(void *get_api_base_fn); +// Tries ORT_GO_API_VERSION_MAX first, falls back to ORT_GO_API_VERSION_MIN. +// Returns 0 on success, 1 if apiBase is NULL, 2 if even min version unsupported. +// Sets *actual_version to the version that was loaded. +int ort_init_api(void *get_api_base_fn, int *actual_version); // Environment OrtStatusPtr ort_CreateEnv(OrtLoggingLevel level, const char *logid, OrtEnv **out); @@ -172,6 +174,7 @@ OrtStatusPtr ort_AddInitializer(OrtSessionOptions *opts, const char *name, const OrtStatusPtr ort_SessionEndProfiling(OrtSession *session, OrtAllocator *allocator, char **out); // Session options getters (since 1.27) +int ort_api_version(void); OrtStatusPtr ort_GetMemPatternEnabled(const OrtSessionOptions *opts, int *out); OrtStatusPtr ort_GetSessionExecutionMode(const OrtSessionOptions *opts, ExecutionMode *out); diff --git a/go/onnxruntime/options.go b/go/onnxruntime/options.go index b4673cb0e8ff5..b09c144ae69ea 100644 --- a/go/onnxruntime/options.go +++ b/go/onnxruntime/options.go @@ -6,6 +6,7 @@ package onnxruntime */ import "C" import ( + "fmt" "sort" "unsafe" ) @@ -133,6 +134,9 @@ func (o *SessionOptions) SetExecutionMode(mode ExecutionMode) error { // GetExecutionMode returns the current execution mode. Requires ORT >= 1.27. func (o *SessionOptions) GetExecutionMode() (ExecutionMode, error) { + if apiVersion < 27 { + return 0, fmt.Errorf("ort: GetExecutionMode requires ORT >= 1.27 (have API version %d)", apiVersion) + } var mode C.ExecutionMode if err := checkStatus(C.ort_GetSessionExecutionMode(o.handle, &mode)); err != nil { return 0, wrapErr("get execution mode", err) @@ -142,6 +146,9 @@ func (o *SessionOptions) GetExecutionMode() (ExecutionMode, error) { // IsMemPatternEnabled reports whether memory pattern optimization is enabled. Requires ORT >= 1.27. func (o *SessionOptions) IsMemPatternEnabled() (bool, error) { + if apiVersion < 27 { + return false, fmt.Errorf("ort: IsMemPatternEnabled requires ORT >= 1.27 (have API version %d)", apiVersion) + } var out C.int if err := checkStatus(C.ort_GetMemPatternEnabled(o.handle, &out)); err != nil { return false, wrapErr("get mem pattern enabled", err) diff --git a/go/onnxruntime/ort.go b/go/onnxruntime/ort.go index ef02338a89ba0..96e5bd462c1bc 100644 --- a/go/onnxruntime/ort.go +++ b/go/onnxruntime/ort.go @@ -19,6 +19,7 @@ var ( libPath string initialized bool shutdown bool + apiVersion int env *C.OrtEnv cpuMemInfo *C.OrtMemoryInfo sessionCount atomic.Int64 @@ -53,13 +54,15 @@ func Init() error { return wrapErr("load library", err) } - rc := C.ort_init_api(fn) + var cVersion C.int + rc := C.ort_init_api(fn, &cVersion) if rc == 1 { return wrapErr("init api", fmt.Errorf("OrtGetApiBase returned NULL")) } if rc == 2 { - return wrapErr("init api", fmt.Errorf("GetApi(%d) returned NULL; ORT library may be too old", C.ORT_GO_API_VERSION)) + return wrapErr("init api", fmt.Errorf("GetApi failed; ORT library may be too old (need API >= %d)", C.ORT_GO_API_VERSION_MIN)) } + apiVersion = int(cVersion) cLogID := C.CString("onnxruntime-go") defer C.free(unsafe.Pointer(cLogID)) From 54c30aa53140e64b778267abe3fa81d63bab60e9 Mon Sep 17 00:00:00 2001 From: Danny Date: Thu, 9 Jul 2026 12:50:14 +0700 Subject: [PATCH 4/4] [Go] Add version API, cached Run, telemetry, sequence/map creation Add GetVersion/APIVersion, cache CString names in Session for zero- alloc Run hot path, enable/disable telemetry, SetOptimizedModelFilePath, RegisterCustomOpsLibrary, Has/GetSessionConfigEntry, NewSequence, NewMap, NewMapFromGoMap. Fix API version probing to try 27 down to 17 for broader ORT compatibility. Fix IsSequence/IsMap enum values. --- go/onnxruntime/cshim.c | 64 ++++++++++++++++++++------ go/onnxruntime/cshim.h | 19 +++++++- go/onnxruntime/options.go | 45 ++++++++++++++++++ go/onnxruntime/options_test.go | 34 ++++++++++++++ go/onnxruntime/ort.go | 38 ++++++++++++++++ go/onnxruntime/ort_test.go | 20 ++++++++ go/onnxruntime/session.go | 58 +++++++++++++++++------- go/onnxruntime/tensor.go | 83 ++++++++++++++++++++++++++++++++-- go/onnxruntime/tensor_test.go | 68 ++++++++++++++++++++++++++++ 9 files changed, 393 insertions(+), 36 deletions(-) diff --git a/go/onnxruntime/cshim.c b/go/onnxruntime/cshim.c index a9a2b13e8d194..da241a9b4f8eb 100644 --- a/go/onnxruntime/cshim.c +++ b/go/onnxruntime/cshim.c @@ -2,25 +2,27 @@ static const OrtApi *g_api = NULL; static int g_api_version = 0; +static const char *g_ort_version = NULL; int ort_init_api(void *get_api_base_fn, int *actual_version) { const OrtApiBase *base = ((const OrtApiBase *(*)(void))get_api_base_fn)(); if (base == NULL) return 1; - g_api = base->GetApi(ORT_GO_API_VERSION_MAX); - if (g_api != NULL) { - g_api_version = ORT_GO_API_VERSION_MAX; - *actual_version = g_api_version; - return 0; - } - g_api = base->GetApi(ORT_GO_API_VERSION_MIN); - if (g_api != NULL) { - g_api_version = ORT_GO_API_VERSION_MIN; - *actual_version = g_api_version; - return 0; + g_ort_version = base->GetVersionString(); + for (int v = ORT_GO_API_VERSION_MAX; v >= ORT_GO_API_VERSION_MIN; v--) { + g_api = base->GetApi((uint32_t)v); + if (g_api != NULL) { + g_api_version = v; + *actual_version = v; + return 0; + } } return 2; } +const char *ort_GetVersionString(void) { + return g_ort_version; +} + // Environment OrtStatusPtr ort_CreateEnv(OrtLoggingLevel level, const char *logid, OrtEnv **out) { @@ -462,6 +464,13 @@ OrtStatusPtr ort_GetValue(const OrtValue *value, int index, OrtAllocator *alloca return g_api->GetValue(value, index, allocator, out); } +// Value creation (sequence/map) + +OrtStatusPtr ort_CreateValue(const OrtValue *const *in, size_t num_values, + enum ONNXType value_type, OrtValue **out) { + return g_api->CreateValue(in, num_values, value_type, out); +} + // Memory info OrtStatusPtr ort_CreateMemoryInfo(const char *name, enum OrtAllocatorType type, @@ -471,10 +480,6 @@ OrtStatusPtr ort_CreateMemoryInfo(const char *name, enum OrtAllocatorType type, // Session options getters (since 1.27) -int ort_api_version(void) { - return g_api_version; -} - OrtStatusPtr ort_GetMemPatternEnabled(const OrtSessionOptions *opts, int *out) { return g_api->GetMemPatternEnabled(opts, out); } @@ -483,6 +488,35 @@ OrtStatusPtr ort_GetSessionExecutionMode(const OrtSessionOptions *opts, Executio return g_api->GetSessionExecutionMode(opts, out); } +// Telemetry + +OrtStatusPtr ort_EnableTelemetryEvents(const OrtEnv *env) { + return g_api->EnableTelemetryEvents(env); +} + +OrtStatusPtr ort_DisableTelemetryEvents(const OrtEnv *env) { + return g_api->DisableTelemetryEvents(env); +} + +// Additional session options (continued) + +OrtStatusPtr ort_SetOptimizedModelFilePath(OrtSessionOptions *opts, const char *path) { + return g_api->SetOptimizedModelFilePath(opts, path); +} + +OrtStatusPtr ort_RegisterCustomOpsLibrary_V2(OrtSessionOptions *opts, const char *path) { + return g_api->RegisterCustomOpsLibrary_V2(opts, path); +} + +OrtStatusPtr ort_HasSessionConfigEntry(const OrtSessionOptions *opts, const char *key, int *out) { + return g_api->HasSessionConfigEntry(opts, key, out); +} + +OrtStatusPtr ort_GetSessionConfigEntry(const OrtSessionOptions *opts, const char *key, + char *value, size_t *size) { + return g_api->GetSessionConfigEntry(opts, key, value, size); +} + // Error handling OrtErrorCode ort_GetErrorCode(const OrtStatus *status) { diff --git a/go/onnxruntime/cshim.h b/go/onnxruntime/cshim.h index 295386bea2d2d..c4204f8ce4edf 100644 --- a/go/onnxruntime/cshim.h +++ b/go/onnxruntime/cshim.h @@ -173,8 +173,10 @@ OrtStatusPtr ort_AddInitializer(OrtSessionOptions *opts, const char *name, const // Session profiling OrtStatusPtr ort_SessionEndProfiling(OrtSession *session, OrtAllocator *allocator, char **out); +// Version +const char *ort_GetVersionString(void); + // Session options getters (since 1.27) -int ort_api_version(void); OrtStatusPtr ort_GetMemPatternEnabled(const OrtSessionOptions *opts, int *out); OrtStatusPtr ort_GetSessionExecutionMode(const OrtSessionOptions *opts, ExecutionMode *out); @@ -183,10 +185,25 @@ OrtStatusPtr ort_GetValueType(const OrtValue *value, enum ONNXType *out); OrtStatusPtr ort_GetValueCount(const OrtValue *value, size_t *out); OrtStatusPtr ort_GetValue(const OrtValue *value, int index, OrtAllocator *allocator, OrtValue **out); +// Value creation (sequence/map) +OrtStatusPtr ort_CreateValue(const OrtValue *const *in, size_t num_values, + enum ONNXType value_type, OrtValue **out); + // Memory info OrtStatusPtr ort_CreateMemoryInfo(const char *name, enum OrtAllocatorType type, int id, enum OrtMemType mem_type, OrtMemoryInfo **out); +// Telemetry +OrtStatusPtr ort_EnableTelemetryEvents(const OrtEnv *env); +OrtStatusPtr ort_DisableTelemetryEvents(const OrtEnv *env); + +// Additional session options (continued) +OrtStatusPtr ort_SetOptimizedModelFilePath(OrtSessionOptions *opts, const char *path); +OrtStatusPtr ort_RegisterCustomOpsLibrary_V2(OrtSessionOptions *opts, const char *path); +OrtStatusPtr ort_HasSessionConfigEntry(const OrtSessionOptions *opts, const char *key, int *out); +OrtStatusPtr ort_GetSessionConfigEntry(const OrtSessionOptions *opts, const char *key, + char *value, size_t *size); + // Error handling OrtErrorCode ort_GetErrorCode(const OrtStatus *status); const char *ort_GetErrorMessage(const OrtStatus *status); diff --git a/go/onnxruntime/options.go b/go/onnxruntime/options.go index b09c144ae69ea..6902a23883a6c 100644 --- a/go/onnxruntime/options.go +++ b/go/onnxruntime/options.go @@ -163,6 +163,51 @@ func (o *SessionOptions) AddInitializer(name string, value *Tensor) error { return wrapErr("add initializer", checkStatus(C.ort_AddInitializer(o.handle, cName, value.value))) } +// SetOptimizedModelFilePath sets a path to save the optimized model to. +func (o *SessionOptions) SetOptimizedModelFilePath(path string) error { + cPath := C.CString(path) + defer C.free(unsafe.Pointer(cPath)) + return wrapErr("set optimized model path", + checkStatus(C.ort_SetOptimizedModelFilePath(o.handle, cPath))) +} + +// RegisterCustomOpsLibrary loads a shared library containing custom ops. +func (o *SessionOptions) RegisterCustomOpsLibrary(libraryPath string) error { + cPath := C.CString(libraryPath) + defer C.free(unsafe.Pointer(cPath)) + return wrapErr("register custom ops library", + checkStatus(C.ort_RegisterCustomOpsLibrary_V2(o.handle, cPath))) +} + +// HasSessionConfigEntry reports whether the config key exists. +func (o *SessionOptions) HasSessionConfigEntry(key string) (bool, error) { + cKey := C.CString(key) + defer C.free(unsafe.Pointer(cKey)) + var out C.int + if err := checkStatus(C.ort_HasSessionConfigEntry(o.handle, cKey, &out)); err != nil { + return false, wrapErr("has session config entry", err) + } + return out != 0, nil +} + +// GetSessionConfigEntry returns the value for the given config key. +func (o *SessionOptions) GetSessionConfigEntry(key string) (string, error) { + cKey := C.CString(key) + defer C.free(unsafe.Pointer(cKey)) + var size C.size_t + if err := checkStatus(C.ort_GetSessionConfigEntry(o.handle, cKey, nil, &size)); err != nil { + return "", wrapErr("get session config entry size", err) + } + if size == 0 { + return "", nil + } + buf := make([]C.char, size) + if err := checkStatus(C.ort_GetSessionConfigEntry(o.handle, cKey, &buf[0], &size)); err != nil { + return "", wrapErr("get session config entry", err) + } + return C.GoString(&buf[0]), nil +} + // Close releases the session options. It is idempotent. func (o *SessionOptions) Close() error { if o.handle != nil { diff --git a/go/onnxruntime/options_test.go b/go/onnxruntime/options_test.go index 422bfad486166..cb7eec8ab1147 100644 --- a/go/onnxruntime/options_test.go +++ b/go/onnxruntime/options_test.go @@ -180,6 +180,40 @@ func TestAppendExecutionProviderUnknown(t *testing.T) { } } +func TestSessionConfigEntry(t *testing.T) { + opts, err := NewSessionOptions() + if err != nil { + t.Fatal(err) + } + defer opts.Close() + + opts.AddConfigEntry("test.key", "test.value") + + has, err := opts.HasSessionConfigEntry("test.key") + if err != nil { + t.Fatal(err) + } + if !has { + t.Error("expected config entry to exist") + } + + val, err := opts.GetSessionConfigEntry("test.key") + if err != nil { + t.Fatal(err) + } + if val != "test.value" { + t.Errorf("expected 'test.value', got %q", val) + } + + has, err = opts.HasSessionConfigEntry("nonexistent") + if err != nil { + t.Fatal(err) + } + if has { + t.Error("expected config entry to not exist") + } +} + func TestTensorIsTensor(t *testing.T) { tensor, err := CreateTensor[float32]([]int64{2}, []float32{1, 2}) if err != nil { diff --git a/go/onnxruntime/ort.go b/go/onnxruntime/ort.go index 96e5bd462c1bc..1cef993f79997 100644 --- a/go/onnxruntime/ort.go +++ b/go/onnxruntime/ort.go @@ -84,6 +84,24 @@ func Init() error { return nil } +// GetVersion returns the ORT library version string (e.g. "1.27.0"). +// Must be called after Init. +func GetVersion() (string, error) { + mu.Lock() + defer mu.Unlock() + if !initialized { + return "", errNotInitialized + } + return C.GoString(C.ort_GetVersionString()), nil +} + +// APIVersion returns the ORT API version negotiated during Init. +func APIVersion() int { + mu.Lock() + defer mu.Unlock() + return apiVersion +} + // IsInitialized reports whether Init has completed successfully. func IsInitialized() bool { mu.Lock() @@ -118,6 +136,26 @@ func Shutdown() error { return nil } +// EnableTelemetry enables platform telemetry collection. +func EnableTelemetry() error { + mu.Lock() + defer mu.Unlock() + if !initialized { + return errNotInitialized + } + return wrapErr("enable telemetry", checkStatus(C.ort_EnableTelemetryEvents(env))) +} + +// DisableTelemetry disables platform telemetry collection. +func DisableTelemetry() error { + mu.Lock() + defer mu.Unlock() + if !initialized { + return errNotInitialized + } + return wrapErr("disable telemetry", checkStatus(C.ort_DisableTelemetryEvents(env))) +} + // AvailableProviders returns the execution providers available in the loaded // ORT library. func AvailableProviders() ([]string, error) { diff --git a/go/onnxruntime/ort_test.go b/go/onnxruntime/ort_test.go index d246ea6d90ce6..973418785a919 100644 --- a/go/onnxruntime/ort_test.go +++ b/go/onnxruntime/ort_test.go @@ -37,6 +37,26 @@ func TestIsInitialized(t *testing.T) { } } +func TestGetVersion(t *testing.T) { + v, err := GetVersion() + if err != nil { + t.Fatal(err) + } + if v == "" { + t.Fatal("expected non-empty version string") + } + t.Logf("ORT version: %s, API version: %d", v, APIVersion()) +} + +func TestTelemetry(t *testing.T) { + if err := DisableTelemetry(); err != nil { + t.Fatal(err) + } + if err := EnableTelemetry(); err != nil { + t.Fatal(err) + } +} + func TestAvailableProviders(t *testing.T) { providers, err := AvailableProviders() if err != nil { diff --git a/go/onnxruntime/session.go b/go/onnxruntime/session.go index 92181d3db211e..5312c296b9892 100644 --- a/go/onnxruntime/session.go +++ b/go/onnxruntime/session.go @@ -22,11 +22,12 @@ type IOInfo struct { // Session wraps an ORT inference session. It is safe for concurrent use: // multiple goroutines may call Run simultaneously. type Session struct { - mu sync.RWMutex - handle *C.OrtSession - inputs []IOInfo - outputs []IOInfo - closed bool + mu sync.RWMutex + handle *C.OrtSession + inputs []IOInfo + outputs []IOInfo + nameCache map[string]*C.char + closed bool } // NewSession creates a session from an ONNX model file. @@ -160,25 +161,33 @@ func (s *Session) runInner(ctx context.Context, inputs map[string]*Tensor, outpu nOutputs := len(outputNames) cInputNames := make([]*C.char, nInputs) cInputValues := make([]*C.OrtValue, nInputs) + var tempNames []*C.char i := 0 for name, tensor := range inputs { - cInputNames[i] = C.CString(name) + if cached, ok := s.nameCache[name]; ok { + cInputNames[i] = cached + } else { + cs := C.CString(name) + cInputNames[i] = cs + tempNames = append(tempNames, cs) + } cInputValues[i] = tensor.value i++ } - defer func() { - for _, cn := range cInputNames { - C.free(unsafe.Pointer(cn)) - } - }() cOutputNames := make([]*C.char, nOutputs) for i, name := range outputNames { - cOutputNames[i] = C.CString(name) + if cached, ok := s.nameCache[name]; ok { + cOutputNames[i] = cached + } else { + cs := C.CString(name) + cOutputNames[i] = cs + tempNames = append(tempNames, cs) + } } defer func() { - for _, cn := range cOutputNames { - C.free(unsafe.Pointer(cn)) + for _, cs := range tempNames { + C.free(unsafe.Pointer(cs)) } }() @@ -292,6 +301,10 @@ func (s *Session) Close() error { return nil } s.closed = true + for _, cs := range s.nameCache { + C.free(unsafe.Pointer(cs)) + } + s.nameCache = nil C.ort_ReleaseSession(s.handle) s.handle = nil sessionCount.Add(-1) @@ -322,11 +335,22 @@ func finalizeSession(handle *C.OrtSession) (*Session, error) { return nil, err } + cache := make(map[string]*C.char, len(inputs)+len(outputs)) + for _, info := range inputs { + cache[info.Name] = C.CString(info.Name) + } + for _, info := range outputs { + if _, ok := cache[info.Name]; !ok { + cache[info.Name] = C.CString(info.Name) + } + } + sessionCount.Add(1) return &Session{ - handle: handle, - inputs: inputs, - outputs: outputs, + handle: handle, + inputs: inputs, + outputs: outputs, + nameCache: cache, }, nil } diff --git a/go/onnxruntime/tensor.go b/go/onnxruntime/tensor.go index f72ea7387969c..81abc166c9809 100644 --- a/go/onnxruntime/tensor.go +++ b/go/onnxruntime/tensor.go @@ -191,19 +191,19 @@ func (t *Tensor) ValueType() (int, error) { // IsTensor reports whether this value is a tensor (as opposed to sequence/map). func (t *Tensor) IsTensor() bool { vt, err := t.ValueType() - return err == nil && vt == 1 // ONNX_TYPE_TENSOR + return err == nil && vt == int(C.ONNX_TYPE_TENSOR) } // IsSequence reports whether this value is a sequence. func (t *Tensor) IsSequence() bool { vt, err := t.ValueType() - return err == nil && vt == 3 // ONNX_TYPE_SEQUENCE + return err == nil && vt == int(C.ONNX_TYPE_SEQUENCE) } // IsMap reports whether this value is a map. func (t *Tensor) IsMap() bool { vt, err := t.ValueType() - return err == nil && vt == 4 // ONNX_TYPE_MAP + return err == nil && vt == int(C.ONNX_TYPE_MAP) } // SequenceLen returns the number of elements in a sequence value. @@ -235,6 +235,83 @@ func (t *Tensor) SequenceAt(index int) (*Tensor, error) { return wrapOutputTensor(value) } +// NewSequence creates a sequence value from the given tensors. +// All elements must be the same ONNX type. The returned value must be closed. +func NewSequence(elements []*Tensor) (*Tensor, error) { + if err := checkInit(); err != nil { + return nil, err + } + if len(elements) == 0 { + return nil, fmt.Errorf("ort: create sequence: at least one element required") + } + + values := make([]*C.OrtValue, len(elements)) + for i, e := range elements { + if e == nil { + return nil, fmt.Errorf("ort: create sequence: element %d is nil", i) + } + values[i] = e.value + } + + var out *C.OrtValue + if err := checkStatus(C.ort_CreateValue(&values[0], C.size_t(len(values)), + C.ONNX_TYPE_SEQUENCE, &out)); err != nil { + return nil, wrapErr("create sequence", err) + } + + return &Tensor{value: out, dtype: TensorElementDataTypeUndefined, owned: true}, nil +} + +// NewMap creates a map value from key and value tensors. +// Keys must be a 1-D tensor of int64 or string type. Values must be a 1-D tensor. +func NewMap(keys, values *Tensor) (*Tensor, error) { + if err := checkInit(); err != nil { + return nil, err + } + if keys == nil || values == nil { + return nil, fmt.Errorf("ort: create map: keys and values must not be nil") + } + + ins := []*C.OrtValue{keys.value, values.value} + var out *C.OrtValue + if err := checkStatus(C.ort_CreateValue(&ins[0], 2, + C.ONNX_TYPE_MAP, &out)); err != nil { + return nil, wrapErr("create map", err) + } + + return &Tensor{value: out, dtype: TensorElementDataTypeUndefined, owned: true}, nil +} + +// NewMapFromGoMap creates a map OrtValue from a Go map. +// Key type must be int64. Value type must be a TensorElement type. +func NewMapFromGoMap[K int64, V TensorElement](m map[K]V) (*Tensor, error) { + if err := checkInit(); err != nil { + return nil, err + } + + n := len(m) + keys := make([]K, 0, n) + vals := make([]V, 0, n) + for k, v := range m { + keys = append(keys, k) + vals = append(vals, v) + } + + keyTensor, err := CreateTensor[K]([]int64{int64(n)}, keys) + if err != nil { + return nil, wrapErr("create map keys", err) + } + defer func() { _ = keyTensor.Close() }() + + valTensor, err := CreateTensor[V]([]int64{int64(n)}, vals) + if err != nil { + return nil, wrapErr("create map values", err) + } + defer func() { _ = valTensor.Close() }() + + return NewMap(keyTensor, valTensor) +} + // Close releases the tensor's resources. It is idempotent. func (t *Tensor) Close() error { if t.closed { diff --git a/go/onnxruntime/tensor_test.go b/go/onnxruntime/tensor_test.go index fed2815c80d3b..f76eeb564ffd1 100644 --- a/go/onnxruntime/tensor_test.go +++ b/go/onnxruntime/tensor_test.go @@ -390,3 +390,71 @@ func TestNewTensorFromBytesFloat16(t *testing.T) { } } } + +func TestNewSequence(t *testing.T) { + t1, _ := CreateTensor[float32]([]int64{3}, []float32{1, 2, 3}) + t2, _ := CreateTensor[float32]([]int64{3}, []float32{4, 5, 6}) + defer t1.Close() + defer t2.Close() + + seq, err := NewSequence([]*Tensor{t1, t2}) + if err != nil { + t.Fatal(err) + } + defer seq.Close() + + if !seq.IsSequence() { + t.Error("expected IsSequence() = true") + } + + n, err := seq.SequenceLen() + if err != nil { + t.Fatal(err) + } + if n != 2 { + t.Errorf("expected 2 elements, got %d", n) + } + + elem, err := seq.SequenceAt(0) + if err != nil { + t.Fatal(err) + } + defer elem.Close() + + data, err := TensorData[float32](elem) + if err != nil { + t.Fatal(err) + } + if data[0] != 1 || data[1] != 2 || data[2] != 3 { + t.Errorf("expected [1,2,3], got %v", data) + } +} + +func TestNewMap(t *testing.T) { + keys, _ := CreateTensor[int64]([]int64{2}, []int64{10, 20}) + vals, _ := CreateTensor[float32]([]int64{2}, []float32{1.5, 2.5}) + defer keys.Close() + defer vals.Close() + + m, err := NewMap(keys, vals) + if err != nil { + t.Fatal(err) + } + defer m.Close() + + if !m.IsMap() { + t.Error("expected IsMap() = true") + } +} + +func TestNewMapFromGoMap(t *testing.T) { + m, err := NewMapFromGoMap[int64, float32](map[int64]float32{1: 10.5, 2: 20.5}) + if err != nil { + t.Fatal(err) + } + defer m.Close() + + if !m.IsMap() { + t.Error("expected IsMap() = true") + } +}