include/onnxruntime/onnxruntime_run_options_config_keys.h

0001 // Copyright (c) Microsoft Corporation. All rights reserved.
0002 // Licensed under the MIT License.
0003
0004 #pragma once
0005
0006 /*
0007  * This file defines RunOptions Config Keys and format of the Config Values.
0008  *
0009  * The Naming Convention for a RunOptions Config Key,
0010  * "[Area][.[SubArea1].[SubArea2]...].[Keyname]"
0011  * Such as "ep.cuda.use_arena"
0012  * The Config Key cannot be empty
0013  * The maximum length of the Config Key is 128
0014  *
0015  * The string format of a RunOptions Config Value is defined individually for each Config.
0016  * The maximum length of the Config Value is 1024
0017  */
0018
0019 // Key for enabling shrinkages of user listed device memory arenas.
0020 // Expects a list of semi-colon separated key value pairs separated by colon in the following format:
0021 // "device_0:device_id_0;device_1:device_id_1"
0022 // No white-spaces allowed in the provided list string.
0023 // Currently, the only supported devices are : "cpu", "gpu" (case sensitive).
0024 // If "cpu" is included in the list, DisableCpuMemArena() API must not be called (i.e.) arena for cpu should be enabled.
0025 // Example usage: "cpu:0;gpu:0" (or) "gpu:0"
0026 // By default, the value for this key is empty (i.e.) no memory arenas are shrunk
0027 static const char* const kOrtRunOptionsConfigEnableMemoryArenaShrinkage = "memory.enable_memory_arena_shrinkage";
0028
0029 // Set to '1' to not synchronize execution providers with CPU at the end of session run.
0030 // Per default it will be set to '0'
0031 // Taking CUDA EP as an example, it omit triggering cudaStreamSynchronize on the compute stream.
0032 static const char* const kOrtRunOptionsConfigDisableSynchronizeExecutionProviders = "disable_synchronize_execution_providers";
0033
0034 // Set HTP performance mode for QNN HTP backend before session run.
0035 // options for HTP performance mode: "burst", "balanced", "default", "high_performance",
0036 // "high_power_saver", "low_balanced", "extreme_power_saver", "low_power_saver", "power_saver",
0037 // "sustained_high_performance". Default to "default".
0038 static const char* const kOrtRunOptionsConfigQnnPerfMode = "qnn.htp_perf_mode";
0039
0040 // Set HTP performance mode for QNN HTP backend post session run.
0041 static const char* const kOrtRunOptionsConfigQnnPerfModePostRun = "qnn.htp_perf_mode_post_run";
0042
0043 // Set RPC control latency for QNN HTP backend
0044 static const char* const kOrtRunOptionsConfigQnnRpcControlLatency = "qnn.rpc_control_latency";
0045
0046 // Set graph annotation id for CUDA EP. Use with enable_cuda_graph=true.
0047 // The value should be an integer. If the value is not set, the default value is 0 and
0048 // ORT session only captures one cuda graph before another capture is requested.
0049 // If the value is set to -1, cuda graph capture/replay is disabled in that run.
0050 // User are not expected to set the value to 0 as it is reserved for internal use.
0051 static const char* const kOrtRunOptionsConfigCudaGraphAnnotation = "gpu_graph_id";