From c6d8aecb48124bb45e6de18b47254c375d9024e8 Mon Sep 17 00:00:00 2001 From: Shubham Singh Date: Mon, 13 Jul 2026 07:51:29 +0530 Subject: [PATCH 1/9] Expose adaptive context APIs for Swift LLM # Conflicts: # sdk/runanywhere-swift/Sources/RunAnywhere/CRACommons/include/rac_llm_component.h # sdk/runanywhere-swift/Sources/RunAnywhere/CRACommons/include/rac_llm_service.h --- .../exports/RACommons.exports | 8 + .../rac/features/llm/rac_llm_component.h | 58 +++++ .../rac/features/llm/rac_llm_service.h | 34 ++- .../src/features/llm/llm_module.cpp | 219 ++++++++++++++++++ .../Bridge/Extensions/CppBridge+LLM.swift | 83 ++++++- .../LLM/RunAnywhere+TextGeneration.swift | 39 ++++ 6 files changed, 439 insertions(+), 2 deletions(-) diff --git a/sdk/runanywhere-commons/exports/RACommons.exports b/sdk/runanywhere-commons/exports/RACommons.exports index 764cbbcacc..55fb08a21b 100644 --- a/sdk/runanywhere-commons/exports/RACommons.exports +++ b/sdk/runanywhere-commons/exports/RACommons.exports @@ -534,16 +534,22 @@ _rac_platform_tts_set_callbacks # Kotlin bridges during the proto-byte migration (CPP_PROTO_OWNERSHIP.md ยง2.1). _rac_llm_cancel _rac_llm_component_cancel +_rac_llm_component_append_context _rac_llm_component_cleanup +_rac_llm_component_clear_context _rac_llm_component_create _rac_llm_component_destroy _rac_llm_component_generate +_rac_llm_component_generate_from_context _rac_llm_component_generate_stream _rac_llm_component_get_model_id +_rac_llm_component_inject_system_prompt _rac_llm_component_is_loaded _rac_llm_component_load_model _rac_llm_component_supports_streaming _rac_llm_component_unload +_rac_llm_append_context_lifecycle +_rac_llm_clear_context_lifecycle _rac_llm_create _rac_llm_generate _rac_llm_generate_stream @@ -949,7 +955,9 @@ _rac_llm_component_load_lora _rac_llm_component_remove_lora _rac_llm_destroy _rac_llm_generate_from_context +_rac_llm_generate_from_context_proto _rac_llm_get_info +_rac_llm_inject_system_prompt_lifecycle _rac_llm_initialize _rac_llm_inject_system_prompt _rac_llm_platform_create diff --git a/sdk/runanywhere-commons/include/rac/features/llm/rac_llm_component.h b/sdk/runanywhere-commons/include/rac/features/llm/rac_llm_component.h index aa01859964..ce3caa6428 100644 --- a/sdk/runanywhere-commons/include/rac/features/llm/rac_llm_component.h +++ b/sdk/runanywhere-commons/include/rac/features/llm/rac_llm_component.h @@ -226,6 +226,64 @@ RAC_API rac_lifecycle_state_t rac_llm_component_get_state(rac_handle_t handle); RAC_API rac_result_t rac_llm_component_get_metrics(rac_handle_t handle, rac_lifecycle_metrics_t* out_metrics); +// ============================================================================= +// ADAPTIVE CONTEXT API +// ============================================================================= + +/** + * @brief Inject a system prompt into the loaded model's adaptive context + * + * Clears existing KV cache, then seeds it with the given prompt. + * Optional - returns RAC_ERROR_NOT_SUPPORTED if backend doesn't support it. + * + * @param handle Component handle + * @param prompt System prompt text + * @return RAC_SUCCESS or error code + */ +RAC_API rac_result_t rac_llm_component_inject_system_prompt(rac_handle_t handle, + const char* prompt); + +/** + * @brief Append text to the loaded model's adaptive context + * + * Does not clear existing KV state - accumulates context incrementally. + * Optional - returns RAC_ERROR_NOT_SUPPORTED if backend doesn't support it. + * + * @param handle Component handle + * @param text Text to append + * @return RAC_SUCCESS or error code + */ +RAC_API rac_result_t rac_llm_component_append_context(rac_handle_t handle, const char* text); + +/** + * @brief Generate a response from accumulated adaptive context + * + * Unlike rac_llm_component_generate(), this does not clear the KV cache first. + * Use after inject_system_prompt + append_context to generate from accumulated state. + * Optional - returns RAC_ERROR_NOT_SUPPORTED if backend doesn't support it. + * + * @param handle Component handle + * @param query Query/suffix text to append before generation + * @param options Generation options (can be NULL for defaults) + * @param out_result Output: Generation result (caller must free with rac_llm_result_free) + * @return RAC_SUCCESS or error code + */ +RAC_API rac_result_t rac_llm_component_generate_from_context(rac_handle_t handle, + const char* query, + const rac_llm_options_t* options, + rac_llm_result_t* out_result); + +/** + * @brief Clear all adaptive context state + * + * Resets the LLM's context for a fresh adaptive query cycle. + * Optional - returns RAC_ERROR_NOT_SUPPORTED if backend doesn't support it. + * + * @param handle Component handle + * @return RAC_SUCCESS or error code + */ +RAC_API rac_result_t rac_llm_component_clear_context(rac_handle_t handle); + // ============================================================================= // LORA ADAPTER API // ============================================================================= diff --git a/sdk/runanywhere-commons/include/rac/features/llm/rac_llm_service.h b/sdk/runanywhere-commons/include/rac/features/llm/rac_llm_service.h index ed718edb74..ae016d7b17 100644 --- a/sdk/runanywhere-commons/include/rac/features/llm/rac_llm_service.h +++ b/sdk/runanywhere-commons/include/rac/features/llm/rac_llm_service.h @@ -10,7 +10,8 @@ * - rac_llm_service_ops_t and rac_llm_service_t: `internal`. Engine * dispatch contract. * - Proto-byte APIs (rac_llm_generate_proto, - * rac_llm_generate_stream_proto, rac_llm_cancel_proto): + * rac_llm_generate_stream_proto, rac_llm_cancel_proto, plus + * lifecycle adaptive-context helpers): * `SDK-facing default` over runanywhere.v1.LLMGenerateRequest / * LLMGenerationResult / LLMStreamEvent / SDKEvent bytes. * - Struct APIs (rac_llm_create, initialize, generate, @@ -277,6 +278,37 @@ RAC_API rac_result_t rac_llm_generate_stream_proto(const uint8_t* request_proto_ */ RAC_API rac_result_t rac_llm_cancel_proto(rac_proto_buffer_t* out_event); +/** + * @brief Inject a system prompt into the lifecycle-owned LLM's adaptive context + * + * Uses the LLM model loaded through rac_model_lifecycle_load_proto(). + */ +RAC_API rac_result_t rac_llm_inject_system_prompt_lifecycle(const char* prompt); + +/** + * @brief Append text to the lifecycle-owned LLM's adaptive context + * + * Uses the LLM model loaded through rac_model_lifecycle_load_proto(). + */ +RAC_API rac_result_t rac_llm_append_context_lifecycle(const char* text); + +/** + * @brief Generate from the lifecycle-owned LLM's accumulated adaptive context + * + * Accepts serialized runanywhere.v1.LLMGenerateRequest bytes and returns a + * serialized runanywhere.v1.LLMGenerationResult in out_result. + */ +RAC_API rac_result_t rac_llm_generate_from_context_proto(const uint8_t* request_proto_bytes, + size_t request_proto_size, + rac_proto_buffer_t* out_result); + +/** + * @brief Clear the lifecycle-owned LLM's adaptive context + * + * Uses the LLM model loaded through rac_model_lifecycle_load_proto(). + */ +RAC_API rac_result_t rac_llm_clear_context_lifecycle(void); + // ============================================================================= // ADAPTIVE CONTEXT API - For RAG and similar pipelines // ============================================================================= diff --git a/sdk/runanywhere-commons/src/features/llm/llm_module.cpp b/sdk/runanywhere-commons/src/features/llm/llm_module.cpp index 5c5a447a97..05b4060ca9 100644 --- a/sdk/runanywhere-commons/src/features/llm/llm_module.cpp +++ b/sdk/runanywhere-commons/src/features/llm/llm_module.cpp @@ -1076,6 +1076,82 @@ extern "C" rac_result_t rac_llm_component_cancel(rac_handle_t handle) { return RAC_SUCCESS; } +// ============================================================================= +// ADAPTIVE CONTEXT API +// ============================================================================= + +extern "C" rac_result_t rac_llm_component_inject_system_prompt(rac_handle_t handle, + const char* prompt) { + if (!handle) + return RAC_ERROR_INVALID_HANDLE; + if (!prompt) + return RAC_ERROR_INVALID_ARGUMENT; + + auto* component = reinterpret_cast(handle); + std::lock_guard lock(component->mtx); + + rac_handle_t service = rac_lifecycle_get_service(component->lifecycle); + if (!service) { + RAC_LOG_ERROR("LLM.Component", "Cannot inject system prompt: no model loaded"); + return RAC_ERROR_COMPONENT_NOT_READY; + } + + return rac_llm_inject_system_prompt(service, prompt); +} + +extern "C" rac_result_t rac_llm_component_append_context(rac_handle_t handle, const char* text) { + if (!handle) + return RAC_ERROR_INVALID_HANDLE; + if (!text) + return RAC_ERROR_INVALID_ARGUMENT; + + auto* component = reinterpret_cast(handle); + std::lock_guard lock(component->mtx); + + rac_handle_t service = rac_lifecycle_get_service(component->lifecycle); + if (!service) { + RAC_LOG_ERROR("LLM.Component", "Cannot append context: no model loaded"); + return RAC_ERROR_COMPONENT_NOT_READY; + } + + return rac_llm_append_context(service, text); +} + +extern "C" rac_result_t rac_llm_component_generate_from_context( + rac_handle_t handle, const char* query, const rac_llm_options_t* options, + rac_llm_result_t* out_result) { + if (!handle) + return RAC_ERROR_INVALID_HANDLE; + if (!query || !out_result) + return RAC_ERROR_INVALID_ARGUMENT; + + auto* component = reinterpret_cast(handle); + std::lock_guard lock(component->mtx); + + rac_handle_t service = rac_lifecycle_get_service(component->lifecycle); + if (!service) { + RAC_LOG_ERROR("LLM.Component", "Cannot generate from context: no model loaded"); + return RAC_ERROR_COMPONENT_NOT_READY; + } + + return rac_llm_generate_from_context(service, query, options, out_result); +} + +extern "C" rac_result_t rac_llm_component_clear_context(rac_handle_t handle) { + if (!handle) + return RAC_ERROR_INVALID_HANDLE; + + auto* component = reinterpret_cast(handle); + std::lock_guard lock(component->mtx); + + rac_handle_t service = rac_lifecycle_get_service(component->lifecycle); + if (!service) { + return RAC_SUCCESS; + } + + return rac_llm_clear_context(service); +} + // ============================================================================= // LORA ADAPTER API // ============================================================================= @@ -2272,4 +2348,147 @@ rac_result_t rac_llm_cancel_proto(rac_proto_buffer_t* out_event) { #endif } +rac_result_t rac_llm_inject_system_prompt_lifecycle(const char* prompt) { + if (!prompt) { + return RAC_ERROR_INVALID_ARGUMENT; + } +#if !defined(RAC_HAVE_PROTOBUF) + return RAC_ERROR_FEATURE_NOT_AVAILABLE; +#else + rac::llm::LifecycleLlmRef ref; + rac_result_t rc = rac::llm::acquire_lifecycle_llm(&ref); + if (rc != RAC_SUCCESS) { + return rc; + } + + rc = (ref.ops && ref.ops->inject_system_prompt) + ? ref.ops->inject_system_prompt(ref.impl, prompt) + : RAC_ERROR_NOT_SUPPORTED; + rac::llm::release_lifecycle_llm(&ref); + return rc; +#endif +} + +rac_result_t rac_llm_append_context_lifecycle(const char* text) { + if (!text) { + return RAC_ERROR_INVALID_ARGUMENT; + } +#if !defined(RAC_HAVE_PROTOBUF) + return RAC_ERROR_FEATURE_NOT_AVAILABLE; +#else + rac::llm::LifecycleLlmRef ref; + rac_result_t rc = rac::llm::acquire_lifecycle_llm(&ref); + if (rc != RAC_SUCCESS) { + return rc; + } + + rc = (ref.ops && ref.ops->append_context) ? ref.ops->append_context(ref.impl, text) + : RAC_ERROR_NOT_SUPPORTED; + rac::llm::release_lifecycle_llm(&ref); + return rc; +#endif +} + +rac_result_t rac_llm_generate_from_context_proto(const uint8_t* request_proto_bytes, + size_t request_proto_size, + rac_proto_buffer_t* out_result) { + if (!out_result) { + return RAC_ERROR_NULL_POINTER; + } +#if !defined(RAC_HAVE_PROTOBUF) + (void)request_proto_bytes; + (void)request_proto_size; + return feature_unavailable(out_result); +#else + if (!valid_bytes(request_proto_bytes, request_proto_size)) { + return parse_error(out_result, "LLMGenerateRequest bytes are empty or too large"); + } + + LLMGenerateRequest request; + if (!request.ParseFromArray(parse_data(request_proto_bytes, request_proto_size), + static_cast(request_proto_size))) { + return parse_error(out_result, "failed to parse LLMGenerateRequest"); + } + + rac::llm::LifecycleLlmRef ref; + rac_result_t rc = rac::llm::acquire_lifecycle_llm(&ref); + if (rc != RAC_SUCCESS) { + return rac_proto_buffer_set_error(out_result, rc, "no lifecycle LLM model loaded"); + } + + rac::llm::clear_lifecycle_llm_cancel(&ref); + publish_generation_event(runanywhere::v1::GENERATION_EVENT_KIND_STARTED, + request.prompt().c_str(), nullptr, nullptr, nullptr, ref.model_id, 0, + 0); + + const std::string system_prompt = system_prompt_from_request(request); + std::vector stop_storage; + std::vector stop_ptrs; + std::string grammar_storage; + rac_llm_options_t options = + options_from_request(request, system_prompt, stop_storage, stop_ptrs, grammar_storage); + options.streaming_enabled = RAC_FALSE; + + rac_llm_result_t raw{}; + const std::string effective_query = + rac::llm::apply_no_think_directive(request.prompt(), options.disable_thinking); + const int64_t started = now_ms(); + rc = (ref.ops && ref.ops->generate_from_context) + ? ref.ops->generate_from_context(ref.impl, effective_query.c_str(), &options, &raw) + : RAC_ERROR_NOT_SUPPORTED; + const int64_t elapsed = now_ms() - started; + + if (rc != RAC_SUCCESS) { + publish_generation_event(runanywhere::v1::GENERATION_EVENT_KIND_FAILED, + request.prompt().c_str(), nullptr, nullptr, rac_error_message(rc), + ref.model_id, 0, elapsed); + rac::llm::release_lifecycle_llm(&ref); + return rac_proto_buffer_set_error(out_result, rc, rac_error_message(rc)); + } + + const char* response = nullptr; + size_t response_len = 0; + const char* thinking = nullptr; + size_t thinking_len = 0; + const char* raw_text = raw.text ? raw.text : ""; + (void)rac_llm_extract_thinking(raw_text, &response, &response_len, &thinking, &thinking_len); + + int32_t thinking_tokens = 0; + int32_t response_tokens = raw.completion_tokens; + (void)rac_llm_split_thinking_tokens(raw.completion_tokens, response, thinking, &thinking_tokens, + &response_tokens); + + LLMGenerationResult result; + set_result_from_raw(ref, raw, response, response_len, thinking, thinking_len, thinking_tokens, + response_tokens, options.max_tokens, &result); + set_structured_output_if_present(response, &result); + + publish_generation_event( + runanywhere::v1::GENERATION_EVENT_KIND_COMPLETED, request.prompt().c_str(), nullptr, + response, nullptr, ref.model_id, raw.completion_tokens, + raw.total_time_ms > 0 ? raw.total_time_ms : elapsed, raw.prompt_tokens); + + rac_llm_result_free(&raw); + rac::llm::release_lifecycle_llm(&ref); + return copy_proto(result, out_result); +#endif +} + +rac_result_t rac_llm_clear_context_lifecycle(void) { +#if !defined(RAC_HAVE_PROTOBUF) + return RAC_ERROR_FEATURE_NOT_AVAILABLE; +#else + rac::llm::LifecycleLlmRef ref; + rac_result_t rc = rac::llm::acquire_lifecycle_llm(&ref); + if (rc != RAC_SUCCESS) { + return rc; + } + + rc = (ref.ops && ref.ops->clear_context) ? ref.ops->clear_context(ref.impl) + : RAC_ERROR_NOT_SUPPORTED; + rac::llm::release_lifecycle_llm(&ref); + return rc; +#endif +} + } // extern "C" diff --git a/sdk/runanywhere-swift/Sources/RunAnywhere/Foundation/Bridge/Extensions/CppBridge+LLM.swift b/sdk/runanywhere-swift/Sources/RunAnywhere/Foundation/Bridge/Extensions/CppBridge+LLM.swift index 61912a5d23..af586d4e0e 100644 --- a/sdk/runanywhere-swift/Sources/RunAnywhere/Foundation/Bridge/Extensions/CppBridge+LLM.swift +++ b/sdk/runanywhere-swift/Sources/RunAnywhere/Foundation/Bridge/Extensions/CppBridge+LLM.swift @@ -6,11 +6,31 @@ // // All generic scaffolding (handle creation, isLoaded, loadModel, // unload, destroy) lives in `CppBridge.ComponentActor`; this file -// only adds the LLM-specific `cancel()` op on top. +// adds LLM-specific operations such as cancel and adaptive context on top. // import CRACommons +private enum LLMAdaptiveContextABI { + typealias StringCall = @convention(c) (UnsafePointer?) -> rac_result_t + typealias VoidCall = @convention(c) () -> rac_result_t + typealias Generate = @convention(c) ( + UnsafePointer?, + Int, + UnsafeMutablePointer? + ) -> rac_result_t + + static let injectSystemPromptName = "rac_llm_inject_system_prompt_lifecycle" + static let appendContextName = "rac_llm_append_context_lifecycle" + static let generateFromContextName = "rac_llm_generate_from_context_proto" + static let clearContextName = "rac_llm_clear_context_lifecycle" + + static let injectSystemPrompt = NativeProtoABI.load(injectSystemPromptName, as: StringCall.self) + static let appendContext = NativeProtoABI.load(appendContextName, as: StringCall.self) + static let generateFromContext = NativeProtoABI.load(generateFromContextName, as: Generate.self) + static let clearContext = NativeProtoABI.load(clearContextName, as: VoidCall.self) +} + // MARK: - LLM Component Bridge extension CppBridge { @@ -57,11 +77,72 @@ extension CppBridge { rac_llm_component_cancel(handle.rawValue) } + // MARK: - Adaptive Context + + /// Seed the lifecycle-owned LLM context with a reusable system prompt. + public func injectSystemPrompt(_ prompt: String) async throws { + let symbol = try NativeProtoABI.require( + LLMAdaptiveContextABI.injectSystemPrompt, + named: LLMAdaptiveContextABI.injectSystemPromptName + ) + let status = prompt.withCString { + symbol($0) + } + try Self.throwIfFailed(status, operation: "inject system prompt") + } + + /// Append user or retrieval text to the lifecycle-owned LLM context. + public func appendContext(_ text: String) async throws { + let symbol = try NativeProtoABI.require( + LLMAdaptiveContextABI.appendContext, + named: LLMAdaptiveContextABI.appendContextName + ) + let status = text.withCString { + symbol($0) + } + try Self.throwIfFailed(status, operation: "append context") + } + + /// Generate without clearing the accumulated lifecycle-owned LLM context. + public func generateFromContext( + query: String, + options: RALLMGenerationOptions? = nil + ) async throws -> RALLMGenerationResult { + var request = (options ?? .defaults()).toRALLMGenerateRequest(prompt: query) + request.streamingEnabled = false + return try NativeProtoABI.invoke( + request, + symbol: LLMAdaptiveContextABI.generateFromContext, + symbolName: LLMAdaptiveContextABI.generateFromContextName, + responseType: RALLMGenerationResult.self + ) + } + + /// Clear accumulated adaptive context on the lifecycle-owned LLM. + public func clearContext() async throws { + let symbol = try NativeProtoABI.require( + LLMAdaptiveContextABI.clearContext, + named: LLMAdaptiveContextABI.clearContextName + ) + let status = symbol() + try Self.throwIfFailed(status, operation: "clear context") + } + // MARK: - Cleanup /// Destroy the component public func destroy() async { await inner.destroy() } + + private static func throwIfFailed(_ status: rac_result_t, operation: String) throws { + guard status == RAC_SUCCESS else { + throw SDKException( + code: .processingFailed, + message: "LLM adaptive context \(operation) failed: \(status)", + category: .component + ) + } + } } } diff --git a/sdk/runanywhere-swift/Sources/RunAnywhere/Public/Extensions/LLM/RunAnywhere+TextGeneration.swift b/sdk/runanywhere-swift/Sources/RunAnywhere/Public/Extensions/LLM/RunAnywhere+TextGeneration.swift index 63977822a0..b43d77d1a7 100644 --- a/sdk/runanywhere-swift/Sources/RunAnywhere/Public/Extensions/LLM/RunAnywhere+TextGeneration.swift +++ b/sdk/runanywhere-swift/Sources/RunAnywhere/Public/Extensions/LLM/RunAnywhere+TextGeneration.swift @@ -41,6 +41,45 @@ public extension RunAnywhere { return try await generateStream(request) } + /// Seed the loaded on-device model's adaptive context with a reusable system prompt. + static func injectSystemPrompt(_ prompt: String) async throws { + guard isInitialized else { + throw SDKException(code: .notInitialized, message: "SDK not initialized", category: .internal) + } + try await ensureServicesReady() + try await CppBridge.LLM.shared.injectSystemPrompt(prompt) + } + + /// Append text to the loaded on-device model's adaptive context. + static func appendContext(_ text: String) async throws { + guard isInitialized else { + throw SDKException(code: .notInitialized, message: "SDK not initialized", category: .internal) + } + try await ensureServicesReady() + try await CppBridge.LLM.shared.appendContext(text) + } + + /// Generate from the accumulated adaptive context without clearing the KV cache first. + static func generateFromContext( + query: String, + options: RALLMGenerationOptions? = nil + ) async throws -> RALLMGenerationResult { + guard isInitialized else { + throw SDKException(code: .notInitialized, message: "SDK not initialized", category: .internal) + } + try await ensureServicesReady() + return try await CppBridge.LLM.shared.generateFromContext(query: query, options: options) + } + + /// Clear the loaded on-device model's adaptive context. + static func clearContext() async throws { + guard isInitialized else { + throw SDKException(code: .notInitialized, message: "SDK not initialized", category: .internal) + } + try await ensureServicesReady() + try await CppBridge.LLM.shared.clearContext() + } + /// Generate text through the generated-proto C++ LLM service ABI. static func generate(_ request: RALLMGenerateRequest) async throws -> RALLMGenerationResult { guard isInitialized else { From 25099a236ea0adb7136d52673069410d9a6f94d9 Mon Sep 17 00:00:00 2001 From: Shubham Singh Date: Thu, 2 Jul 2026 15:48:07 +0530 Subject: [PATCH 2/9] fix(llm): preserve adaptive context defaults and errors --- sdk/runanywhere-commons/src/features/llm/llm_module.cpp | 3 ++- .../Foundation/Bridge/Extensions/CppBridge+LLM.swift | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/sdk/runanywhere-commons/src/features/llm/llm_module.cpp b/sdk/runanywhere-commons/src/features/llm/llm_module.cpp index 05b4060ca9..cdb502bd72 100644 --- a/sdk/runanywhere-commons/src/features/llm/llm_module.cpp +++ b/sdk/runanywhere-commons/src/features/llm/llm_module.cpp @@ -1134,7 +1134,8 @@ extern "C" rac_result_t rac_llm_component_generate_from_context( return RAC_ERROR_COMPONENT_NOT_READY; } - return rac_llm_generate_from_context(service, query, options, out_result); + const rac_llm_options_t* effective_options = options ? options : &component->default_options; + return rac_llm_generate_from_context(service, query, effective_options, out_result); } extern "C" rac_result_t rac_llm_component_clear_context(rac_handle_t handle) { diff --git a/sdk/runanywhere-swift/Sources/RunAnywhere/Foundation/Bridge/Extensions/CppBridge+LLM.swift b/sdk/runanywhere-swift/Sources/RunAnywhere/Foundation/Bridge/Extensions/CppBridge+LLM.swift index af586d4e0e..4b347b18c6 100644 --- a/sdk/runanywhere-swift/Sources/RunAnywhere/Foundation/Bridge/Extensions/CppBridge+LLM.swift +++ b/sdk/runanywhere-swift/Sources/RunAnywhere/Foundation/Bridge/Extensions/CppBridge+LLM.swift @@ -137,9 +137,10 @@ extension CppBridge { private static func throwIfFailed(_ status: rac_result_t, operation: String) throws { guard status == RAC_SUCCESS else { + let nativeMessage = String(cString: rac_error_message(status)) throw SDKException( code: .processingFailed, - message: "LLM adaptive context \(operation) failed: \(status)", + message: "LLM adaptive context \(operation) failed: \(nativeMessage) (\(status))", category: .component ) } From 61651cb4e0b3d40e61e1b43dceff985f651635ef Mon Sep 17 00:00:00 2001 From: Shubham Singh Date: Fri, 3 Jul 2026 05:17:50 +0530 Subject: [PATCH 3/9] docs(llm): document adaptive context entry points --- sdk/runanywhere-commons/src/features/llm/llm_module.cpp | 8 ++++++++ .../Foundation/Bridge/Extensions/CppBridge+LLM.swift | 1 + 2 files changed, 9 insertions(+) diff --git a/sdk/runanywhere-commons/src/features/llm/llm_module.cpp b/sdk/runanywhere-commons/src/features/llm/llm_module.cpp index cdb502bd72..369338bc19 100644 --- a/sdk/runanywhere-commons/src/features/llm/llm_module.cpp +++ b/sdk/runanywhere-commons/src/features/llm/llm_module.cpp @@ -1080,6 +1080,7 @@ extern "C" rac_result_t rac_llm_component_cancel(rac_handle_t handle) { // ADAPTIVE CONTEXT API // ============================================================================= +/** Seed a component's adaptive KV context with a reusable system prompt. */ extern "C" rac_result_t rac_llm_component_inject_system_prompt(rac_handle_t handle, const char* prompt) { if (!handle) @@ -1099,6 +1100,7 @@ extern "C" rac_result_t rac_llm_component_inject_system_prompt(rac_handle_t hand return rac_llm_inject_system_prompt(service, prompt); } +/** Append text to a component's existing adaptive KV context. */ extern "C" rac_result_t rac_llm_component_append_context(rac_handle_t handle, const char* text) { if (!handle) return RAC_ERROR_INVALID_HANDLE; @@ -1117,6 +1119,7 @@ extern "C" rac_result_t rac_llm_component_append_context(rac_handle_t handle, co return rac_llm_append_context(service, text); } +/** Generate from a component's accumulated adaptive context. */ extern "C" rac_result_t rac_llm_component_generate_from_context( rac_handle_t handle, const char* query, const rac_llm_options_t* options, rac_llm_result_t* out_result) { @@ -1138,6 +1141,7 @@ extern "C" rac_result_t rac_llm_component_generate_from_context( return rac_llm_generate_from_context(service, query, effective_options, out_result); } +/** Clear all adaptive context retained by a component. */ extern "C" rac_result_t rac_llm_component_clear_context(rac_handle_t handle) { if (!handle) return RAC_ERROR_INVALID_HANDLE; @@ -2349,6 +2353,7 @@ rac_result_t rac_llm_cancel_proto(rac_proto_buffer_t* out_event) { #endif } +/** Seed the lifecycle-owned LLM's adaptive context with a system prompt. */ rac_result_t rac_llm_inject_system_prompt_lifecycle(const char* prompt) { if (!prompt) { return RAC_ERROR_INVALID_ARGUMENT; @@ -2370,6 +2375,7 @@ rac_result_t rac_llm_inject_system_prompt_lifecycle(const char* prompt) { #endif } +/** Append text to the lifecycle-owned LLM's adaptive context. */ rac_result_t rac_llm_append_context_lifecycle(const char* text) { if (!text) { return RAC_ERROR_INVALID_ARGUMENT; @@ -2390,6 +2396,7 @@ rac_result_t rac_llm_append_context_lifecycle(const char* text) { #endif } +/** Generate a protobuf result from the lifecycle-owned LLM's adaptive context. */ rac_result_t rac_llm_generate_from_context_proto(const uint8_t* request_proto_bytes, size_t request_proto_size, rac_proto_buffer_t* out_result) { @@ -2475,6 +2482,7 @@ rac_result_t rac_llm_generate_from_context_proto(const uint8_t* request_proto_by #endif } +/** Clear adaptive context retained by the lifecycle-owned LLM. */ rac_result_t rac_llm_clear_context_lifecycle(void) { #if !defined(RAC_HAVE_PROTOBUF) return RAC_ERROR_FEATURE_NOT_AVAILABLE; diff --git a/sdk/runanywhere-swift/Sources/RunAnywhere/Foundation/Bridge/Extensions/CppBridge+LLM.swift b/sdk/runanywhere-swift/Sources/RunAnywhere/Foundation/Bridge/Extensions/CppBridge+LLM.swift index 4b347b18c6..2062db862f 100644 --- a/sdk/runanywhere-swift/Sources/RunAnywhere/Foundation/Bridge/Extensions/CppBridge+LLM.swift +++ b/sdk/runanywhere-swift/Sources/RunAnywhere/Foundation/Bridge/Extensions/CppBridge+LLM.swift @@ -135,6 +135,7 @@ extension CppBridge { await inner.destroy() } + /// Convert a failed native adaptive-context status into an SDK error with backend detail. private static func throwIfFailed(_ status: rac_result_t, operation: String) throws { guard status == RAC_SUCCESS else { let nativeMessage = String(cString: rac_error_message(status)) From 8cc1ffc50677e86afb0f56777629315583623c65 Mon Sep 17 00:00:00 2001 From: Shubham Singh Date: Sun, 5 Jul 2026 14:48:47 +0530 Subject: [PATCH 4/9] refactor(llm): DRY lifecycle wrappers with shared template --- .../src/features/llm/llm_module.cpp | 52 ++++++------------- 1 file changed, 17 insertions(+), 35 deletions(-) diff --git a/sdk/runanywhere-commons/src/features/llm/llm_module.cpp b/sdk/runanywhere-commons/src/features/llm/llm_module.cpp index 369338bc19..c61f582b17 100644 --- a/sdk/runanywhere-commons/src/features/llm/llm_module.cpp +++ b/sdk/runanywhere-commons/src/features/llm/llm_module.cpp @@ -2353,12 +2353,11 @@ rac_result_t rac_llm_cancel_proto(rac_proto_buffer_t* out_event) { #endif } -/** Seed the lifecycle-owned LLM's adaptive context with a system prompt. */ -rac_result_t rac_llm_inject_system_prompt_lifecycle(const char* prompt) { - if (!prompt) { - return RAC_ERROR_INVALID_ARGUMENT; - } +namespace { +template +rac_result_t call_lifecycle_op(Op op, Args&&... args) { #if !defined(RAC_HAVE_PROTOBUF) + (void)op; return RAC_ERROR_FEATURE_NOT_AVAILABLE; #else rac::llm::LifecycleLlmRef ref; @@ -2367,33 +2366,29 @@ rac_result_t rac_llm_inject_system_prompt_lifecycle(const char* prompt) { return rc; } - rc = (ref.ops && ref.ops->inject_system_prompt) - ? ref.ops->inject_system_prompt(ref.impl, prompt) + rc = (ref.ops && (ref.ops->*op)) + ? (ref.ops->*op)(ref.impl, std::forward(args)...) : RAC_ERROR_NOT_SUPPORTED; rac::llm::release_lifecycle_llm(&ref); return rc; #endif } +} // namespace + +/** Seed the lifecycle-owned LLM's adaptive context with a system prompt. */ +rac_result_t rac_llm_inject_system_prompt_lifecycle(const char* prompt) { + if (!prompt) { + return RAC_ERROR_INVALID_ARGUMENT; + } + return call_lifecycle_op(&rac_llm_service_ops_t::inject_system_prompt, prompt); +} /** Append text to the lifecycle-owned LLM's adaptive context. */ rac_result_t rac_llm_append_context_lifecycle(const char* text) { if (!text) { return RAC_ERROR_INVALID_ARGUMENT; } -#if !defined(RAC_HAVE_PROTOBUF) - return RAC_ERROR_FEATURE_NOT_AVAILABLE; -#else - rac::llm::LifecycleLlmRef ref; - rac_result_t rc = rac::llm::acquire_lifecycle_llm(&ref); - if (rc != RAC_SUCCESS) { - return rc; - } - - rc = (ref.ops && ref.ops->append_context) ? ref.ops->append_context(ref.impl, text) - : RAC_ERROR_NOT_SUPPORTED; - rac::llm::release_lifecycle_llm(&ref); - return rc; -#endif + return call_lifecycle_op(&rac_llm_service_ops_t::append_context, text); } /** Generate a protobuf result from the lifecycle-owned LLM's adaptive context. */ @@ -2484,20 +2479,7 @@ rac_result_t rac_llm_generate_from_context_proto(const uint8_t* request_proto_by /** Clear adaptive context retained by the lifecycle-owned LLM. */ rac_result_t rac_llm_clear_context_lifecycle(void) { -#if !defined(RAC_HAVE_PROTOBUF) - return RAC_ERROR_FEATURE_NOT_AVAILABLE; -#else - rac::llm::LifecycleLlmRef ref; - rac_result_t rc = rac::llm::acquire_lifecycle_llm(&ref); - if (rc != RAC_SUCCESS) { - return rc; - } - - rc = (ref.ops && ref.ops->clear_context) ? ref.ops->clear_context(ref.impl) - : RAC_ERROR_NOT_SUPPORTED; - rac::llm::release_lifecycle_llm(&ref); - return rc; -#endif + return call_lifecycle_op(&rac_llm_service_ops_t::clear_context); } } // extern "C" From a58bfd1aae8f595eb625e04fc3e5c2294c5782b2 Mon Sep 17 00:00:00 2001 From: Shubham Singh Date: Sun, 5 Jul 2026 15:23:21 +0530 Subject: [PATCH 5/9] fix: resolve C++ compilation errors in adaptive context PR --- .../src/features/llm/llm_module.cpp | 48 ++++++++++--------- 1 file changed, 25 insertions(+), 23 deletions(-) diff --git a/sdk/runanywhere-commons/src/features/llm/llm_module.cpp b/sdk/runanywhere-commons/src/features/llm/llm_module.cpp index c61f582b17..d821192ee7 100644 --- a/sdk/runanywhere-commons/src/features/llm/llm_module.cpp +++ b/sdk/runanywhere-commons/src/features/llm/llm_module.cpp @@ -2070,6 +2070,26 @@ rac_bool_t stream_token_callback(const char* token, void* user_data) { #endif // RAC_HAVE_PROTOBUF +template +rac_result_t call_lifecycle_op(Op op, Args&&... args) { +#if !defined(RAC_HAVE_PROTOBUF) + (void)op; + return RAC_ERROR_FEATURE_NOT_AVAILABLE; +#else + rac::llm::LifecycleLlmRef ref; + rac_result_t rc = rac::llm::acquire_lifecycle_llm(&ref); + if (rc != RAC_SUCCESS) { + return rc; + } + + rc = (ref.ops && (ref.ops->*op)) + ? (ref.ops->*op)(ref.impl, std::forward(args)...) + : RAC_ERROR_NOT_SUPPORTED; + rac::llm::release_lifecycle_llm(&ref); + return rc; +#endif +} + } // namespace extern "C" { @@ -2353,27 +2373,6 @@ rac_result_t rac_llm_cancel_proto(rac_proto_buffer_t* out_event) { #endif } -namespace { -template -rac_result_t call_lifecycle_op(Op op, Args&&... args) { -#if !defined(RAC_HAVE_PROTOBUF) - (void)op; - return RAC_ERROR_FEATURE_NOT_AVAILABLE; -#else - rac::llm::LifecycleLlmRef ref; - rac_result_t rc = rac::llm::acquire_lifecycle_llm(&ref); - if (rc != RAC_SUCCESS) { - return rc; - } - - rc = (ref.ops && (ref.ops->*op)) - ? (ref.ops->*op)(ref.impl, std::forward(args)...) - : RAC_ERROR_NOT_SUPPORTED; - rac::llm::release_lifecycle_llm(&ref); - return rc; -#endif -} -} // namespace /** Seed the lifecycle-owned LLM's adaptive context with a system prompt. */ rac_result_t rac_llm_inject_system_prompt_lifecycle(const char* prompt) { @@ -2428,8 +2427,11 @@ rac_result_t rac_llm_generate_from_context_proto(const uint8_t* request_proto_by std::vector stop_storage; std::vector stop_ptrs; std::string grammar_storage; - rac_llm_options_t options = - options_from_request(request, system_prompt, stop_storage, stop_ptrs, grammar_storage); + std::vector history_storage; + std::vector history_ptrs; + rac_llm_options_t options = options_from_request( + request, system_prompt, stop_storage, stop_ptrs, grammar_storage, history_storage, + history_ptrs); options.streaming_enabled = RAC_FALSE; rac_llm_result_t raw{}; From 4783629f86b79997572e21ab1738186d7761d1d6 Mon Sep 17 00:00:00 2001 From: Shubham Singh Date: Fri, 10 Jul 2026 08:38:26 +0530 Subject: [PATCH 6/9] fix(swift): honor custom thinking tags in adaptive context --- sdk/runanywhere-commons/src/features/llm/llm_module.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/sdk/runanywhere-commons/src/features/llm/llm_module.cpp b/sdk/runanywhere-commons/src/features/llm/llm_module.cpp index d821192ee7..2b7bffe08e 100644 --- a/sdk/runanywhere-commons/src/features/llm/llm_module.cpp +++ b/sdk/runanywhere-commons/src/features/llm/llm_module.cpp @@ -2456,7 +2456,13 @@ rac_result_t rac_llm_generate_from_context_proto(const uint8_t* request_proto_by const char* thinking = nullptr; size_t thinking_len = 0; const char* raw_text = raw.text ? raw.text : ""; - (void)rac_llm_extract_thinking(raw_text, &response, &response_len, &thinking, &thinking_len); + std::string thinking_open_tag; + std::string thinking_close_tag; + thinking_tags_from_request_or_model(request, ref, &thinking_open_tag, &thinking_close_tag); + (void)rac_llm_extract_thinking_with_tags( + raw_text, thinking_open_tag.empty() ? nullptr : thinking_open_tag.c_str(), + thinking_close_tag.empty() ? nullptr : thinking_close_tag.c_str(), &response, + &response_len, &thinking, &thinking_len); int32_t thinking_tokens = 0; int32_t response_tokens = raw.completion_tokens; From df3862a630507a3ea224a634c0d84ef990d557b5 Mon Sep 17 00:00:00 2001 From: Shubham Singh Date: Fri, 10 Jul 2026 08:39:21 +0530 Subject: [PATCH 7/9] fix(swift): preserve adaptive context error codes --- .../Bridge/Extensions/CppBridge+LLM.swift | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/sdk/runanywhere-swift/Sources/RunAnywhere/Foundation/Bridge/Extensions/CppBridge+LLM.swift b/sdk/runanywhere-swift/Sources/RunAnywhere/Foundation/Bridge/Extensions/CppBridge+LLM.swift index 2062db862f..a8d11235c2 100644 --- a/sdk/runanywhere-swift/Sources/RunAnywhere/Foundation/Bridge/Extensions/CppBridge+LLM.swift +++ b/sdk/runanywhere-swift/Sources/RunAnywhere/Foundation/Bridge/Extensions/CppBridge+LLM.swift @@ -137,14 +137,11 @@ extension CppBridge { /// Convert a failed native adaptive-context status into an SDK error with backend detail. private static func throwIfFailed(_ status: rac_result_t, operation: String) throws { - guard status == RAC_SUCCESS else { - let nativeMessage = String(cString: rac_error_message(status)) - throw SDKException( - code: .processingFailed, - message: "LLM adaptive context \(operation) failed: \(nativeMessage) (\(status))", - category: .component - ) - } + guard let exception = SDKException.from(rcResult: status) else { return } + let nativeMessage = String(cString: rac_error_message(status)) + var proto = exception.proto + proto.message = "LLM adaptive context \(operation) failed: \(nativeMessage) (\(status))" + throw SDKException(proto: proto, underlying: exception.underlying) } } } From e4535bc58a4df4fca7ea838af273c76e9056a92e Mon Sep 17 00:00:00 2001 From: Shubham Singh Date: Mon, 13 Jul 2026 07:52:05 +0530 Subject: [PATCH 8/9] fix(llm): null-terminate extracted thinking slices # Conflicts: # sdk/runanywhere-commons/src/features/llm/llm_module.cpp --- .../src/features/llm/llm_module.cpp | 42 ++++++++++++------- 1 file changed, 28 insertions(+), 14 deletions(-) diff --git a/sdk/runanywhere-commons/src/features/llm/llm_module.cpp b/sdk/runanywhere-commons/src/features/llm/llm_module.cpp index 2b7bffe08e..b503e738a4 100644 --- a/sdk/runanywhere-commons/src/features/llm/llm_module.cpp +++ b/sdk/runanywhere-commons/src/features/llm/llm_module.cpp @@ -2170,22 +2170,29 @@ rac_result_t rac_llm_generate_proto(const uint8_t* request_proto_bytes, size_t r thinking_tags_from_request_or_model(request, ref, &thinking_open_tag, &thinking_close_tag); (void)rac_llm_extract_thinking_with_tags( raw_text, thinking_open_tag.empty() ? nullptr : thinking_open_tag.c_str(), - thinking_close_tag.empty() ? nullptr : thinking_close_tag.c_str(), &response, &response_len, - &thinking, &thinking_len); + thinking_close_tag.empty() ? nullptr : thinking_close_tag.c_str(), &response, + &response_len, &thinking, &thinking_len); + const std::string response_text = + response ? std::string(response, response_len) : std::string(); + const std::string thinking_text = + thinking ? std::string(thinking, thinking_len) : std::string(); + const char* response_cstr = response_text.c_str(); + const char* thinking_cstr = thinking_text.empty() ? nullptr : thinking_text.c_str(); int32_t thinking_tokens = 0; int32_t response_tokens = raw.completion_tokens; - (void)rac_llm_split_thinking_tokens(raw.completion_tokens, response, thinking, &thinking_tokens, - &response_tokens); + (void)rac_llm_split_thinking_tokens(raw.completion_tokens, response_cstr, thinking_cstr, + &thinking_tokens, &response_tokens); LLMGenerationResult result; - set_result_from_raw(ref, raw, response, response_len, thinking, thinking_len, thinking_tokens, - response_tokens, options.max_tokens, &result); - set_structured_output_if_present(response, &result); + set_result_from_raw(ref, raw, response_cstr, response_text.size(), thinking_cstr, + thinking_text.size(), thinking_tokens, response_tokens, options.max_tokens, + &result); + set_structured_output_if_present(response_cstr, &result); publish_generation_event( runanywhere::v1::GENERATION_EVENT_KIND_COMPLETED, request.prompt().c_str(), nullptr, - response, nullptr, ref.model_id, raw.completion_tokens, + response_cstr, nullptr, ref.model_id, raw.completion_tokens, raw.total_time_ms > 0 ? raw.total_time_ms : elapsed, raw.prompt_tokens > 0 ? raw.prompt_tokens : estimate_tokens(request.prompt().c_str()), ref.framework_name, static_cast(raw.tokens_per_second), @@ -2463,20 +2470,27 @@ rac_result_t rac_llm_generate_from_context_proto(const uint8_t* request_proto_by raw_text, thinking_open_tag.empty() ? nullptr : thinking_open_tag.c_str(), thinking_close_tag.empty() ? nullptr : thinking_close_tag.c_str(), &response, &response_len, &thinking, &thinking_len); + const std::string response_text = + response ? std::string(response, response_len) : std::string(); + const std::string thinking_text = + thinking ? std::string(thinking, thinking_len) : std::string(); + const char* response_cstr = response_text.c_str(); + const char* thinking_cstr = thinking_text.empty() ? nullptr : thinking_text.c_str(); int32_t thinking_tokens = 0; int32_t response_tokens = raw.completion_tokens; - (void)rac_llm_split_thinking_tokens(raw.completion_tokens, response, thinking, &thinking_tokens, - &response_tokens); + (void)rac_llm_split_thinking_tokens(raw.completion_tokens, response_cstr, thinking_cstr, + &thinking_tokens, &response_tokens); LLMGenerationResult result; - set_result_from_raw(ref, raw, response, response_len, thinking, thinking_len, thinking_tokens, - response_tokens, options.max_tokens, &result); - set_structured_output_if_present(response, &result); + set_result_from_raw(ref, raw, response_cstr, response_text.size(), thinking_cstr, + thinking_text.size(), thinking_tokens, response_tokens, options.max_tokens, + &result); + set_structured_output_if_present(response_cstr, &result); publish_generation_event( runanywhere::v1::GENERATION_EVENT_KIND_COMPLETED, request.prompt().c_str(), nullptr, - response, nullptr, ref.model_id, raw.completion_tokens, + response_cstr, nullptr, ref.model_id, raw.completion_tokens, raw.total_time_ms > 0 ? raw.total_time_ms : elapsed, raw.prompt_tokens); rac_llm_result_free(&raw); From 30ce3caaca4373f58549ce17f3225e4087a682d1 Mon Sep 17 00:00:00 2001 From: Shubham Singh Date: Mon, 13 Jul 2026 15:23:28 +0530 Subject: [PATCH 9/9] fix(swift): set adaptive-context streaming flag on options --- .../Foundation/Bridge/Extensions/CppBridge+LLM.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/runanywhere-swift/Sources/RunAnywhere/Foundation/Bridge/Extensions/CppBridge+LLM.swift b/sdk/runanywhere-swift/Sources/RunAnywhere/Foundation/Bridge/Extensions/CppBridge+LLM.swift index a8d11235c2..c4945371fc 100644 --- a/sdk/runanywhere-swift/Sources/RunAnywhere/Foundation/Bridge/Extensions/CppBridge+LLM.swift +++ b/sdk/runanywhere-swift/Sources/RunAnywhere/Foundation/Bridge/Extensions/CppBridge+LLM.swift @@ -109,7 +109,7 @@ extension CppBridge { options: RALLMGenerationOptions? = nil ) async throws -> RALLMGenerationResult { var request = (options ?? .defaults()).toRALLMGenerateRequest(prompt: query) - request.streamingEnabled = false + request.options.streamingEnabled = false return try NativeProtoABI.invoke( request, symbol: LLMAdaptiveContextABI.generateFromContext,