Skip to content

Commit b6d73f1

Browse files
bmehta001Copilot
andcommitted
Add SDK download cancellation support
Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
1 parent 58bba93 commit b6d73f1

41 files changed

Lines changed: 1374 additions & 197 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

samples/cpp/live-audio-transcription/README.md

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,3 +26,7 @@ g++ -std=c++20 main.cpp -lfoundry_local -o live-audio-transcription-example
2626
# Synthetic 440Hz sine wave (no microphone needed)
2727
./live-audio-transcription-example --synth
2828
```
29+
30+
Press `Ctrl+C` to request a graceful stop. The sample passes that signal to
31+
execution-provider and model downloads so long-running downloads can be
32+
cancelled before transcription starts.

samples/cpp/live-audio-transcription/main.cpp

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -122,7 +122,8 @@ int main(int argc, char* argv[]) {
122122

123123
foundry_local::Manager::Create(config);
124124
auto& manager = foundry_local::Manager::Instance();
125-
manager.EnsureEpsDownloaded();
125+
auto isCancellationRequested = [] { return !g_running.load(); };
126+
manager.EnsureEpsDownloaded(nullptr, isCancellationRequested);
126127

127128
auto& catalog = manager.GetCatalog();
128129
auto* model = catalog.GetModel("nemotron-speech-streaming-en-0.6b");
@@ -131,9 +132,11 @@ int main(int argc, char* argv[]) {
131132
}
132133

133134
std::cout << "Downloading model (if needed)..." << std::endl;
134-
model->Download([](float pct) {
135-
std::cout << "\rDownloading: " << pct << "% " << std::flush;
136-
});
135+
model->Download(
136+
[](float pct) {
137+
std::cout << "\rDownloading: " << pct << "% " << std::flush;
138+
},
139+
isCancellationRequested);
137140
std::cout << std::endl;
138141
std::cout << "Loading model..." << std::endl;
139142
model->Load();

sdk/cpp/include/foundry_local_manager.h

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
#include <string>
66
#include <vector>
77
#include <memory>
8+
#include <functional>
89

910
#include <gsl/pointers>
1011
#include <gsl/span>
@@ -19,6 +20,8 @@ namespace foundry_local::Internal {
1920

2021
namespace foundry_local {
2122

23+
using ExecutionProviderDownloadProgressCallback = std::function<void(std::string epName, double percentage)>;
24+
2225
class Manager final {
2326
public:
2427
Manager(const Manager&) = delete;
@@ -61,7 +64,11 @@ namespace foundry_local {
6164

6265
/// Ensure execution providers are downloaded and registered.
6366
/// Once downloaded, EPs are not re-downloaded unless a new version is available.
64-
void EnsureEpsDownloaded() const;
67+
/// @param onProgress Optional callback receiving execution provider name and percentage progress.
68+
/// @param isCancellationRequested Optional callback checked on each progress update.
69+
/// Return true to cancel the in-progress download.
70+
void EnsureEpsDownloaded(ExecutionProviderDownloadProgressCallback onProgress = nullptr,
71+
CancellationCallback isCancellationRequested = nullptr) const;
6572

6673
private:
6774
explicit Manager(Configuration configuration, ILogger* logger);

sdk/cpp/include/model.h

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
#include <memory>
1111
#include <functional>
1212
#include <filesystem>
13+
#include <utility>
1314

1415
#include <gsl/pointers>
1516
#include <gsl/span>
@@ -33,6 +34,7 @@ namespace foundry_local {
3334
#endif
3435

3536
using DownloadProgressCallback = std::function<void(float percentage)>;
37+
using CancellationCallback = std::function<bool()>;
3638

3739
class IModel {
3840
public:
@@ -43,7 +45,13 @@ namespace foundry_local {
4345
virtual bool IsLoaded() const = 0;
4446
virtual bool IsCached() const = 0;
4547
virtual const std::filesystem::path& GetPath() const = 0;
46-
virtual void Download(DownloadProgressCallback onProgress = nullptr) = 0;
48+
49+
/// Download the model to the local cache.
50+
/// @param onProgress Optional callback receiving percentage progress.
51+
/// @param isCancellationRequested Optional callback checked on each progress update.
52+
/// Return true to cancel the in-progress download.
53+
virtual void Download(DownloadProgressCallback onProgress = nullptr,
54+
CancellationCallback isCancellationRequested = nullptr) = 0;
4755
virtual void Load() = 0;
4856
virtual void Unload() = 0;
4957
virtual void RemoveFromCache() = 0;
@@ -123,7 +131,8 @@ namespace foundry_local {
123131

124132
const ModelInfo& GetInfo() const;
125133
const std::filesystem::path& GetPath() const override;
126-
void Download(DownloadProgressCallback onProgress = nullptr) override;
134+
void Download(DownloadProgressCallback onProgress = nullptr,
135+
CancellationCallback isCancellationRequested = nullptr) override;
127136
void Load() override;
128137

129138
bool IsLoaded() const override;
@@ -158,8 +167,9 @@ namespace foundry_local {
158167
bool IsLoaded() const override { return SelectedVariant().IsLoaded(); }
159168
bool IsCached() const override { return SelectedVariant().IsCached(); }
160169
const std::filesystem::path& GetPath() const override { return SelectedVariant().GetPath(); }
161-
void Download(DownloadProgressCallback onProgress = nullptr) override {
162-
SelectedVariant().Download(std::move(onProgress));
170+
void Download(DownloadProgressCallback onProgress = nullptr,
171+
CancellationCallback isCancellationRequested = nullptr) override {
172+
SelectedVariant().Download(std::move(onProgress), std::move(isCancellationRequested));
163173
}
164174
void Load() override { SelectedVariant().Load(); }
165175
void Unload() override { SelectedVariant().Unload(); }

sdk/cpp/sample/main.cpp

Lines changed: 22 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,12 +3,26 @@
33

44
#include "foundry_local.h"
55

6+
#include <atomic>
7+
#include <csignal>
68
#include <iostream>
79
#include <string>
810
#include <vector>
911

1012
using namespace foundry_local;
1113

14+
namespace {
15+
std::atomic<bool> g_cancelRequested{false};
16+
17+
void SignalHandler(int /*signum*/) {
18+
g_cancelRequested.store(true);
19+
}
20+
21+
bool IsCancellationRequested() {
22+
return g_cancelRequested.load();
23+
}
24+
} // namespace
25+
1226
// ---------------------------------------------------------------------------
1327
// Logger
1428
// ---------------------------------------------------------------------------
@@ -93,7 +107,8 @@ void ChatNonStreaming(Manager& manager, const std::string& alias) {
93107
return;
94108
}
95109

96-
model->Download([](float pct) { std::cout << "\rDownloading: " << pct << "% " << std::flush; });
110+
model->Download([](float pct) { std::cout << "\rDownloading: " << pct << "% " << std::flush; },
111+
IsCancellationRequested);
97112
std::cout << "\n";
98113

99114
model->Load();
@@ -176,7 +191,8 @@ void TranscribeAudio(Manager& manager, const std::string& alias, const std::stri
176191
return;
177192
}
178193

179-
model->Download([](float pct) { std::cout << "\rDownloading: " << pct << "% " << std::flush; });
194+
model->Download([](float pct) { std::cout << "\rDownloading: " << pct << "% " << std::flush; },
195+
IsCancellationRequested);
180196
std::cout << "\n";
181197

182198
model->Load();
@@ -223,7 +239,8 @@ void ChatWithToolCalling(Manager& manager, const std::string& alias) {
223239
return;
224240
}
225241

226-
model->Download([](float pct) { std::cout << "\rDownloading: " << pct << "% " << std::flush; });
242+
model->Download([](float pct) { std::cout << "\rDownloading: " << pct << "% " << std::flush; },
243+
IsCancellationRequested);
227244
std::cout << "\n";
228245

229246
model->Load();
@@ -327,6 +344,8 @@ void ChatWithToolCalling(Manager& manager, const std::string& alias) {
327344
// ---------------------------------------------------------------------------
328345
int main() {
329346
try {
347+
std::signal(SIGINT, SignalHandler);
348+
330349
StdLogger logger;
331350
Manager::Create({"SampleApp"}, &logger);
332351
auto& manager = Manager::Instance();

sdk/cpp/src/core.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,7 @@ namespace foundry_local {
7373
std::unique_ptr<ResponseBuffer, decltype(safeDeleter)> responseGuard(&response, safeDeleter);
7474

7575
if (callback != nullptr) {
76-
execCbCmd_(&request, &response, reinterpret_cast<void*>(callback), data);
76+
execCbCmd_(&request, &response, callback, data);
7777
}
7878
else {
7979
execCmd_(&request, &response);

sdk/cpp/src/core_helpers.h

Lines changed: 65 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -6,12 +6,15 @@
66

77
#pragma once
88

9+
#include <charconv>
910
#include <string>
1011
#include <string_view>
1112
#include <vector>
1213
#include <functional>
1314
#include <exception>
15+
#include <system_error>
1416
#include <unordered_map>
17+
#include <utility>
1518

1619
#include <nlohmann/json.hpp>
1720

@@ -47,37 +50,79 @@ namespace foundry_local::detail {
4750
return core->call(command, logger, &payload, callback, userData);
4851
}
4952

53+
inline bool TryParseFloatToken(std::string_view token, float& value) {
54+
if (token.empty()) {
55+
return false;
56+
}
57+
58+
const auto* begin = token.data();
59+
const auto* end = begin + token.size();
60+
const auto result = std::from_chars(begin, end, value);
61+
return result.ec == std::errc{} && result.ptr == end;
62+
}
63+
64+
inline bool TryParseDoubleToken(std::string_view token, double& value) {
65+
if (token.empty()) {
66+
return false;
67+
}
68+
69+
const auto* begin = token.data();
70+
const auto* end = begin + token.size();
71+
const auto result = std::from_chars(begin, end, value);
72+
return result.ec == std::errc{} && result.ptr == end;
73+
}
74+
5075
// Serialize + call with a streaming chunk handler.
5176
// Wraps the caller-supplied onChunk with the native callback boilerplate
5277
// (null/length checks, exception capture, rethrow after the call).
5378
// The errorContext string is used to prefix any core-layer error message.
5479
inline CoreResponse CallWithStreamingCallback(Internal::IFoundryLocalCore* core, std::string_view command,
55-
const std::string& payload, ILogger& logger,
56-
const std::function<void(const std::string&)>& onChunk,
57-
std::string_view errorContext) {
80+
const std::string* payload, ILogger& logger,
81+
const std::function<void(const std::string&)>& onChunk,
82+
std::string_view errorContext,
83+
CancellationCallback isCancellationRequested = nullptr) {
5884
struct State {
5985
const std::function<void(const std::string&)>* cb;
86+
CancellationCallback isCancellationRequested;
87+
bool cancellationObserved = false;
6088
std::exception_ptr exception;
61-
} state{&onChunk, nullptr};
62-
63-
auto nativeCallback = [](void* data, int32_t len, void* user) {
64-
if (!data || len <= 0)
65-
return;
89+
} state{&onChunk, std::move(isCancellationRequested), false, nullptr};
6690

91+
auto nativeCallback = [](const void* data, int32_t len, void* user) -> int32_t {
6792
auto* st = static_cast<State*>(user);
68-
if (st->exception)
69-
return;
93+
if (!st) {
94+
return 0;
95+
}
96+
97+
if (st->exception || st->cancellationObserved) {
98+
return 1;
99+
}
100+
101+
if (!data || len <= 0)
102+
return 0;
70103

71104
try {
105+
if (st->isCancellationRequested && st->isCancellationRequested()) {
106+
st->cancellationObserved = true;
107+
return 1;
108+
}
109+
72110
std::string chunk(static_cast<const char*>(data), static_cast<size_t>(len));
73111
(*(st->cb))(chunk);
74112
}
75113
catch (...) {
76114
st->exception = std::current_exception();
115+
return 1;
77116
}
117+
118+
return 0;
78119
};
79120

80-
auto response = core->call(command, logger, &payload, +nativeCallback, &state);
121+
auto response = core->call(command, logger, payload, +nativeCallback, &state);
122+
if (state.cancellationObserved) {
123+
throw Exception("Operation cancelled", logger);
124+
}
125+
81126
if (response.HasError()) {
82127
throw Exception(std::string(errorContext) + response.error, logger);
83128
}
@@ -89,6 +134,15 @@ namespace foundry_local::detail {
89134
return response;
90135
}
91136

137+
inline CoreResponse CallWithStreamingCallback(Internal::IFoundryLocalCore* core, std::string_view command,
138+
const std::string& payload, ILogger& logger,
139+
const std::function<void(const std::string&)>& onChunk,
140+
std::string_view errorContext,
141+
CancellationCallback isCancellationRequested = nullptr) {
142+
return CallWithStreamingCallback(core, command, &payload, logger, onChunk, errorContext,
143+
std::move(isCancellationRequested));
144+
}
145+
92146
// Overload: allow Params object directly
93147
inline CoreResponse CallWithParams(Internal::IFoundryLocalCore* core, std::string_view command,
94148
const nlohmann::json& params, ILogger& logger) {

sdk/cpp/src/flcore_native.h

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -23,8 +23,9 @@ extern "C"
2323
int32_t ErrorLength;
2424
};
2525

26-
// Callback signature: void(*)(void* data, int length, void* userData)
27-
using UserCallbackFn = void(__cdecl*)(void*, int32_t, void*);
26+
// Callback signature: int32_t(*)(const void* data, int length, void* userData)
27+
// Return 0 to continue, 1 to cancel.
28+
using UserCallbackFn = int32_t(__cdecl*)(const void*, int32_t, void*);
2829

2930
struct StreamingRequestBuffer {
3031
const void* Command;
@@ -37,8 +38,8 @@ extern "C"
3738

3839
// Exported function pointer types
3940
using execute_command_fn = void(__cdecl*)(RequestBuffer*, ResponseBuffer*);
40-
using execute_command_with_callback_fn = void(__cdecl*)(RequestBuffer*, ResponseBuffer*, void* /*callback*/,
41-
void* /*userData*/);
41+
using execute_command_with_callback_fn = void(__cdecl*)(RequestBuffer*, ResponseBuffer*,
42+
UserCallbackFn /*callback*/, void* /*userData*/);
4243
using execute_command_with_binary_fn = void(__cdecl*)(StreamingRequestBuffer*, ResponseBuffer*);
4344
using free_response_fn = void(__cdecl*)(ResponseBuffer*);
4445

sdk/cpp/src/foundry_local_internal_core.h

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,8 @@ namespace foundry_local {
1212

1313
/// Native callback signature used by the core DLL interop.
1414
/// Parameters: (data, dataLength, userData).
15-
using NativeCallbackFn = void (*)(void*, int32_t, void*);
15+
/// Return 0 to continue, 1 to cancel the native operation.
16+
using NativeCallbackFn = int32_t(__cdecl*)(const void*, int32_t, void*);
1617

1718
/// Value returned by IFoundryLocalCore::call().
1819
/// On success, `data` contains the response payload and `error` is empty.
@@ -40,4 +41,4 @@ namespace foundry_local {
4041
};
4142

4243
} // namespace Internal
43-
} // namespace foundry_local
44+
} // namespace foundry_local

0 commit comments

Comments
 (0)