Skip to content

Commit 839f912

Browse files
davide221davide221
authored
Merge pull request #236 from weicj/refactor-cpp-placement-config
feat(server): add C++ backend placement foundation
2 parents 41a5bab + 4aaa065 commit 839f912

15 files changed

Lines changed: 294 additions & 49 deletions

‎dflash/src/common/backend_factory.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ std::unique_ptr<ModelBackend> create_backend(const BackendArgs & args) {
3737
cfg.target_path = args.model_path;
3838
cfg.draft_path = args.draft_path;
3939
cfg.device = args.device;
40-
cfg.draft_gpu = args.draft_gpu;
40+
cfg.draft_gpu = args.draft_device.gpu;
4141
cfg.stream_fd = args.stream_fd;
4242
cfg.fa_window = args.fa_window;
4343
cfg.kq_stride_pad = args.kq_stride_pad;

‎dflash/src/common/backend_factory.h‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111
#pragma once
1212

1313
#include "model_backend.h"
14-
#include "device_placement.h"
14+
#include "placement/placement_config.h"
1515

1616
#include <memory>
1717
#include <string>
@@ -30,7 +30,7 @@ struct BackendArgs {
3030

3131
// Device placement
3232
DevicePlacement device;
33-
int draft_gpu = 0;
33+
DevicePlacement draft_device;
3434

3535
// I/O — only used when running under daemon_loop (legacy). The new
3636
// server passes -1 and uses on_token callbacks instead.

‎dflash/src/common/device_placement.h‎

Lines changed: 3 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -1,31 +1,6 @@
1-
// Device placement configuration for model backends.
2-
//
3-
// Describes which GPU(s) to use for a model. Supports:
4-
// - Single-GPU: just gpu field
5-
// - Multi-GPU layer-split: layer_split_gpus + optional weights
6-
// - Peer access between GPUs
1+
// Compatibility include for older common/device_placement.h users.
2+
// New C++ placement code should include placement/placement_config.h.
73

84
#pragma once
95

10-
#include <vector>
11-
12-
namespace dflash::common {
13-
14-
struct DevicePlacement {
15-
int gpu = 0; // primary GPU (single-GPU mode)
16-
17-
// Multi-GPU layer-split. Empty = single GPU mode.
18-
std::vector<int> layer_split_gpus; // GPU IDs for each shard
19-
std::vector<double> layer_split_weights; // proportional layer distribution (optional)
20-
21-
bool peer_access = false; // enable CUDA peer access between GPUs
22-
int max_ctx = 8192; // max KV cache context length
23-
24-
bool is_layer_split() const { return layer_split_gpus.size() > 1; }
25-
26-
int primary_gpu() const {
27-
return layer_split_gpus.empty() ? gpu : layer_split_gpus[0];
28-
}
29-
};
30-
31-
} // namespace dflash::common
6+
#include "placement/placement_config.h"

‎dflash/src/common/layer_split_utils.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55

66
#pragma once
77

8-
#include "device_placement.h"
8+
#include "placement/placement_config.h"
99

1010
#include <string>
1111
#include <utility>

‎dflash/src/gemma4/gemma4_backend.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
#pragma once
77

88
#include "common/model_backend.h"
9-
#include "common/device_placement.h"
9+
#include "placement/placement_config.h"
1010
#include "gemma4_internal.h"
1111
#include "common/sampler.h"
1212

‎dflash/src/gemma4/gemma4_daemon.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22

33
#pragma once
44

5-
#include "device_placement.h"
5+
#include "placement/placement_config.h"
66
#include <string>
77

88
namespace dflash::common {

‎dflash/src/laguna/laguna_daemon.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@
1313

1414
#pragma once
1515

16-
#include "device_placement.h"
16+
#include "placement/placement_config.h"
1717
#include <string>
1818
#include "ggml.h"
1919

‎dflash/src/placement/placement_backend.h‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
// Backend placement identifiers shared by C++ server/runtime code.
2+
3+
#pragma once
4+
5+
#include <string>
6+
7+
namespace dflash::common {
8+
9+
enum class PlacementBackend {
10+
Auto,
11+
Cuda,
12+
Hip,
13+
};
14+
15+
inline const char * placement_backend_name(PlacementBackend backend) {
16+
switch (backend) {
17+
case PlacementBackend::Auto: return "auto";
18+
case PlacementBackend::Cuda: return "cuda";
19+
case PlacementBackend::Hip: return "hip";
20+
}
21+
return "auto";
22+
}
23+
24+
inline bool parse_placement_backend(const std::string & value,
25+
PlacementBackend & out) {
26+
if (value == "auto") {
27+
out = PlacementBackend::Auto;
28+
return true;
29+
}
30+
if (value == "cuda") {
31+
out = PlacementBackend::Cuda;
32+
return true;
33+
}
34+
if (value == "hip") {
35+
out = PlacementBackend::Hip;
36+
return true;
37+
}
38+
return false;
39+
}
40+
41+
inline PlacementBackend compiled_placement_backend() {
42+
#if defined(DFLASH27B_BACKEND_HIP) || defined(GGML_USE_HIP)
43+
return PlacementBackend::Hip;
44+
#else
45+
return PlacementBackend::Cuda;
46+
#endif
47+
}
48+
49+
inline bool placement_backend_supported(PlacementBackend backend) {
50+
return backend == PlacementBackend::Auto ||
51+
backend == compiled_placement_backend();
52+
}
53+
54+
} // namespace dflash::common

‎dflash/src/placement/placement_config.h‎

Lines changed: 105 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,105 @@
1+
// Device placement configuration for model backends.
2+
//
3+
// Describes which backend device(s) to use for a model. Supports:
4+
// - Single-GPU: backend + gpu fields, exposed as cuda:0 / hip:0 / auto:0
5+
// - Multi-GPU layer-split: one backend + layer_split_gpus + optional weights
6+
// - Peer access between GPUs
7+
8+
#pragma once
9+
10+
#include "placement_backend.h"
11+
12+
#include <cstdlib>
13+
#include <string>
14+
#include <vector>
15+
16+
namespace dflash::common {
17+
18+
struct DevicePlacement {
19+
PlacementBackend backend = PlacementBackend::Auto;
20+
int gpu = 0; // primary GPU (single-GPU mode)
21+
22+
// Multi-GPU layer-split. Empty = single GPU mode.
23+
std::vector<int> layer_split_gpus; // GPU IDs for each shard
24+
std::vector<double> layer_split_weights; // proportional layer distribution (optional)
25+
26+
bool peer_access = false; // enable CUDA/HIP peer access between GPUs
27+
int max_ctx = 8192; // max KV cache context length
28+
29+
bool is_layer_split() const { return layer_split_gpus.size() > 1; }
30+
31+
int primary_gpu() const {
32+
return layer_split_gpus.empty() ? gpu : layer_split_gpus[0];
33+
}
34+
};
35+
36+
inline std::string placement_device_name(const DevicePlacement & device) {
37+
return std::string(placement_backend_name(device.backend)) + ":" +
38+
std::to_string(device.primary_gpu());
39+
}
40+
41+
inline bool parse_placement_device(const std::string & value,
42+
DevicePlacement & out) {
43+
const std::size_t sep = value.find(':');
44+
if (sep == std::string::npos || sep == 0 || sep + 1 >= value.size()) {
45+
return false;
46+
}
47+
48+
PlacementBackend backend = PlacementBackend::Auto;
49+
if (!parse_placement_backend(value.substr(0, sep), backend)) {
50+
return false;
51+
}
52+
53+
const std::string gpu_text = value.substr(sep + 1);
54+
char * end = nullptr;
55+
long gpu = std::strtol(gpu_text.c_str(), &end, 10);
56+
if (end == gpu_text.c_str() || *end != '\0' || gpu < 0) {
57+
return false;
58+
}
59+
60+
out.backend = backend;
61+
out.gpu = static_cast<int>(gpu);
62+
out.layer_split_gpus.clear();
63+
out.layer_split_weights.clear();
64+
return true;
65+
}
66+
67+
inline bool parse_placement_device_list(const std::string & value,
68+
DevicePlacement & out) {
69+
if (value.empty()) return false;
70+
71+
std::vector<int> gpus;
72+
PlacementBackend backend = PlacementBackend::Auto;
73+
bool have_backend = false;
74+
75+
std::size_t begin = 0;
76+
while (begin < value.size()) {
77+
const std::size_t end = value.find(',', begin);
78+
const std::string item = value.substr(
79+
begin,
80+
end == std::string::npos ? std::string::npos : end - begin);
81+
if (item.empty()) return false;
82+
83+
DevicePlacement parsed;
84+
if (!parse_placement_device(item, parsed)) return false;
85+
if (!have_backend) {
86+
backend = parsed.backend;
87+
have_backend = true;
88+
} else if (parsed.backend != backend) {
89+
return false;
90+
}
91+
gpus.push_back(parsed.gpu);
92+
93+
if (end == std::string::npos) break;
94+
begin = end + 1;
95+
}
96+
97+
if (gpus.empty()) return false;
98+
out.backend = backend;
99+
out.gpu = gpus[0];
100+
out.layer_split_gpus = gpus.size() > 1 ? gpus : std::vector<int>{};
101+
out.layer_split_weights.clear();
102+
return true;
103+
}
104+
105+
} // namespace dflash::common

‎dflash/src/qwen3/qwen3_backend.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
#pragma once
1313

1414
#include "common/model_backend.h"
15-
#include "common/device_placement.h"
15+
#include "placement/placement_config.h"
1616
#include "qwen3_drafter_model.h"
1717
#include "qwen3_drafter.h"
1818
#include "common/sampler.h"

0 commit comments

Comments
 (0)