Skip to content

Commit 3faea98

Browse files
srossitto79srossitto79
committed
Merge branch 'feat/qlora-training'
2 parents 6de1bc6 + 22277e3 commit 3faea98

23 files changed

Lines changed: 2617 additions & 43 deletions

‎common/arg.cpp‎

Lines changed: 83 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -3606,32 +3606,108 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
36063606
{ "-lr", "--learning-rate" }, "ALPHA",
36073607
string_format("adamw or sgd optimizer alpha (default: %.2g); note: sgd alpha recommended ~10x (no momentum)", (double) params.lr.lr0),
36083608
[](common_params & params, const std::string & value) { params.lr.lr0 = std::stof(value); }
3609-
).set_examples({ LLAMA_EXAMPLE_FINETUNE }));
3609+
).set_examples({ LLAMA_EXAMPLE_FINETUNE, LLAMA_EXAMPLE_FINETUNE_QLORA }));
36103610
add_opt(common_arg({ "-lr-min", "--learning-rate-min" }, "ALPHA",
36113611
string_format("(if >0) final learning rate after decay (if -decay-epochs is set, default=%.2g)",
36123612
(double) params.lr.lr_min),
36133613
[](common_params & params, const std::string & value) { params.lr.lr_min = std::stof(value); }
3614-
).set_examples({ LLAMA_EXAMPLE_FINETUNE }));
3614+
).set_examples({ LLAMA_EXAMPLE_FINETUNE, LLAMA_EXAMPLE_FINETUNE_QLORA }));
36153615
add_opt(common_arg(
36163616
{"-decay-epochs", "--learning-rate-decay-epochs"}, "ALPHA",
36173617
string_format("(if >0) decay learning rate to -lr-min after this many epochs (exponential decay, default=%.2g)", (double) params.lr.decay_epochs),
36183618
[](common_params & params, const std::string & value) { params.lr.decay_epochs = std::stof(value); }
3619-
).set_examples({ LLAMA_EXAMPLE_FINETUNE }));
3619+
).set_examples({ LLAMA_EXAMPLE_FINETUNE, LLAMA_EXAMPLE_FINETUNE_QLORA }));
36203620
add_opt(common_arg(
36213621
{"-wd", "--weight-decay"}, "WD",
36223622
string_format("adamw or sgd optimizer weight decay (0 is off; recommend very small e.g. 1e-9) (default: %.2g).", (double) params.lr.wd),
36233623
[](common_params & params, const std::string & value) { params.lr.wd = std::stof(value); }
3624-
).set_examples({ LLAMA_EXAMPLE_FINETUNE }));
3624+
).set_examples({ LLAMA_EXAMPLE_FINETUNE, LLAMA_EXAMPLE_FINETUNE_QLORA }));
36253625
add_opt(common_arg(
36263626
{"-val-split", "--val-split"}, "FRACTION",
36273627
string_format("fraction of data to use as validation set for training (default: %.2g).", (double) params.val_split),
36283628
[](common_params & params, const std::string & value) { params.val_split = std::stof(value); }
3629-
).set_examples({ LLAMA_EXAMPLE_FINETUNE }));
3629+
).set_examples({ LLAMA_EXAMPLE_FINETUNE, LLAMA_EXAMPLE_FINETUNE_QLORA }));
3630+
// qlora flags
3631+
add_opt(common_arg(
3632+
{"--lora-rank"}, "N",
3633+
string_format("LoRA rank r (default: %d)", params.lora_rank),
3634+
[](common_params & params, int value) { params.lora_rank = value; }
3635+
).set_examples({ LLAMA_EXAMPLE_FINETUNE_QLORA }));
3636+
add_opt(common_arg(
3637+
{"--lora-alpha"}, "F",
3638+
string_format("LoRA alpha (default: %d = use rank value)", (int) params.lora_alpha),
3639+
[](common_params & params, const std::string & value) { params.lora_alpha = std::stof(value); }
3640+
).set_examples({ LLAMA_EXAMPLE_FINETUNE_QLORA }));
3641+
add_opt(common_arg(
3642+
{"--lora-targets"}, "SUBSTRINGS",
3643+
string_format("comma-separated substrings of tensor names to add LoRA to (default: %s)", params.lora_targets.c_str()),
3644+
[](common_params & params, const std::string & value) { params.lora_targets = value; }
3645+
).set_examples({ LLAMA_EXAMPLE_FINETUNE_QLORA }));
3646+
add_opt(common_arg(
3647+
{"--lora-out"}, "FNAME",
3648+
string_format("output LoRA adapter GGUF path (default: %s)", params.lora_out.c_str()),
3649+
[](common_params & params, const std::string & value) { params.lora_out = value; }
3650+
).set_examples({ LLAMA_EXAMPLE_FINETUNE_QLORA }));
3651+
add_opt(common_arg(
3652+
{"--train-file"}, "FNAME",
3653+
"JSONL training dataset (fields: messages|prompt+response|text)",
3654+
[](common_params & params, const std::string & value) { params.train_file = value; }
3655+
).set_examples({ LLAMA_EXAMPLE_FINETUNE_QLORA }));
3656+
add_opt(common_arg(
3657+
{"--save-every"}, "N",
3658+
"save adapter checkpoint every N dataset windows during training (default: 0 = only at end)",
3659+
[](common_params & params, int value) { params.save_every = value; }
3660+
).set_examples({ LLAMA_EXAMPLE_FINETUNE_QLORA }));
3661+
add_opt(common_arg(
3662+
{"--freeze-layers"}, "N",
3663+
"freeze first N transformer layers — no LoRA adapters allocated for blk.0..blk.N-1 (default: 0 = train all layers)",
3664+
[](common_params & params, int value) { params.lora_freeze_layers = value; }
3665+
).set_examples({ LLAMA_EXAMPLE_FINETUNE_QLORA }));
3666+
add_opt(common_arg(
3667+
{"--grad-checkpoint"}, "N",
3668+
"gradient checkpointing interval to reduce peak activation VRAM (0 = disabled, default: 0)",
3669+
[](common_params & params, int value) { params.grad_checkpoint_interval = value; }
3670+
).set_examples({ LLAMA_EXAMPLE_FINETUNE_QLORA }));
3671+
add_opt(common_arg(
3672+
{"--train-on-prompt"},
3673+
"compute loss on prompt tokens too, not just the response (default: response-only loss)",
3674+
[](common_params & params) { params.train_on_prompt = true; }
3675+
).set_examples({ LLAMA_EXAMPLE_FINETUNE_QLORA }));
3676+
add_opt(common_arg(
3677+
{"--shuffle-dataset"},
3678+
"shuffle dataset windows at the start of each epoch (default: sequential order)",
3679+
[](common_params & params) { params.shuffle_dataset = true; }
3680+
).set_examples({ LLAMA_EXAMPLE_FINETUNE_QLORA }));
3681+
add_opt(common_arg(
3682+
{"--grpo-mode"},
3683+
"enable GRPO IPC training loop (prompts and rewards supplied via stdin/stdout)",
3684+
[](common_params & params) { params.grpo_mode = true; }
3685+
).set_examples({ LLAMA_EXAMPLE_FINETUNE_QLORA }));
3686+
add_opt(common_arg(
3687+
{"--n-gen"}, "N",
3688+
string_format("GRPO: number of generations per prompt (default: %d)", params.grpo_n_gen),
3689+
[](common_params & params, int value) { params.grpo_n_gen = value; }
3690+
).set_examples({ LLAMA_EXAMPLE_FINETUNE_QLORA }));
3691+
add_opt(common_arg(
3692+
{"--n-steps"}, "N",
3693+
string_format("GRPO: total optimizer steps (default: %d)", params.grpo_n_steps),
3694+
[](common_params & params, int value) { params.grpo_n_steps = value; }
3695+
).set_examples({ LLAMA_EXAMPLE_FINETUNE_QLORA }));
3696+
add_opt(common_arg(
3697+
{"--grpo-temp"}, "F",
3698+
string_format("GRPO: sampling temperature for rollout generation (default: %.2f)", (double) params.grpo_temperature),
3699+
[](common_params & params, const std::string & value) { params.grpo_temperature = std::stof(value); }
3700+
).set_examples({ LLAMA_EXAMPLE_FINETUNE_QLORA }));
3701+
add_opt(common_arg(
3702+
{"--grpo-max-tokens"}, "N",
3703+
string_format("GRPO: max tokens per generation (default: %d)", params.grpo_max_tokens),
3704+
[](common_params & params, int value) { params.grpo_max_tokens = value; }
3705+
).set_examples({ LLAMA_EXAMPLE_FINETUNE_QLORA }));
36303706
add_opt(common_arg(
36313707
{"-epochs", "--epochs"}, "N",
36323708
string_format("optimizer max # of epochs (default: %d)", params.lr.epochs),
36333709
[](common_params & params, int epochs) { params.lr.epochs = epochs; }
3634-
).set_examples({ LLAMA_EXAMPLE_FINETUNE }));
3710+
).set_examples({ LLAMA_EXAMPLE_FINETUNE, LLAMA_EXAMPLE_FINETUNE_QLORA }));
36353711
add_opt(common_arg(
36363712
{"-opt", "--optimizer"}, "sgd|adamw", "adamw or sgd",
36373713
[](common_params & params, const std::string & name) {
@@ -3640,7 +3716,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
36403716
throw std::invalid_argument("invalid --optimizer, valid options: adamw, sgd");
36413717
}
36423718
}
3643-
).set_examples({ LLAMA_EXAMPLE_FINETUNE }));
3719+
).set_examples({ LLAMA_EXAMPLE_FINETUNE, LLAMA_EXAMPLE_FINETUNE_QLORA }));
36443720
add_opt(common_arg(
36453721
{"--check"},
36463722
string_format("check rather than generate results (default: %s)", params.check ? "true" : "false"),

‎common/common.h‎

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -103,6 +103,7 @@ enum llama_example {
103103
LLAMA_EXAMPLE_TTS,
104104
LLAMA_EXAMPLE_DIFFUSION,
105105
LLAMA_EXAMPLE_FINETUNE,
106+
LLAMA_EXAMPLE_FINETUNE_QLORA,
106107
LLAMA_EXAMPLE_FIT_PARAMS,
107108
LLAMA_EXAMPLE_RESULTS,
108109

@@ -517,7 +518,26 @@ struct common_params {
517518
// finetune
518519
struct lr_opt lr;
519520
enum ggml_opt_optimizer_type optimizer = GGML_OPT_OPTIMIZER_TYPE_ADAMW;
520-
float val_split = 0.05f; // fraction of the data used for the validation set
521+
float val_split = 0.05f; // fraction of the data used for the validation set
522+
523+
// qlora fine-tuning
524+
int32_t lora_rank = 16; // LoRA rank (r)
525+
float lora_alpha = 0.0f; // LoRA alpha (0 = use rank value)
526+
std::string lora_targets = "attn_q,attn_output,ffn_gate,ffn_up,ffn_down"; // comma-separated substrings to match trainable tensors
527+
std::string lora_out = "adapter.gguf"; // output adapter GGUF path
528+
std::string train_file = ""; // JSONL training dataset path
529+
int32_t save_every = 0; // save checkpoint every N optimizer steps (0 = disabled)
530+
int32_t lora_freeze_layers = 0; // do not apply LoRA to the first N transformer layers
531+
int32_t grad_checkpoint_interval = 0; // gradient checkpointing interval to reduce peak VRAM (0 = disabled)
532+
bool train_on_prompt = false; // include prompt tokens in training loss (default: response tokens only)
533+
bool shuffle_dataset = false; // shuffle dataset windows at the start of each epoch
534+
535+
// grpo training
536+
bool grpo_mode = false; // enable GRPO IPC training loop
537+
int32_t grpo_n_gen = 8; // generations per prompt
538+
int32_t grpo_n_steps = 500; // total GRPO optimizer steps
539+
float grpo_temperature = 0.8f; // sampling temperature for rollouts
540+
int32_t grpo_max_tokens = 512; // max tokens per generation
521541

522542
// embedding
523543
bool embedding = false; // get only sentence embedding

‎examples/CMakeLists.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@ else()
3434
add_subdirectory(speculative-simple)
3535
add_subdirectory(gen-docs)
3636
add_subdirectory(training)
37+
add_subdirectory(qlora_training)
3738
add_subdirectory(diffusion)
3839
if (NOT GGML_BACKEND_DL)
3940
add_subdirectory(convert-llama2c-to-ggml)

‎examples/qlora_training/CMakeLists.txt‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
set(TARGET llama-finetune-qlora)
2+
add_executable(${TARGET} finetune_qlora.cpp)
3+
install(TARGETS ${TARGET} RUNTIME)
4+
target_link_libraries(${TARGET} PRIVATE common llama ${CMAKE_THREAD_LIBS_INIT})
5+
target_compile_features(${TARGET} PRIVATE cxx_std_17)

0 commit comments

Comments
 (0)