1+ """
2+ $TYPEDEF
3+
4+ Mirror Descent algorithm for learning coordinated solutions.
5+
6+ This algorithm is designed for stochastic benchmarks.
7+
8+ Reference: <https://arxiv.org/abs/2505.04757>
9+
10+ # Fields
11+ $TYPEDFIELDS
12+ """
13+ @kwdef struct MirrorDescent{A<: PerturbedFenchelYoungLossImitation } <: AbstractAlgorithm
14+ " inner imitation algorithm for supervised learning"
15+ inner_algorithm:: A = PerturbedFenchelYoungLossImitation ()
16+ end
17+
18+ """
19+ $TYPEDSIGNATURES
20+
21+ Train a DFLPolicy using the Mirror Descent algorithm on a provided training dataset.
22+
23+ # Core training method
24+
25+ # Arguments
26+ - `epochs`: number of training epochs per iteration
27+ - `iterations`: number of mirror descent iterations
28+ - `κ`: scaling factor for the perturbation magnitude
29+ - `metrics`: tuple of metrics to track during training
30+ - `verbose`: if true, prints progress at each iteration
31+ - `imitation_start`: if true, the first iteration uses pure imitation learning (no perturbation)
32+ """
33+
34+ function train_policy! (
35+ benchmark:: ExogenousStochasticBenchmark ,
36+ algorithm:: MirrorDescent ,
37+ policy:: DFLPolicy ,
38+ train_dataset,
39+ anticipative_solver,
40+ perturbed_anticipative_solver;
41+ epochs= 10 ,
42+ iterations= 10 ,
43+ κ= 1.0 ,
44+ metrics:: Tuple = (),
45+ verbose:: Bool = false ,
46+ imitation_start:: Bool = true
47+ )
48+
49+ augmented_dataset = train_dataset
50+ return map (1 : iterations) do n_it
51+ if verbose
52+ println (" Iteration $n_it / $iterations "
53+ end
54+
55+ perturb = n_it > 1 || ! imitation_start
56+
57+ augmented_dataset = augment_dataset (
58+ benchmark, augmented_dataset, policy. statistical_model, anticipative_solver, perturbed_anticipative_solver;
59+ κ= κ, perturb= perturb
60+ )
61+
62+ train_policy! (
63+ algorithm. inner_algorithm,
64+ policy,
65+ augmented_dataset;
66+ epochs= epochs,
67+ metrics= metrics,
68+ maximizer_kwargs= sample -> sample. context,
69+ )
70+ end
71+ end
72+
73+ """
74+ $TYPEDSIGNATURES
75+
76+ Generate a dataset for the provided benchmark and train a DFLPolicy using the Mirror Descent algorithm.
77+
78+ # Benchmark convenience wrapper
79+
80+ This high-level function handles all setup from the benchmark and returns a trained policy.
81+
82+ # Arguments
83+ - `dataset_size`: number of samples in the training dataset
84+ - `epochs`: number of training epochs per iteration
85+ - `iterations`: number of mirror descent iterations
86+ - `κ`: scaling factor for the perturbation magnitude
87+ - `metrics`: tuple of metrics to track during training
88+ - `seed`: random seed for reproducibility
89+ - `verbose`: if true, prints progress at each iteration
90+ - `imitation_start`: if true, the first iteration uses pure imitation learning (no perturbation)
91+ - `model_kwargs`: additional keyword arguments passed to `generate_statistical_model`
92+ - `maximizer_kwargs`: additional keyword arguments passed to `generate_maximizer`
93+ - `solver_kwargs`: additional keyword arguments passed to `generate_anticipative_solver` and `generate_parametric_anticipative_solver`
94+ - `nb_scenarios`: number of scenarios per instance.
95+ - `context_per_instance`: number of contexts per instance.
96+ """
97+
98+
99+
100+ function train_policy (
101+ algorithm:: MirrorDescent ,
102+ benchmark:: ExogenousStochasticBenchmark ;
103+ dataset_size= 30 ,
104+ epochs= 10 ,
105+ iterations= 10 ,
106+ κ= 1.0 ,
107+ metrics:: Tuple = (),
108+ seed= nothing ,
109+ verbose:: Bool = false ,
110+ imitation_start:: Bool = true ,
111+ model_kwargs= (;),
112+ maximizer_kwargs= (;),
113+ solver_kwargs= (;),
114+ nb_scenarios = 1 ,
115+ context_per_instance = 1 ,
116+ )
117+ train_dataset = generate_dataset (benchmark, dataset_size; nb_scenarios= nb_scenarios, contexts_per_instance= context_per_instance, seed= seed)
118+
119+ model = generate_statistical_model (benchmark; seed= seed, model_kwargs... )
120+ maximizer = generate_maximizer (benchmark; maximizer_kwargs... )
121+ policy = DFLPolicy (model, maximizer)
122+
123+ anticipative_solver = generate_anticipative_solver (benchmark; solver_kwargs... )
124+ parametric_anticipative_solver = generate_parametric_anticipative_solver (benchmark; solver_kwargs... )
125+ (; nb_samples, ε, threaded, seed) = algorithm. inner_algorithm
126+ perturbed_anticipative_solver = PerturbedAdditive ((θ; scenario, kwargs... ) -> parametric_anticipative_solver (θ, scenario; kwargs... ); ε= κ* ε, nb_samples= nb_samples, seed= seed, threaded= threaded)
127+
128+
129+ histories_per_iteration = train_policy! (
130+ benchmark, algorithm, policy, train_dataset, anticipative_solver, perturbed_anticipative_solver;
131+ epochs= epochs, iterations= iterations, κ= κ, metrics= metrics, verbose= verbose, imitation_start= imitation_start
132+ )
133+
134+ return histories_per_iteration, policy
135+ end
136+
137+ function augment_dataset (
138+ bench:: ExogenousStochasticBenchmark ,
139+ train_dataset:: AbstractArray ,
140+ model,
141+ anticipative_solver,
142+ perturbed_anticipative_solver;
143+ κ= 1.0 ,
144+ perturb= false
145+ )
146+ return _augment_dataset (
147+ Val (fieldtype (eltype (train_dataset), :y ) != = Nothing),
148+ bench, train_dataset, model, anticipative_solver, perturbed_anticipative_solver;
149+ κ= κ, perturb= perturb
150+ )
151+ end
152+
153+ # Raw dataset (samples have no y) → create new DataSamples
154+ function _augment_dataset (
155+ :: Val{false} ,
156+ bench, train_dataset, model, anticipative_solver, perturbed_anticipative_solver;
157+ κ= 1.0 , perturb= false
158+ )
159+ return map (train_dataset) do sample
160+ θ = model (sample. x)
161+ if perturb
162+ if is_minimization_problem (bench)
163+ y = perturbed_anticipative_solver (- κ* θ; scenario= sample. scenario, sample. context... )
164+ else
165+ y = perturbed_anticipative_solver (κ* θ; scenario= sample. scenario, sample. context... )
166+ end
167+ else
168+ y = anticipative_solver (sample. scenario; sample. context... )
169+ end
170+ DataSample (sample; y= y)
171+ end
172+ end
173+
174+ # Augmented dataset (samples already have y) → update y in place
175+ function _augment_dataset (
176+ :: Val{true} ,
177+ bench, train_dataset, model, anticipative_solver, perturbed_anticipative_solver;
178+ κ= 1.0 , perturb= false
179+ )
180+ for (i, sample) in enumerate (train_dataset)
181+ θ = model (sample. x)
182+ if perturb
183+ if is_minimization_problem (bench)
184+ y = perturbed_anticipative_solver (- κ* θ; scenario= sample. scenario, sample. context... )
185+ else
186+ y = perturbed_anticipative_solver (κ* θ; scenario= sample. scenario, sample. context... )
187+ end
188+ else
189+ y = anticipative_solver (sample. scenario; sample. context... )
190+ end
191+ ET = eltype (sample. y)
192+ y_converted = convert (typeof (sample. y), ET <: Integer ? round .(ET, y) : y)
193+ train_dataset[i] = DataSample (sample; y= y_converted)
194+ end
195+ return train_dataset
196+ end
0 commit comments