Skip to content

Commit dfae6d5

Browse files
guilhermebodinguilhermebodin
authored
Add Backtest (#125)
* Add backtest and time_limit_sec * Add backtest
1 parent 4a70667 commit dfae6d5

10 files changed

Lines changed: 82 additions & 13 deletions

File tree

‎Project.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
name = "ScoreDrivenModels"
22
uuid = "4a87933e-d659-11e9-0e65-7f40dedd4a3a"
33
authors = ["guilhermebodin <guilherme.b.moraes@gmail.com>, raphaelsaavedra <raphael.saavedra93@gmail.com>"]
4-
version = "0.1.5"
4+
version = "0.1.6"
55

66
[deps]
77
Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f"

‎src/MLE.jl‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -95,7 +95,8 @@ function fit(gas::Model{D, T}, y::Vector{T};
9595
initial_params::Matrix{T} = DEFAULT_INITIAL_PARAM,
9696
opt_method::AbstractOptimizationMethod = NelderMead(gas, DEFAULT_NUM_SEEDS),
9797
verbose::Int = DEFAULT_VERBOSE,
98-
throw_errors::Bool = false) where {D, T}
98+
throw_errors::Bool = false,
99+
time_limit_sec::Int = 10^8) where {D, T}
99100

100101
verbose in [0, 1, 2, 3] || throw(ErrorException, "verbose argument must be in [0, 1, 2, 3]")
101102
# Number of initial_points and number of params to estimate
@@ -119,7 +120,7 @@ function fit(gas::Model{D, T}, y::Vector{T};
119120
func = TwiceDifferentiable(psi_tilde -> log_lik(psi_tilde, y, gas_fit,
120121
initial_params, unknowns, n),
121122
opt_method.initial_points[i])
122-
opt_result = optimize(func, opt_method, verbose, i)
123+
opt_result = optimize(func, opt_method, verbose, i, time_limit_sec)
123124
update_aux_estimation!(aux_est, func, opt_result)
124125
verbose >= 1 && println("Round $i of $n_initial_points - Log-likelihood: $(-opt_result.minimum * length(y))")
125126
catch err

‎src/ScoreDrivenModels.jl‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ include("simulate.jl")
2020
include("diagnostics.jl")
2121
include("univariate_score_driven_recursion.jl")
2222
include("MLE.jl")
23+
include("backtest.jl")
2324
include("prints.jl")
2425

2526
# Optimization methods

‎src/backtest.jl‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
export backtest
2+
3+
struct Backtest
4+
abs_errors::Matrix{Float64}
5+
crps_scores::Matrix{Float64}
6+
function Backtest(n::Int, steps_ahead::Int)
7+
abs_errors = Matrix{Float64}(undef, n, steps_ahead)
8+
crps_scores = Matrix{Float64}(undef, n, steps_ahead)
9+
return new(abs_errors, crps_scores)
10+
end
11+
end
12+
13+
discrete_crps_indicator_function(val::T, z::T) where {T} = val < z
14+
function crps(val::T, scenarios::Vector{T}) where {T}
15+
sorted_scenarios = sort(scenarios)
16+
m = length(scenarios)
17+
crps_score = zero(T)
18+
for i = 1:m
19+
crps_score +=
20+
(sorted_scenarios[i] - val) *
21+
(m * discrete_crps_indicator_function(val, sorted_scenarios[i]) - i + 0.5)
22+
end
23+
return (2 / m^2) * crps_score
24+
end
25+
evaluate_abs_error(y::Vector{T}, forecast::Vector{T}) where T = abs.(y - forecast)
26+
function evaluate_crps(y::Vector{T}, scenarios::Matrix{T}) where {T}
27+
crps_scores = Vector{T}(undef, length(y))
28+
for k = 1:length(y)
29+
crps_scores[k] = crps(y[k], scenarios[k, :])
30+
end
31+
return crps_scores
32+
end
33+
34+
"""
35+
TODO
36+
"""
37+
function backtest(gas::Model{<:Distribution, T}, y::Vector{T}, steps_ahead::Int, start_idx::Int;
38+
S::Int = 1000,
39+
initial_params::Matrix{T} = stationary_initial_params(gas),
40+
opt_method = NelderMead(gas, DEFAULT_NUM_SEEDS)) where T
41+
num_mle = length(y) - start_idx - steps_ahead
42+
backtest = Backtest(num_mle, steps_ahead)
43+
for i in 1:num_mle
44+
println("Backtest: step $i of $num_mle")
45+
gas_to_fit = deepcopy(gas)
46+
y_to_fit = y[1:start_idx - 1 + i]
47+
y_to_verify = y[start_idx + i:start_idx - 1 + i + steps_ahead]
48+
ScoreDrivenModels.fit!(gas_to_fit, y_to_fit; initial_params=initial_params, opt_method=opt_method)
49+
forec = forecast(y_to_fit, gas_to_fit, steps_ahead; S=S, initial_params=initial_params)
50+
abs_errors = evaluate_abs_error(y_to_verify, forec.observation_forecast)
51+
crps_scores = evaluate_crps(y_to_verify, forec.observation_scenarios)
52+
backtest.abs_errors[i, :] = abs_errors
53+
backtest.crps_scores[i, :] = crps_scores
54+
end
55+
return backtest
56+
end

‎src/opt_methods/IPNewton.jl‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ function IPNewton(model::Model{D, T}, initial_points::Vector{Vector{T}}; f_tol::
3939
return IPNewton{T}(f_tol, g_tol, ub, lb, iterations, initial_points)
4040
end
4141

42-
function optimize(func::Optim.TwiceDifferentiable, opt_method::IPNewton{T}, verbose::Int, i::Int) where T
42+
function optimize(func::Optim.TwiceDifferentiable, opt_method::IPNewton{T}, verbose::Int, i::Int, time_limit_sec::Int) where T
4343
cons = Optim.TwiceDifferentiableConstraints(opt_method.lb, opt_method.ub)
44-
return optimize(func, opt_method, cons, Optim.IPNewton(), verbose, i)
44+
return optimize(func, opt_method, cons, Optim.IPNewton(), verbose, i, time_limit_sec)
4545
end

‎src/opt_methods/LBFGS.jl‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,6 @@ function LBFGS(model::Model{D, T}, initial_points::Vector{Vector{T}}; f_tol::T =
3030
return LBFGS{T}(f_tol, g_tol, iterations, initial_points)
3131
end
3232

33-
function optimize(func::Optim.TwiceDifferentiable, opt_method::LBFGS{T}, verbose::Int, i::Int) where T
34-
return optimize(func, opt_method, Optim.LBFGS(), verbose, i)
33+
function optimize(func::Optim.TwiceDifferentiable, opt_method::LBFGS{T}, verbose::Int, i::Int, time_limit_sec::Int) where T
34+
return optimize(func, opt_method, Optim.LBFGS(), verbose, i, time_limit_sec)
3535
end

‎src/opt_methods/NelderMead.jl‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,6 @@ function NelderMead(model::Model{D, T}, initial_points::Vector{Vector{T}}; f_tol
3030
return NelderMead{T}(f_tol, g_tol, iterations, initial_points)
3131
end
3232

33-
function optimize(func::Optim.TwiceDifferentiable, opt_method::NelderMead{T}, verbose::Int, i::Int) where T
34-
return optimize(func, opt_method, Optim.NelderMead(), verbose, i)
33+
function optimize(func::Optim.TwiceDifferentiable, opt_method::NelderMead{T}, verbose::Int, i::Int, time_limit_sec::Int) where T
34+
return optimize(func, opt_method, Optim.NelderMead(), verbose, i, time_limit_sec)
3535
end

‎src/opt_methods/common_methods.jl‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -26,25 +26,27 @@ function create_initial_points(model::Model{D, T}, n_initial_points::Int, LB::T,
2626
end
2727

2828
function optimize(func::Optim.TwiceDifferentiable, opt_method::AbstractOptimizationMethod{T},
29-
optimizer::Optim.AbstractOptimizer, verbose::Int, i::Int) where T
29+
optimizer::Optim.AbstractOptimizer, verbose::Int, i::Int, time_limit_sec::Int) where T
3030

3131
return Optim.optimize(func, opt_method.initial_points[i], optimizer,
3232
Optim.Options(f_tol = opt_method.f_tol,
3333
g_tol = opt_method.g_tol,
3434
iterations = opt_method.iterations,
35-
show_trace = show_trace(verbose) ))
35+
show_trace = show_trace(verbose),
36+
time_limit = time_limit_sec))
3637
end
3738

3839
function optimize(func::Optim.TwiceDifferentiable, opt_method::AbstractOptimizationMethod{T},
3940
cons::Optim.TwiceDifferentiableConstraints,
4041
optimizer::Optim.AbstractOptimizer,
41-
verbose::Int, i::Int) where T
42+
verbose::Int, i::Int, time_limit_sec::Int) where T
4243

4344
return Optim.optimize(func, cons, opt_method.initial_points[i], optimizer,
4445
Optim.Options(f_tol = opt_method.f_tol,
4546
g_tol = opt_method.g_tol,
4647
iterations = opt_method.iterations,
47-
show_trace = show_trace(verbose) ))
48+
show_trace = show_trace(verbose),
49+
time_limit = time_limit_sec))
4850
end
4951

5052
show_trace(verbose::Int) = (verbose == 3 ? true : false)

‎test/runtests.jl‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,3 +11,4 @@ include("test_initial_params.jl")
1111
include("test_diagnostics.jl")
1212
include("test_estimate.jl")
1313
include("test_simulate.jl")
14+
include("test_backtest.jl")

‎test/test_backtest.jl‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
@testset "Backtest" begin
2+
ω = [0.1, 0.1]
3+
A = [0.5 0; 0 0.5]
4+
B = [0.5 0; 0 0.5]
5+
simulation = simulate_GAS_1_1(Normal, 0.0, ω, A, B, 1)
6+
gas = ScoreDrivenModels.Model(1, 1, Normal, 0.0)
7+
bac = backtest(gas, simulation, 10, 4985)
8+
end

0 commit comments

Comments
 (0)