Rename unconstrained_convex_optimizationto unconstrained_convex_minimization

Author: Shuvomoy

examples/unconstrained_convex_minimization/accelerated_gradient_convex.jl

@@ -0,0 +1,57 @@
+using PEPit, OrderedCollections, Mosek, MosekTools
+
+function wc_accelerated_gradient_convex(mu, L, n; solver=Mosek.Optimizer, verbose=true)
+
+    problem = PEP()
+
+
+    param = OrderedDict("mu" => mu, "L" => L)
+    func = declare_function!(problem, SmoothStronglyConvexFunction, param)
+
+
+    xs = stationary_point!(func)
+    fs = value!(func, xs)
+
+
+    x0 = set_initial_point!(problem)
+
+
+    set_initial_condition!(problem, (x0 - xs)^2 <= 1)
+
+
+    x = x0
+    y = x0
+    lam = 1.0
+    lam_old = lam
+
+    for _ in 1:n
+        lam_old = lam
+        lam = (1 + sqrt(4 * lam_old^2 + 1)) / 2
+        x_old = x
+        x = y - 1 / L * gradient!(func, y)
+        y = x + (lam_old - 1) / lam * (x - x_old)
+    end
+
+
+    set_performance_metric!(problem, value!(func, x) - fs)
+
+
+    pepit_tau = solve!(problem; solver=solver, verbose=verbose)
+
+
+    theoretical_tau = L / (2 * lam_old^2)
+
+    mu != 0 && @warn "Momentum is tuned for non-strongly convex functions."
+
+
+    if verbose
+        println("*** Example file: worst-case performance of accelerated gradient method ***")
+        println("\tPEPit guarantee:\t f(x_n)-f_* <= $(round(pepit_tau, digits=6)) ||x_0 - x_*||^2")
+        println("\tTheoretical guarantee:\t f(x_n)-f_* <= $(round(theoretical_tau, digits=6)) ||x_0 - x_*||^2")
+    end
+
+
+    return pepit_tau, theoretical_tau
+end
+
+pepit_tau, theoretical_tau = wc_accelerated_gradient_convex(0, 1, 1; solver=Mosek.Optimizer, verbose=true)

examples/unconstrained_convex_minimization/accelerated_gradient_convex_simplified.jl

@@ -0,0 +1,52 @@
+using PEPit, OrderedCollections, Mosek, MosekTools
+
+function wc_accelerated_gradient_convex_simplified(mu, L, n; solver=Mosek.Optimizer, verbose=true)
+
+    problem = PEP()
+
+
+    param = OrderedDict("mu" => mu, "L" => L)
+    func = declare_function!(problem, SmoothStronglyConvexFunction, param)
+
+
+    xs = stationary_point!(func)
+    fs = value!(func, xs)
+
+
+    x0 = set_initial_point!(problem)
+
+
+    set_initial_condition!(problem, (x0 - xs)^2 <= 1)
+
+
+    x_new = x0
+    y = x0
+    for i in 1:n
+        ipy = i - 1
+        x_old = x_new
+        x_new = y - 1 / L * gradient!(func, y)
+        y = x_new + ipy / (ipy + 3) * (x_new - x_old)
+    end
+
+
+    set_performance_metric!(problem, value!(func, x_new) - fs)
+
+
+    pepit_tau = solve!(problem; solver=solver, verbose=verbose)
+
+
+    theoretical_tau = 2 * L / (n^2 + 5 * n + 6)
+    mu != 0 && @warn "Momentum is tuned for non-strongly convex functions."
+
+
+    if verbose
+        println("*** Example file: worst-case performance of accelerated gradient method ***")
+        println("\tPEPit guarantee:\t f(x_n)-f_* <= $(round(pepit_tau, digits=6)) ||x_0 - x_*||^2")
+        println("\tTheoretical guarantee:\t f(x_n)-f_* <= $(round(theoretical_tau, digits=6)) ||x_0 - x_*||^2")
+    end
+
+
+    return pepit_tau, theoretical_tau
+end
+
+pepit_tau, theoretical_tau = wc_accelerated_gradient_convex_simplified(0, 1, 1; solver=Mosek.Optimizer, verbose=true)

…/accelerated_gradient_strongly_convex.jl …/accelerated_gradient_strongly_convex.jlexamples/unconstrained_convex_optimization/accelerated_gradient_strongly_convex.jl renamed to examples/unconstrained_convex_minimization/accelerated_gradient_strongly_convex.jl examples/unconstrained_convex_optimization/accelerated_gradient_strongly_convex.jl renamed to examples/unconstrained_convex_minimization/accelerated_gradient_strongly_convex.jl

@@ -28,7 +28,7 @@ function wc_accelerated_gradient_strongly_convex(mu, L, n; verbose=false)
     mu == 0 && @warn "Momentum is tuned for strongly convex functions!"
 
     if verbose
-        @info "🐱 Example file: worst-case performance of the accelerated gradient method" 
+        @info "🐱 Example file: worst-case performance of the accelerated gradient method"
         @info "💻  PEPit guarantee: f(x_n)-f_*  <= $(round(PEPit_tau, digits=6)) (f(x_0) - f(x_*) + mu/2*||x_0 - x_*||^2)"
         @info "📝 Theoretical guarantee: f(x_n)-f_*  <= $(round(theoretical_tau, digits=6)) (f(x_0) - f(x_*) + mu/2*||x_0 - x_*||^2)"
     end
@@ -50,4 +50,3 @@ function wc_accelerated_gradient_strongly_convex(mu, L, n; verbose=false)
 end
 
 PEPit_val, theoretical_val = wc_accelerated_gradient_strongly_convex(0.1, 1.0, 2, verbose=true)
-

examples/unconstrained_convex_minimization/accelerated_proximal_point.jl

@@ -0,0 +1,56 @@
+using PEPit, OrderedCollections, Mosek, MosekTools
+
+function wc_accelerated_proximal_point(A0, gammas, n; solver=Mosek.Optimizer, verbose=true)
+
+    problem = PEP()
+
+
+    func = declare_function!(problem, ConvexFunction, OrderedDict())
+
+
+    xs = stationary_point!(func)
+    fs = value!(func, xs)
+
+
+    x0 = set_initial_point!(problem)
+
+
+    set_initial_condition!(problem, value!(func, x0) - fs + A0 / 2 * (x0 - xs)^2 <= 1)
+
+
+    x, v = x0, x0
+    A = A0
+    for i in 1:n
+        alpha = (sqrt((A * gammas[i])^2 + 4 * A * gammas[i]) - A * gammas[i]) / 2
+        y = (1 - alpha) * x + alpha * v
+        x, _, _ = proximal_step!(y, func, gammas[i])
+        v = v + 1 / alpha * (x - y)
+        A = (1 - alpha) * A
+    end
+
+
+    set_performance_metric!(problem, value!(func, x) - fs)
+
+
+    pepit_tau = solve!(problem; solver=solver, verbose=verbose)
+
+
+    accumulation = 0.0
+    for i in 1:n
+        accumulation += sqrt(gammas[i])
+    end
+    theoretical_tau = 4 / A0 / accumulation^2
+
+
+    if verbose
+        println("*** Example file: worst-case performance of fast proximal point method ***")
+        println("\tPEPit guarantee:\t f(x_n)-f_* <= $(round(pepit_tau, digits=6)) (f(x_0) - f_* + A/2* ||x_0 - x_*||^2)")
+        println("\tTheoretical guarantee:\t f(x_n)-f_* <= $(round(theoretical_tau, digits=6)) (f(x_0) - f_* + A/2* ||x_0 - x_*||^2)")
+    end
+
+
+    return pepit_tau, theoretical_tau
+end
+
+
+pepit_tau, theoretical_tau = wc_accelerated_proximal_point(5, [(i + 1) / 1.1 for i in 0:2], 3; solver=Mosek.Optimizer, verbose=true)

examples/unconstrained_convex_minimization/conjugate_gradient.jl

@@ -0,0 +1,59 @@
+using PEPit, OrderedCollections, Mosek, MosekTools
+
+function wc_conjugate_gradient(L, n; solver=Mosek.Optimizer, verbose=true)
+
+    problem = PEP()
+
+
+    param = OrderedDict("L" => L)
+    func = declare_function!(problem, SmoothConvexFunction, param)
+
+
+    xs = stationary_point!(func)
+    fs = value!(func, xs)
+
+
+    x0 = set_initial_point!(problem)
+
+
+    set_initial_condition!(problem, (x0 - xs)^2 <= 1)
+
+
+    x_new = x0
+    g0, f0 = oracle!(func, x0)
+    span = Any[g0]
+    local gx, fx
+    for i in 1:n
+        x_old = x_new
+        x_new, gx, fx = exact_linesearch_step!(x_new, func, span)
+        push!(span, gx)
+        push!(span, x_old - x_new)
+    end
+
+
+    set_performance_metric!(problem, fx - fs)
+
+
+    pepit_tau = solve!(problem; solver=solver, verbose=verbose)
+
+
+    theta_new = 1.0
+    for i in 0:(n - 1)
+        if i < n - 1
+            theta_new = (1 + sqrt(4 * theta_new^2 + 1)) / 2
+        else
+            theta_new = (1 + sqrt(8 * theta_new^2 + 1)) / 2
+        end
+    end
+    theoretical_tau = L / (2 * theta_new^2)
+
+    if verbose
+        println("*** Example file: worst-case performance of conjugate gradient method ***")
+        println("\tPEPit guarantee:\t f(x_n)-f_* <= $(round(pepit_tau, digits=6)) ||x_0 - x_*||^2")
+        println("\tTheoretical guarantee:\t f(x_n)-f_* <= $(round(theoretical_tau, digits=6)) ||x_0 - x_*||^2")
+    end
+
+    return pepit_tau, theoretical_tau
+end
+
+pepit_tau, theoretical_tau = wc_conjugate_gradient(1.0, 2; solver=Mosek.Optimizer, verbose=true)

examples/unconstrained_convex_minimization/conjugate_gradient_qg_convex.jl

@@ -0,0 +1,51 @@
+using PEPit, OrderedCollections, Mosek, MosekTools
+
+function wc_conjugate_gradient_qg_convex(L, n; solver=Mosek.Optimizer, verbose=true)
+
+    problem = PEP()
+
+
+    func = declare_function!(problem, ConvexQGFunction, OrderedDict("L" => L))
+
+
+    xs = stationary_point!(func)
+    fs = value!(func, xs)
+
+
+    x0 = set_initial_point!(problem)
+
+
+    set_initial_condition!(problem, (x0 - xs)^2 <= 1)
+
+
+    x_new = x0
+    g0, f0 = oracle!(func, x0)
+    span = [g0]
+    local gx, fx
+    for i in 1:n
+        x_old = x_new
+        x_new, gx, fx = exact_linesearch_step!(x_new, func, span)
+        push!(span, gx)
+        push!(span, x_old - x_new)
+    end
+
+
+    set_performance_metric!(problem, fx - fs)
+
+
+    pepit_tau = solve!(problem; solver=solver, verbose=verbose)
+
+
+    theoretical_tau = L / (2 * (n + 1))
+
+    if verbose
+        println("*** Example file: worst-case performance of conjugate gradient method ***")
+        println("\tPEPit guarantee:\t f(x_n)-f_* <= $(round(pepit_tau, digits=6)) ||x_0 - x_*||^2")
+        println("\tTheoretical guarantee:\t f(x_n)-f_* <= $(round(theoretical_tau, digits=6)) ||x_0 - x_*||^2")
+    end
+
+
+    return pepit_tau, theoretical_tau
+end
+
+pepit_tau, theoretical_tau = wc_conjugate_gradient_qg_convex(1.0, 12; solver=Mosek.Optimizer, verbose=true)

examples/unconstrained_convex_minimization/cyclic_coordinate_descent.jl

@@ -0,0 +1,51 @@
+using PEPit, OrderedCollections, Mosek, MosekTools
+
+function wc_cyclic_coordinate_descent(L, n; solver=Mosek.Optimizer, verbose=true)
+
+    problem = PEP()
+
+
+    d = length(L)
+    partition = declare_block_partition!(problem, d)
+
+
+    param = OrderedDict("partition" => partition, "L" => L)
+    func = declare_function!(problem, BlockSmoothConvexFunctionCheap, param)
+
+
+    xs = stationary_point!(func)
+    fs = value!(func, xs)
+
+
+    x0 = set_initial_point!(problem)
+
+
+    set_initial_condition!(problem, (x0 - xs)^2 <= 1)
+
+
+    x = x0
+    for k in 0:(n - 1)
+        i = k % d
+        x = x - 1 / L[i + 1] * get_block(partition, gradient!(func, x), i + 1)
+    end
+
+
+    set_performance_metric!(problem, value!(func, x) - fs)
+
+
+    pepit_tau = solve!(problem; solver=solver, verbose=verbose)
+
+
+    theoretical_tau = nothing
+
+    if verbose
+        println("*** Example file: worst-case performance of cyclic coordinate descent with fixed step-sizes ***")
+        println("\tPEPit guarantee:\t f(x_n)-f_* <= $(round(pepit_tau, digits=6)) ||x_0 - x_*||^2")
+    end
+
+    return pepit_tau, theoretical_tau
+end
+
+
+L = [1.0, 2.0, 10.0]
+pepit_tau, theoretical_tau = wc_cyclic_coordinate_descent(L, 9; solver=Mosek.Optimizer, verbose=true)

examples/unconstrained_convex_minimization/epsilon_subgradient_method.jl

@@ -0,0 +1,53 @@
+using PEPit, OrderedCollections, Mosek, MosekTools
+
+function wc_epsilon_subgradient_method(M, n, gamma, eps, R; solver=Mosek.Optimizer, verbose=true)
+
+    problem = PEP()
+
+
+    func = declare_function!(problem, ConvexFunction, OrderedDict())
+
+
+    xs = stationary_point!(func)
+    fs = value!(func, xs)
+
+
+    x0 = set_initial_point!(problem)
+
+
+    set_initial_condition!(problem, (x0 - xs)^2 <= R^2)
+
+
+    x = x0
+    for _ in 1:n
+        x, gx, fx, epsilon = epsilon_subgradient_step!(x, func, gamma)
+        set_performance_metric!(problem, fx - fs)
+        add_constraint!(problem, epsilon <= eps)
+        add_constraint!(problem, gx^2 <= M^2)
+    end
+
+
+    gx, fx = oracle!(func, x)
+    add_constraint!(problem, gx^2 <= M^2)
+    set_performance_metric!(problem, fx - fs)
+
+
+    pepit_tau = solve!(problem; solver=solver, verbose=verbose)
+
+
+    theoretical_tau = (R^2 + 2 * (n + 1) * gamma * eps + (n + 1) * gamma^2 * M^2) / (2 * (n + 1) * gamma)
+
+    if verbose
+        println("*** Example file: worst-case performance of the epsilon-subgradient method ***")
+        println("\tPEPit guarantee:\t min_(0 <= t <= n) f(x_i) - f_* <= $(round(pepit_tau, digits=6))")
+        println("\tTheoretical guarantee:\t min_(0 <= t <= n) f(x_i) - f_* <= $(round(theoretical_tau, digits=6))")
+    end
+
+
+    return pepit_tau, theoretical_tau
+end
+
+
+M, n, eps, R = 2, 6, 0.1, 1
+gamma = 1 / sqrt(n + 1)
+pepit_tau, theoretical_tau = wc_epsilon_subgradient_method(M, n, gamma, eps, R; verbose=true)