diff --git a/.gitignore b/.gitignore index eec2830b5..e92eabf0a 100644 --- a/.gitignore +++ b/.gitignore @@ -7,8 +7,7 @@ __pycache__ .ipynb* Manifest.toml .vscode -benchmark experimental refs *.mathcha - +/benchmark/results/ diff --git a/benchmark/Project.toml b/benchmark/Project.toml new file mode 100644 index 000000000..8bc6e22e2 --- /dev/null +++ b/benchmark/Project.toml @@ -0,0 +1,4 @@ +[deps] +BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf" +CairoMakie = "13f3f980-e62b-5c42-98c6-ff1f3baf88f0" +TensorKit = "07d1fe3e-3e46-537d-9eac-e9e13d0d4cec" diff --git a/benchmark/README.md b/benchmark/README.md new file mode 100644 index 000000000..2ce857f86 --- /dev/null +++ b/benchmark/README.md @@ -0,0 +1,31 @@ +# TensorKit Benchmarks + +This directory contains a benchmark suite for TensorKit. +Most of the benchmarks are designed to capture performance characteristics of the library, and are not intended to be used as a benchmark suite for comparing different libraries. +In particular, the main goal here is to catch performance regressions and/or improvements between different versions of TensorKit. + +## Running the benchmarks + +The benchmarks are written using `BenchmarkTools.jl`, and the full suite can be found in the `SUITE` global variable defined in `benchmarks.jl`. +Sometimes, it is useful to run only a subset of the benchmarks. +To do this, you can use the `--modules` flag to specify which modules to run. +Alternatively, you can use the `TensorKitBenchmarks` module directly, which is designed after `BaseBenchmarks` to allow for conditional loading of the benchmarks. + +For a more streamlined CLI experience, you can use [`AirspeedVelocity.jl`](https://github.com/MilesCranmer/AirspeedVelocity.jl) to run the benchmarks. +The following command will run the benchmarks and compare with the current master branch: + +```bash +benchpkg TensorKit \ + --rev=dirty,master \ + -o benchmark/results/ \ + -exeflags="--threads=4" +``` + +To compare with previous results, the following command can be used: + +```bash +benchpkgtable TensorKit \ + --rev=dirty,master \ + -i benchmark/results/ \ + -o benchmark/results/ \ +``` diff --git a/benchmark/TensorKitBenchmarks/TensorKitBenchmarks.jl b/benchmark/TensorKitBenchmarks/TensorKitBenchmarks.jl new file mode 100644 index 000000000..e276c1704 --- /dev/null +++ b/benchmark/TensorKitBenchmarks/TensorKitBenchmarks.jl @@ -0,0 +1,55 @@ +module TensorKitBenchmarks + +using BenchmarkTools +using TensorKit +using TOML + +BenchmarkTools.DEFAULT_PARAMETERS.seconds = 1.0 +BenchmarkTools.DEFAULT_PARAMETERS.samples = 10000 +BenchmarkTools.DEFAULT_PARAMETERS.time_tolerance = 0.15 +BenchmarkTools.DEFAULT_PARAMETERS.memory_tolerance = 0.01 + +const PARAMS_PATH = joinpath(@__DIR__, "etc", "params.json") +const SUITE = BenchmarkGroup() +const MODULES = Dict("linalg" => :LinalgBenchmarks, + "indexmanipulations" => :IndexManipulationBenchmarks, + "tensornetworks" => :TensorNetworkBenchmarks) + +load!(id::AbstractString; kwargs...) = load!(SUITE, id; kwargs...) + +function load!(group::BenchmarkGroup, id::AbstractString; tune::Bool=false) + modsym = MODULES[id] + modpath = joinpath(dirname(@__FILE__), id, "$(modsym).jl") + Core.eval(@__MODULE__, :(include($modpath))) + mod = Core.eval(@__MODULE__, modsym) + modsuite = @invokelatest getglobal(mod, :SUITE) + group[id] = modsuite + if tune + results = BenchmarkTools.load(PARAMS_PATH)[1] + haskey(results, id) && loadparams!(modsuite, results[id], :evals) + end + return group +end + +loadall!(; kwargs...) = loadall!(SUITE; kwargs...) + +function loadall!(group::BenchmarkGroup; verbose::Bool=true, tune::Bool=false) + for id in keys(MODULES) + if verbose + print("loading group $(repr(id))... ") + time = @elapsed load!(group, id, tune=false) + println("done (took $time seconds)") + else + load!(group, id; tune=false) + end + end + if tune + results = BenchmarkTools.load(PARAMS_PATH)[1] + for (id, suite) in group + haskey(results, id) && loadparams!(suite, results[id], :evals) + end + end + return group +end + +end diff --git a/benchmark/TensorKitBenchmarks/indexmanipulations/IndexManipulationBenchmarks.jl b/benchmark/TensorKitBenchmarks/indexmanipulations/IndexManipulationBenchmarks.jl new file mode 100644 index 000000000..243fbce05 --- /dev/null +++ b/benchmark/TensorKitBenchmarks/indexmanipulations/IndexManipulationBenchmarks.jl @@ -0,0 +1,55 @@ +module IndexManipulationBenchmarks + +include(joinpath(@__DIR__, "..", "utils", "BenchUtils.jl")) + +using .BenchUtils +using BenchmarkTools +using TensorKit +using TOML + +const SUITE = BenchmarkGroup() +const all_parameters = TOML.parsefile(joinpath(@__DIR__, "benchparams.toml")) + +# permute! +# -------- +function init_permute_tensors(T, W, p) + C = randn(T, permute(W, p)) + A = randn(T, W) + return C, A +end +function benchmark_permute!(benchgroup, params::Dict) + haskey(benchgroup, "permute") || addgroup!(benchgroup, "permute") + bench = benchgroup["permute"] + for kwargs in expand_kwargs(params) + benchmark_permute!(bench; kwargs...) + end + return nothing +end +function benchmark_permute!(bench; sigmas=nothing, T="Float64", I="Trivial", dims, p) + T_ = parse_type(T) + I_ = parse_type(I) + + p_ = (Tuple(p[1]), Tuple(p[2])) + Vs = generate_space.(I_, dims, sigmas) + + codomain = mapreduce(Base.Fix1(getindex, Vs), ⊗, p_[1]; init=one(eltype(Vs))) + domain = mapreduce(Base.Fix1(getindex, Vs), ⊗, p_[2]; init=one(eltype(Vs))) + init() = init_permute_tensors(T_, codomain ← domain, p_) + + bench[T, I, dims, sigmas, p] = @benchmarkable permute!(C, A, $p_) setup = ((C, A) = $init()) + return nothing +end + +if haskey(all_parameters, "permute") + g = addgroup!(SUITE, "permute") + for params in all_parameters["permute"] + benchmark_permute!(g, params) + end +end + +# transpose! +# ---------- + +# TODO + +end diff --git a/benchmark/TensorKitBenchmarks/indexmanipulations/benchparams.toml b/benchmark/TensorKitBenchmarks/indexmanipulations/benchparams.toml new file mode 100644 index 000000000..c19aa0f66 --- /dev/null +++ b/benchmark/TensorKitBenchmarks/indexmanipulations/benchparams.toml @@ -0,0 +1,12 @@ +[[permute]] +T = ["Float64"] +I = "Trivial" +p = [[[2, 1], []]] +dims = [[7264, 7264], [43408, 1216]] + +[[permute]] +T = ["Float64"] +I = "Z2Irrep" +p = [[[2, 1], []]] +dims = [[7264, 7264], [43408, 1216]] +sigmas = [[0.5, 0.5]] diff --git a/benchmark/TensorKitBenchmarks/linalg/LinalgBenchmarks.jl b/benchmark/TensorKitBenchmarks/linalg/LinalgBenchmarks.jl new file mode 100644 index 000000000..75ad34a29 --- /dev/null +++ b/benchmark/TensorKitBenchmarks/linalg/LinalgBenchmarks.jl @@ -0,0 +1,81 @@ +module LinalgBenchmarks + +include(joinpath(@__DIR__, "..", "utils", "BenchUtils.jl")) + +using .BenchUtils +using BenchmarkTools +using TensorKit +using TOML + +const SUITE = BenchmarkGroup() +const all_parameters = TOML.parsefile(joinpath(@__DIR__, "benchparams.toml")) + +# mul! +# ---- +function init_mul_tensors(T, V) + A = randn(T, V[1] ← V[2]) + B = randn(T, V[2] ← V[3]) + C = randn(T, V[1] ← V[3]) + return A, B, C +end + +function benchmark_mul!(benchgroup, params::Dict) + haskey(benchgroup, "mul") || addgroup!(benchgroup, "mul") + bench = benchgroup["mul"] + for kwargs in expand_kwargs(params) + benchmark_mul!(bench; kwargs...) + end + return nothing +end + +function benchmark_mul!(bench; sigmas=nothing, T="Float64", I="Trivial", dims) + T_ = parse_type(T) + I_ = parse_type(I) + + Vs = generate_space.(I_, dims, sigmas) + init() = init_mul_tensors(T_, Vs) + + bench[T, I, dims, sigmas] = @benchmarkable mul!(C, A, B) setup = ((A, B, C) = $init()) + + return nothing +end + +if haskey(all_parameters, "mul") + g = addgroup!(SUITE, "mul") + for params in all_parameters["mul"] + benchmark_mul!(g, params) + end +end + +# svd! +# ---- +function init_svd_tensor(T, V) + A = randn(T, V[1] ← V[2]) + return A +end + +function benchmark_svd!(benchgroup, params::Dict) + haskey(benchgroup, "svd") || addgroup!(benchgroup, "svd") + bench = benchgroup["svd"] + for kwargs in expand_kwargs(params) + benchmark_svd!(bench; kwargs...) + end + return nothing +end +function benchmark_svd!(bench; sigmas=nothing, T="Float64", I="Trivial", dims) + T_ = parse_type(T) + I_ = parse_type(I) + Vs = generate_space.(I_, dims, sigmas) + init() = init_svd_tensor(T_, Vs) + bench[T, I, dims, sigmas] = @benchmarkable tsvd!(A) setup = (A = $init()) + return nothing +end + +if haskey(all_parameters, "svd") + g = addgroup!(SUITE, "svd") + for params in all_parameters["svd"] + benchmark_svd!(g, params) + end +end + +end diff --git a/benchmark/TensorKitBenchmarks/linalg/benchparams.toml b/benchmark/TensorKitBenchmarks/linalg/benchparams.toml new file mode 100644 index 000000000..314d106d8 --- /dev/null +++ b/benchmark/TensorKitBenchmarks/linalg/benchparams.toml @@ -0,0 +1,21 @@ +[[mul]] +T = ["Float64", "ComplexF64"] +I = "Trivial" +dims = [[2, 2, 2], [8, 8, 8], [32, 32, 32], [64, 64, 64], [128, 128, 128]] + +[[mul]] +T = ["Float64", "ComplexF64"] +I = "Z2Irrep" +dims = [[2, 2, 2], [8, 8, 8], [32, 32, 32], [64, 64, 64], [128, 128, 128]] +sigmas = [[0.5, 0.5, 0.5]] + +[[svd]] +T = ["Float64", "ComplexF64"] +I = "Trivial" +dims = [[2, 2], [8, 8], [32, 32], [64, 64], [128, 128]] + +[[svd]] +T = ["Float64", "ComplexF64"] +I = "Z2Irrep" +dims = [[2, 2], [8, 8], [32, 32], [64, 64], [128, 128]] +sigmas = [[0.5, 0.5]] diff --git a/benchmark/TensorKitBenchmarks/tensornetworks/TensorNetworkBenchmarks.jl b/benchmark/TensorKitBenchmarks/tensornetworks/TensorNetworkBenchmarks.jl new file mode 100644 index 000000000..becf3935c --- /dev/null +++ b/benchmark/TensorKitBenchmarks/tensornetworks/TensorNetworkBenchmarks.jl @@ -0,0 +1,145 @@ +module TensorNetworkBenchmarks + +include(joinpath(@__DIR__, "..", "utils", "BenchUtils.jl")) + +using .BenchUtils +using BenchmarkTools +using TensorKit +using TOML + +const SUITE = BenchmarkGroup() +const all_parameters = TOML.parsefile(joinpath(@__DIR__, "benchparams.toml")) + +# mpo contraction +# --------------- +function init_mpo_tensors(T, (Vmps, Vmpo, Vphys)) + A = Tensor(randn, T, Vmps ⊗ Vphys ⊗ Vmps') + M = Tensor(randn, T, Vmpo ⊗ Vphys ⊗ Vphys' ⊗ Vmpo') + FL = Tensor(randn, T, Vmps ⊗ Vmpo' ⊗ Vmps') + FR = Tensor(randn, T, Vmps ⊗ Vmpo ⊗ Vmps') + return A, M, FL, FR +end + +function benchmark_mpo(A, M, FL, FR) + return @tensor FL[4, 2, 1] * A[1, 3, 6] * M[2, 5, 3, 7] * conj(A[4, 5, 8]) * FR[6, 7, 8] +end + +function benchmark_mpo!(benchgroup, params::Dict) + haskey(benchgroup, "mpo") || addgroup!(benchgroup, "mpo") + bench = benchgroup["mpo"] + for kwargs in expand_kwargs(params) + benchmark_mpo!(bench; kwargs...) + end + return nothing +end +function benchmark_mpo!(bench; sigmas=nothing, T="Float64", I="Trivial", dims) + T_ = parse_type(T) + I_ = parse_type(I) + + Vs = generate_space.(I_, dims, sigmas) + init() = init_mpo_tensors(T_, Vs) + + bench[T, I, dims, sigmas] = @benchmarkable benchmark_mpo(A, M, FL, FR) setup = ((A, M, FL, FR) = $init()) + + return nothing +end + +if haskey(all_parameters, "mpo") + g = addgroup!(SUITE, "mpo") + for params in all_parameters["mpo"] + benchmark_mpo!(g, params) + end +end + +# pepo contraction +# ---------------- +function init_pepo_tensors(T, (Vpeps, Vpepo, Vphys, Venv)) + A = Tensor(randn, T, Vpeps ⊗ Vpeps ⊗ Vphys ⊗ Vpeps' ⊗ Vpeps') + P = Tensor(randn, T, Vpepo ⊗ Vpepo ⊗ Vphys ⊗ Vphys' ⊗ Vpepo' ⊗ Vpepo') + FL = Tensor(randn, T, Venv ⊗ Vpeps ⊗ Vpepo' ⊗ Vpeps' ⊗ Venv') + FD = Tensor(randn, T, Venv ⊗ Vpeps ⊗ Vpepo' ⊗ Vpeps' ⊗ Venv') + FR = Tensor(randn, T, Venv ⊗ Vpeps ⊗ Vpepo ⊗ Vpeps' ⊗ Venv') + FU = Tensor(randn, T, Venv ⊗ Vpeps ⊗ Vpepo ⊗ Vpeps' ⊗ Venv') + return A, P, FL, FD, FR, FU +end + +function benchmark_pepo(A, P, FL, FD, FR, FU) + return @tensor FL[18, 7, 4, 2, 1] * FU[1, 3, 6, 9, 10] * A[2, 17, 5, 3, 11] * + P[4, 16, 8, 5, 6, 12] * conj(A[7, 15, 8, 9, 13]) * + FR[10, 11, 12, 13, 14] * FD[14, 15, 16, 17, 18] +end + +function benchmark_pepo!(benchgroup, params::Dict) + haskey(benchgroup, "pepo") || addgroup!(benchgroup, "pepo") + bench = benchgroup["pepo"] + for kwargs in expand_kwargs(params) + benchmark_pepo!(bench; kwargs...) + end + return nothing +end +function benchmark_pepo!(bench; sigmas=nothing, T="Float64", I="Trivial", dims) + T_ = parse_type(T) + I_ = parse_type(I) + + Vs = generate_space.(I_, dims, sigmas) + init() = init_pepo_tensors(T_, Vs) + + bench[T, I, dims, sigmas] = @benchmarkable benchmark_pepo(A, P, FL, FD, FR, FU) setup = ((A, P, FL, FD, FR, FU) = $init()) + + return nothing +end + +if haskey(all_parameters, "pepo") + g = addgroup!(SUITE, "pepo") + for params in all_parameters["pepo"] + benchmark_pepo!(g, params) + end +end + +# mera contraction +# ---------------- +function init_mera_tensors(T, V) + u = Tensor(randn, T, V ⊗ V ⊗ V' ⊗ V') + w = Tensor(randn, T, V ⊗ V ⊗ V') + ρ = Tensor(randn, T, V ⊗ V ⊗ V ⊗ V' ⊗ V' ⊗ V') + h = Tensor(randn, T, V ⊗ V ⊗ V ⊗ V' ⊗ V' ⊗ V') + return u, w, ρ, h +end + +function benchmark_mera(u, w, ρ, h) + return @tensor (((((((h[9, 3, 4, 5, 1, 2] * u[1, 2, 7, 12]) * conj(u[3, 4, 11, 13])) * + (u[8, 5, 15, 6] * w[6, 7, 19])) * + (conj(u[8, 9, 17, 10]) * conj(w[10, 11, 22]))) * + ((w[12, 14, 20] * conj(w[13, 14, 23])) * ρ[18, 19, 20, 21, 22, 23])) * + w[16, 15, 18]) * conj(w[16, 17, 21])) +end + +function benchmark_mera!(benchgroup, params::Dict) + haskey(benchgroup, "mera") || addgroup!(benchgroup, "mera") + bench = benchgroup["mera"] + for kwargs in expand_kwargs(params) + benchmark_mera!(bench; kwargs...) + end + return nothing +end + +function benchmark_mera!(bench; sigmas=nothing, T="Float64", I="Trivial", dims) + T_ = parse_type(T) + I_ = parse_type(I) + + Vs = generate_space.(I_, dims, sigmas) + init() = init_mera_tensors(T_, Vs) + + bench[T, I, dims, sigmas] = @benchmarkable benchmark_mera(u, w, ρ, h) setup = ((u, w, ρ, h) = $init()) + + return nothing +end + +if haskey(all_parameters, "mera") + g = addgroup!(SUITE, "mera") + for params in all_parameters["mera"] + benchmark_mera!(g, params) + end +end + +end diff --git a/benchmark/TensorKitBenchmarks/tensornetworks/benchparams.toml b/benchmark/TensorKitBenchmarks/tensornetworks/benchparams.toml new file mode 100644 index 000000000..8d10eb109 --- /dev/null +++ b/benchmark/TensorKitBenchmarks/tensornetworks/benchparams.toml @@ -0,0 +1,62 @@ +# MPO +# --- +# dims = [mps, mpo, phys] + +[[mpo]] +T = ["Float64"] +I = "Trivial" +dims = [[10, 4, 3], [40, 4, 3], [160, 4, 3], [640, 4, 3], [2560, 4, 3], [100, 10, 10], [200, 10, 10], [300, 20, 20]] + +[[mpo]] +T = ["Float64"] +I = "Z2Irrep" +dims = [[10, 4, 4], [40, 4, 4], [160, 4, 4], [640, 4, 4], [2560, 4, 4], [100, 10, 10], [200, 10, 10], [300, 20, 20]] +sigmas = [0.5, 0.5, 0.5] + +[[mpo]] +T = ["Float64"] +I = "U1Irrep" +dims = [[40, 5, 3], [160, 5, 3], [640, 5, 3], [2560, 5, 3], [6120, 5, 3], [200, 20, 20], [400, 20, 20], [400, 40, 40]] +sigmas = [0.5, 0.5, 0.5] + +# PEPO +# ---- +# dims = [peps, pepo, phys, env] + +[[pepo]] +T = ["Float64"] +I = "Trivial" +dims = [[3, 2, 2, 50], [3, 3, 3, 100], [4, 2, 2, 50], [4, 3, 3, 100], [5, 2, 2, 50], [5, 2, 3, 100], [6, 2, 2, 50], [6, 3, 2, 100]] + +[[pepo]] +T = ["Float64"] +I = "Z2Irrep" +dims = [[4, 2, 2, 50], [4, 4, 4, 100], [5, 2, 2, 50], [5, 3, 4, 100], [6, 2, 2, 50], [6, 2, 4, 100], [8, 2, 2, 50], [8, 3, 2, 100]] +sigmas = [0.5, 0.5, 0.5, 0.5] + +[[pepo]] +T = ["Float64"] +I = "U1Irrep" +dims = [[4, 2, 2, 100], [4, 4, 4, 200], [6, 2, 2, 100], [6, 3, 4, 200], [8, 2, 2, 100], [8, 2, 4, 200], [10, 2, 2, 50], [10, 3, 2, 100]] +sigmas = [0.5, 0.5, 0.5, 0.5] + +# MERA +# ---- +# dims = mera + +[[mera]] +T = ["Float64"] +I = "Trivial" +dims = [2, 3, 4, 8, 12, 16] + +[[mera]] +T = ["Float64"] +I = "Z2Irrep" +dims = [2, 4, 8, 12, 16, 20] +sigmas = [0.5] + +[[mera]] +T = ["Float64"] +I = "U1Irrep" +dims = [4, 8, 12, 16, 22, 28] +sigmas = [0.5] diff --git a/benchmark/TensorKitBenchmarks/utils/BenchUtils.jl b/benchmark/TensorKitBenchmarks/utils/BenchUtils.jl new file mode 100644 index 000000000..cefb02c98 --- /dev/null +++ b/benchmark/TensorKitBenchmarks/utils/BenchUtils.jl @@ -0,0 +1,74 @@ +module BenchUtils + +export parse_type, expand_kwargs +export generate_space + +using TensorKit + +parse_type(x::String) = eval(Meta.parse(x)) + +function expand_kwargs(params::Dict) + const_params = NamedTuple(Symbol(key) => value + for (key, value) in params if !(value isa Vector)) + nonconst_keys = Tuple(Symbol(key) for (key, value) in params if value isa Vector) + nonconst_vals = (value for value in values(params) if value isa Vector) + return Iterators.map(Iterators.product(nonconst_vals...)) do expanded_vals + return merge(const_params, NamedTuple{nonconst_keys}(expanded_vals)) + end +end + +""" + generate_space(::Type{I}, D::Int; sigma::Real=1.0) + +Creates a (graded) vector space with sectortype `I` and total dimension `D`, where the distribution of charges is controlled through a spread parameter `sigma`. +""" +function generate_space(::Type{Trivial}, D::Int, sigma::Nothing=nothing) + return ComplexSpace(round(Int, D)) +end +function generate_space(::Type{Z2Irrep}, D::Int, sigma::Real=0.5) + D_even = ceil(Int, sigma * D) + D_odd = D - D_even + return Z2Space(0 => D_even, 1 => D_odd) +end +function generate_space(::Type{U1Irrep}, D::Int, sigma::Real=0.5) + # use ceil here to avoid getting stuck + normal_pdf = let D = D + x -> ceil(Int, D * exp(-0.5 * (x / sigma)^2) / (sigma * sqrt(2π))) + end + + sectors = U1Irrep[] + dims = Int[] + + for sector in values(U1Irrep) + round(sector.charge) == sector.charge || continue + d = normal_pdf(sector.charge) + push!(sectors, sector) + push!(dims, d) + D -= d + D < 1 && break + + if abs(sector.charge) > 20 + error() + end + end + + return U1Space((s => d for (s, d) in zip(sectors, dims))...) +end +function generate_space(::Type{SU2Irrep}, D::Int, sigma::Real=0.5) + poisson_pdf(x) = ceil(Int, D * exp(-sigma) * sigma^x / factorial(x + 1)) + + sectors = SU2Irrep[] + dims = Int[] + + for sector in values(SU2Irrep) + d = poisson_pdf(Int(sector.j * 2)) + push!(sectors, sector) + push!(dims, d) + D -= d * dim(sector) + D < 1 && break + end + + return SU2Space((s => d for (s, d) in zip(sectors, dims))...) +end + +end diff --git a/benchmark/benchmarks.jl b/benchmark/benchmarks.jl new file mode 100644 index 000000000..f119084d5 --- /dev/null +++ b/benchmark/benchmarks.jl @@ -0,0 +1,17 @@ +# Load benchmark code +include("TensorKitBenchmarks/TensorKitBenchmarks.jl") +const SUITE = TensorKitBenchmarks.SUITE + +# Populate benchmarks +# Detect if user supplied extra arguments to load only specific modules +# e.g. julia benchmarks.jl --modules=linalg,tensornetworks +modules_pattern = r"(?:--modules=)(\w+)" +arg_id = findfirst(contains(modules_pattern), ARGS) +if isnothing(arg_id) + TensorKitBenchmarks.loadall!() +else + modules = split(only(match(modules_pattern, ARGS[arg_id]).captures[1]), ",") + for m in modules + TensorKitBenchmarks.load!(m) + end +end diff --git a/benchmark/benchtools.jl b/benchmark/comparisons/benchtools.jl similarity index 100% rename from benchmark/benchtools.jl rename to benchmark/comparisons/benchtools.jl diff --git a/benchmark/benchtools.py b/benchmark/comparisons/benchtools.py similarity index 100% rename from benchmark/benchtools.py rename to benchmark/comparisons/benchtools.py diff --git a/benchmark/bonds.jl b/benchmark/comparisons/bonds.jl similarity index 100% rename from benchmark/bonds.jl rename to benchmark/comparisons/bonds.jl diff --git a/benchmark/bonds.py b/benchmark/comparisons/bonds.py similarity index 100% rename from benchmark/bonds.py rename to benchmark/comparisons/bonds.py diff --git a/benchmark/bonds.txt b/benchmark/comparisons/bonds.txt similarity index 100% rename from benchmark/bonds.txt rename to benchmark/comparisons/bonds.txt diff --git a/benchmark/itensors_timers.jl b/benchmark/comparisons/itensors_timers.jl similarity index 100% rename from benchmark/itensors_timers.jl rename to benchmark/comparisons/itensors_timers.jl diff --git a/benchmark/meraresults.pdf b/benchmark/comparisons/meraresults.pdf similarity index 100% rename from benchmark/meraresults.pdf rename to benchmark/comparisons/meraresults.pdf diff --git a/benchmark/mporesults.pdf b/benchmark/comparisons/mporesults.pdf similarity index 100% rename from benchmark/mporesults.pdf rename to benchmark/comparisons/mporesults.pdf diff --git a/benchmark/comparisons/params.toml b/benchmark/comparisons/params.toml new file mode 100644 index 000000000..e9761975d --- /dev/null +++ b/benchmark/comparisons/params.toml @@ -0,0 +1,85 @@ +# main structure of this file: +# +# [group.benchmark] headers separate benchmarks +# parameters here are expanded over all combinations +# +# [group.benchmark.sector] headers separate different tensor definitions +# parameters here are zipped + +[linalg.mul] + +T = ["Float64", "ComplexF64"] + +[[linalg.mul.spaces]] +I = "Trivial" +dims = [[2, 2, 2], [8, 8, 8], [32, 32, 32], [64, 64, 64], [128, 128, 128]] + +[[linalg.mul.spaces]] +I = "Z2Irrep" +dims = [[2, 2, 2], [8, 8, 8], [32, 32, 32], [64, 64, 64], [128, 128, 128]] +sigmas = [0.5, 0.5, 0.5] + + +[indexmanipulations.permute] + +T = ["Float64", "ComplexF64"] + +[[indexmanipulations.permute.spaces]] +I = "Trivial" +p = [[5, 4, 3], [2, 1]] +dims = [[[2, 2, 2], [2, 2]], [[3, 3, 3], [3, 3]]] + + +# TensorNetwork contractions +[tensornetworks.mpo] +T = ["ComplexF64"] + +[[tensornetworks.mpo.spaces]] +I = "Trivial" +dims = [[10, 4, 3], [40, 4, 3], [160, 4, 3], [640, 4,3], [2560, 4, 3], [100, 10, 10], [200, 10, 10], [300, 20, 20]] + +[[tensornetworks.mpo.spaces]] +I = "Z2Irrep" +dims = [[10, 4, 4], [40, 4, 4], [160, 4, 4], [640, 4, 4], [2560, 4, 4], [100, 10, 10], [200, 10, 10], [300, 20, 20]] +sigmas = [0.5, 0.5, 0.5] + +[[tensornetworks.mpo.spaces]] +I = "U1Irrep" +dims = [[40, 5, 3], [160, 5, 3], [640, 5, 3], [2560, 5, 3], [6120, 5, 3], [200, 20, 20], [400, 20, 20], [400, 40, 40]] +sigmas = [0.5, 0.5, 0.5] + +[tensornetworks.pepo] +T = ["ComplexF64"] + +[[tensornetworks.pepo.spaces]] +I = "Trivial" +dims = [[3, 2, 2, 50], [3, 3, 3, 100], [4, 2, 2, 50], [4, 3, 3, 100], [5, 2, 2, 50], [5, 2, 3, 100], [6, 2, 2, 50], [6, 3, 2, 100]] + +[[tensornetworks.pepo.spaces]] +I = "Z2Irrep" +dims = [[4, 2, 2, 50], [4, 4, 4, 100], [5, 2, 2, 50], [5, 3, 4, 100], [6, 2, 2, 50], [6, 2, 4, 100], [8, 2, 2, 50], [8, 3, 2, 100]] +sigmas = [0.5, 0.5, 0.5, 0.5] + +[[tensornetworks.pepo.spaces]] +I = "U1Irrep" +dims = [[4, 2, 2, 100], [4, 4, 4, 200], [6, 2, 2, 100], [6, 3, 4, 200], [8, 2, 2, 100], [8, 2, 4, 200], [10, 2, 2, 50], [10, 3, 2, 100]] +sigmas = [0.5, 0.5, 0.5, 0.5] + + +[tensornetworks.mera] + +T = ["ComplexF64"] + +[[tensornetworks.mera.spaces]] +I = "Trivial" +dims = [2, 3, 4, 8, 12, 16] + +[[tensornetworks.mera.spaces]] +I = "Z2Irrep" +dims = [2, 4, 8, 12, 16, 20] +sigmas = 0.5 + +[[tensornetworks.mera.spaces]] +I = "U1Irrep" +dims = [4, 8, 12, 16, 22, 28] +sigmas = 0.5 diff --git a/benchmark/peporesults.pdf b/benchmark/comparisons/peporesults.pdf similarity index 100% rename from benchmark/peporesults.pdf rename to benchmark/comparisons/peporesults.pdf diff --git a/benchmark/plotmera.jl b/benchmark/comparisons/plotmera.jl similarity index 100% rename from benchmark/plotmera.jl rename to benchmark/comparisons/plotmera.jl diff --git a/benchmark/plotmpo.jl b/benchmark/comparisons/plotmpo.jl similarity index 100% rename from benchmark/plotmpo.jl rename to benchmark/comparisons/plotmpo.jl diff --git a/benchmark/plotpepo.jl b/benchmark/comparisons/plotpepo.jl similarity index 100% rename from benchmark/plotpepo.jl rename to benchmark/comparisons/plotpepo.jl diff --git a/benchmark/results/itensors_mera_triv.txt b/benchmark/comparisons/results/itensors_mera_triv.txt similarity index 100% rename from benchmark/results/itensors_mera_triv.txt rename to benchmark/comparisons/results/itensors_mera_triv.txt diff --git a/benchmark/results/itensors_mera_triv_gc.txt b/benchmark/comparisons/results/itensors_mera_triv_gc.txt similarity index 100% rename from benchmark/results/itensors_mera_triv_gc.txt rename to benchmark/comparisons/results/itensors_mera_triv_gc.txt diff --git a/benchmark/results/itensors_mera_u1.txt b/benchmark/comparisons/results/itensors_mera_u1.txt similarity index 100% rename from benchmark/results/itensors_mera_u1.txt rename to benchmark/comparisons/results/itensors_mera_u1.txt diff --git a/benchmark/results/itensors_mera_u1_gc.txt b/benchmark/comparisons/results/itensors_mera_u1_gc.txt similarity index 100% rename from benchmark/results/itensors_mera_u1_gc.txt rename to benchmark/comparisons/results/itensors_mera_u1_gc.txt diff --git a/benchmark/results/itensors_mera_z2.txt b/benchmark/comparisons/results/itensors_mera_z2.txt similarity index 100% rename from benchmark/results/itensors_mera_z2.txt rename to benchmark/comparisons/results/itensors_mera_z2.txt diff --git a/benchmark/results/itensors_mera_z2_gc.txt b/benchmark/comparisons/results/itensors_mera_z2_gc.txt similarity index 100% rename from benchmark/results/itensors_mera_z2_gc.txt rename to benchmark/comparisons/results/itensors_mera_z2_gc.txt diff --git a/benchmark/results/itensors_mpo_triv.txt b/benchmark/comparisons/results/itensors_mpo_triv.txt similarity index 100% rename from benchmark/results/itensors_mpo_triv.txt rename to benchmark/comparisons/results/itensors_mpo_triv.txt diff --git a/benchmark/results/itensors_mpo_triv_gc.txt b/benchmark/comparisons/results/itensors_mpo_triv_gc.txt similarity index 100% rename from benchmark/results/itensors_mpo_triv_gc.txt rename to benchmark/comparisons/results/itensors_mpo_triv_gc.txt diff --git a/benchmark/results/itensors_mpo_u1.txt b/benchmark/comparisons/results/itensors_mpo_u1.txt similarity index 100% rename from benchmark/results/itensors_mpo_u1.txt rename to benchmark/comparisons/results/itensors_mpo_u1.txt diff --git a/benchmark/results/itensors_mpo_u1_gc.txt b/benchmark/comparisons/results/itensors_mpo_u1_gc.txt similarity index 100% rename from benchmark/results/itensors_mpo_u1_gc.txt rename to benchmark/comparisons/results/itensors_mpo_u1_gc.txt diff --git a/benchmark/results/itensors_mpo_z2.txt b/benchmark/comparisons/results/itensors_mpo_z2.txt similarity index 100% rename from benchmark/results/itensors_mpo_z2.txt rename to benchmark/comparisons/results/itensors_mpo_z2.txt diff --git a/benchmark/results/itensors_mpo_z2_gc.txt b/benchmark/comparisons/results/itensors_mpo_z2_gc.txt similarity index 100% rename from benchmark/results/itensors_mpo_z2_gc.txt rename to benchmark/comparisons/results/itensors_mpo_z2_gc.txt diff --git a/benchmark/results/itensors_pepo_triv.txt b/benchmark/comparisons/results/itensors_pepo_triv.txt similarity index 100% rename from benchmark/results/itensors_pepo_triv.txt rename to benchmark/comparisons/results/itensors_pepo_triv.txt diff --git a/benchmark/results/itensors_pepo_triv_gc.txt b/benchmark/comparisons/results/itensors_pepo_triv_gc.txt similarity index 100% rename from benchmark/results/itensors_pepo_triv_gc.txt rename to benchmark/comparisons/results/itensors_pepo_triv_gc.txt diff --git a/benchmark/results/itensors_pepo_u1.txt b/benchmark/comparisons/results/itensors_pepo_u1.txt similarity index 100% rename from benchmark/results/itensors_pepo_u1.txt rename to benchmark/comparisons/results/itensors_pepo_u1.txt diff --git a/benchmark/results/itensors_pepo_u1_gc.txt b/benchmark/comparisons/results/itensors_pepo_u1_gc.txt similarity index 100% rename from benchmark/results/itensors_pepo_u1_gc.txt rename to benchmark/comparisons/results/itensors_pepo_u1_gc.txt diff --git a/benchmark/results/itensors_pepo_z2.txt b/benchmark/comparisons/results/itensors_pepo_z2.txt similarity index 100% rename from benchmark/results/itensors_pepo_z2.txt rename to benchmark/comparisons/results/itensors_pepo_z2.txt diff --git a/benchmark/results/itensors_pepo_z2_gc.txt b/benchmark/comparisons/results/itensors_pepo_z2_gc.txt similarity index 100% rename from benchmark/results/itensors_pepo_z2_gc.txt rename to benchmark/comparisons/results/itensors_pepo_z2_gc.txt diff --git a/benchmark/results/tenpy_mera_triv.txt b/benchmark/comparisons/results/tenpy_mera_triv.txt similarity index 100% rename from benchmark/results/tenpy_mera_triv.txt rename to benchmark/comparisons/results/tenpy_mera_triv.txt diff --git a/benchmark/results/tenpy_mera_u1.txt b/benchmark/comparisons/results/tenpy_mera_u1.txt similarity index 100% rename from benchmark/results/tenpy_mera_u1.txt rename to benchmark/comparisons/results/tenpy_mera_u1.txt diff --git a/benchmark/results/tenpy_mera_z2.txt b/benchmark/comparisons/results/tenpy_mera_z2.txt similarity index 100% rename from benchmark/results/tenpy_mera_z2.txt rename to benchmark/comparisons/results/tenpy_mera_z2.txt diff --git a/benchmark/results/tenpy_mpo_triv.txt b/benchmark/comparisons/results/tenpy_mpo_triv.txt similarity index 100% rename from benchmark/results/tenpy_mpo_triv.txt rename to benchmark/comparisons/results/tenpy_mpo_triv.txt diff --git a/benchmark/results/tenpy_mpo_u1.txt b/benchmark/comparisons/results/tenpy_mpo_u1.txt similarity index 100% rename from benchmark/results/tenpy_mpo_u1.txt rename to benchmark/comparisons/results/tenpy_mpo_u1.txt diff --git a/benchmark/results/tenpy_mpo_z2.txt b/benchmark/comparisons/results/tenpy_mpo_z2.txt similarity index 100% rename from benchmark/results/tenpy_mpo_z2.txt rename to benchmark/comparisons/results/tenpy_mpo_z2.txt diff --git a/benchmark/results/tenpy_pepo_triv.txt b/benchmark/comparisons/results/tenpy_pepo_triv.txt similarity index 100% rename from benchmark/results/tenpy_pepo_triv.txt rename to benchmark/comparisons/results/tenpy_pepo_triv.txt diff --git a/benchmark/results/tenpy_pepo_u1.txt b/benchmark/comparisons/results/tenpy_pepo_u1.txt similarity index 100% rename from benchmark/results/tenpy_pepo_u1.txt rename to benchmark/comparisons/results/tenpy_pepo_u1.txt diff --git a/benchmark/results/tenpy_pepo_z2.txt b/benchmark/comparisons/results/tenpy_pepo_z2.txt similarity index 100% rename from benchmark/results/tenpy_pepo_z2.txt rename to benchmark/comparisons/results/tenpy_pepo_z2.txt diff --git a/benchmark/results/tensorkit_mera_triv.txt b/benchmark/comparisons/results/tensorkit_mera_triv.txt similarity index 100% rename from benchmark/results/tensorkit_mera_triv.txt rename to benchmark/comparisons/results/tensorkit_mera_triv.txt diff --git a/benchmark/results/tensorkit_mera_triv_gc.txt b/benchmark/comparisons/results/tensorkit_mera_triv_gc.txt similarity index 100% rename from benchmark/results/tensorkit_mera_triv_gc.txt rename to benchmark/comparisons/results/tensorkit_mera_triv_gc.txt diff --git a/benchmark/results/tensorkit_mera_u1.txt b/benchmark/comparisons/results/tensorkit_mera_u1.txt similarity index 100% rename from benchmark/results/tensorkit_mera_u1.txt rename to benchmark/comparisons/results/tensorkit_mera_u1.txt diff --git a/benchmark/results/tensorkit_mera_u1_gc.txt b/benchmark/comparisons/results/tensorkit_mera_u1_gc.txt similarity index 100% rename from benchmark/results/tensorkit_mera_u1_gc.txt rename to benchmark/comparisons/results/tensorkit_mera_u1_gc.txt diff --git a/benchmark/results/tensorkit_mera_z2.txt b/benchmark/comparisons/results/tensorkit_mera_z2.txt similarity index 100% rename from benchmark/results/tensorkit_mera_z2.txt rename to benchmark/comparisons/results/tensorkit_mera_z2.txt diff --git a/benchmark/results/tensorkit_mera_z2_gc.txt b/benchmark/comparisons/results/tensorkit_mera_z2_gc.txt similarity index 100% rename from benchmark/results/tensorkit_mera_z2_gc.txt rename to benchmark/comparisons/results/tensorkit_mera_z2_gc.txt diff --git a/benchmark/results/tensorkit_mpo_triv.txt b/benchmark/comparisons/results/tensorkit_mpo_triv.txt similarity index 100% rename from benchmark/results/tensorkit_mpo_triv.txt rename to benchmark/comparisons/results/tensorkit_mpo_triv.txt diff --git a/benchmark/results/tensorkit_mpo_triv_gc.txt b/benchmark/comparisons/results/tensorkit_mpo_triv_gc.txt similarity index 100% rename from benchmark/results/tensorkit_mpo_triv_gc.txt rename to benchmark/comparisons/results/tensorkit_mpo_triv_gc.txt diff --git a/benchmark/results/tensorkit_mpo_u1.txt b/benchmark/comparisons/results/tensorkit_mpo_u1.txt similarity index 100% rename from benchmark/results/tensorkit_mpo_u1.txt rename to benchmark/comparisons/results/tensorkit_mpo_u1.txt diff --git a/benchmark/results/tensorkit_mpo_u1_gc.txt b/benchmark/comparisons/results/tensorkit_mpo_u1_gc.txt similarity index 100% rename from benchmark/results/tensorkit_mpo_u1_gc.txt rename to benchmark/comparisons/results/tensorkit_mpo_u1_gc.txt diff --git a/benchmark/results/tensorkit_mpo_z2.txt b/benchmark/comparisons/results/tensorkit_mpo_z2.txt similarity index 100% rename from benchmark/results/tensorkit_mpo_z2.txt rename to benchmark/comparisons/results/tensorkit_mpo_z2.txt diff --git a/benchmark/results/tensorkit_mpo_z2_gc.txt b/benchmark/comparisons/results/tensorkit_mpo_z2_gc.txt similarity index 100% rename from benchmark/results/tensorkit_mpo_z2_gc.txt rename to benchmark/comparisons/results/tensorkit_mpo_z2_gc.txt diff --git a/benchmark/results/tensorkit_pepo_triv.txt b/benchmark/comparisons/results/tensorkit_pepo_triv.txt similarity index 100% rename from benchmark/results/tensorkit_pepo_triv.txt rename to benchmark/comparisons/results/tensorkit_pepo_triv.txt diff --git a/benchmark/results/tensorkit_pepo_triv_gc.txt b/benchmark/comparisons/results/tensorkit_pepo_triv_gc.txt similarity index 100% rename from benchmark/results/tensorkit_pepo_triv_gc.txt rename to benchmark/comparisons/results/tensorkit_pepo_triv_gc.txt diff --git a/benchmark/results/tensorkit_pepo_u1.txt b/benchmark/comparisons/results/tensorkit_pepo_u1.txt similarity index 100% rename from benchmark/results/tensorkit_pepo_u1.txt rename to benchmark/comparisons/results/tensorkit_pepo_u1.txt diff --git a/benchmark/results/tensorkit_pepo_u1_gc.txt b/benchmark/comparisons/results/tensorkit_pepo_u1_gc.txt similarity index 100% rename from benchmark/results/tensorkit_pepo_u1_gc.txt rename to benchmark/comparisons/results/tensorkit_pepo_u1_gc.txt diff --git a/benchmark/results/tensorkit_pepo_z2.txt b/benchmark/comparisons/results/tensorkit_pepo_z2.txt similarity index 100% rename from benchmark/results/tensorkit_pepo_z2.txt rename to benchmark/comparisons/results/tensorkit_pepo_z2.txt diff --git a/benchmark/results/tensorkit_pepo_z2_gc.txt b/benchmark/comparisons/results/tensorkit_pepo_z2_gc.txt similarity index 100% rename from benchmark/results/tensorkit_pepo_z2_gc.txt rename to benchmark/comparisons/results/tensorkit_pepo_z2_gc.txt diff --git a/benchmark/results/tensornetwork_mera_triv.txt b/benchmark/comparisons/results/tensornetwork_mera_triv.txt similarity index 100% rename from benchmark/results/tensornetwork_mera_triv.txt rename to benchmark/comparisons/results/tensornetwork_mera_triv.txt diff --git a/benchmark/results/tensornetwork_mera_u1.txt b/benchmark/comparisons/results/tensornetwork_mera_u1.txt similarity index 100% rename from benchmark/results/tensornetwork_mera_u1.txt rename to benchmark/comparisons/results/tensornetwork_mera_u1.txt diff --git a/benchmark/results/tensornetwork_mera_z2.txt b/benchmark/comparisons/results/tensornetwork_mera_z2.txt similarity index 100% rename from benchmark/results/tensornetwork_mera_z2.txt rename to benchmark/comparisons/results/tensornetwork_mera_z2.txt diff --git a/benchmark/results/tensornetwork_mpo_triv.txt b/benchmark/comparisons/results/tensornetwork_mpo_triv.txt similarity index 100% rename from benchmark/results/tensornetwork_mpo_triv.txt rename to benchmark/comparisons/results/tensornetwork_mpo_triv.txt diff --git a/benchmark/results/tensornetwork_mpo_u1.txt b/benchmark/comparisons/results/tensornetwork_mpo_u1.txt similarity index 100% rename from benchmark/results/tensornetwork_mpo_u1.txt rename to benchmark/comparisons/results/tensornetwork_mpo_u1.txt diff --git a/benchmark/results/tensornetwork_mpo_z2.txt b/benchmark/comparisons/results/tensornetwork_mpo_z2.txt similarity index 100% rename from benchmark/results/tensornetwork_mpo_z2.txt rename to benchmark/comparisons/results/tensornetwork_mpo_z2.txt diff --git a/benchmark/results/tensornetwork_pepo_triv.txt b/benchmark/comparisons/results/tensornetwork_pepo_triv.txt similarity index 100% rename from benchmark/results/tensornetwork_pepo_triv.txt rename to benchmark/comparisons/results/tensornetwork_pepo_triv.txt diff --git a/benchmark/results/tensornetwork_pepo_u1.txt b/benchmark/comparisons/results/tensornetwork_pepo_u1.txt similarity index 100% rename from benchmark/results/tensornetwork_pepo_u1.txt rename to benchmark/comparisons/results/tensornetwork_pepo_u1.txt diff --git a/benchmark/results/tensornetwork_pepo_z2.txt b/benchmark/comparisons/results/tensornetwork_pepo_z2.txt similarity index 100% rename from benchmark/results/tensornetwork_pepo_z2.txt rename to benchmark/comparisons/results/tensornetwork_pepo_z2.txt diff --git a/benchmark/run_itensors.jl b/benchmark/comparisons/run_itensors.jl similarity index 100% rename from benchmark/run_itensors.jl rename to benchmark/comparisons/run_itensors.jl diff --git a/benchmark/run_tenpy.py b/benchmark/comparisons/run_tenpy.py similarity index 100% rename from benchmark/run_tenpy.py rename to benchmark/comparisons/run_tenpy.py diff --git a/benchmark/run_tensorkit.jl b/benchmark/comparisons/run_tensorkit.jl similarity index 100% rename from benchmark/run_tensorkit.jl rename to benchmark/comparisons/run_tensorkit.jl diff --git a/benchmark/run_tensornetwork.py b/benchmark/comparisons/run_tensornetwork.py similarity index 100% rename from benchmark/run_tensornetwork.py rename to benchmark/comparisons/run_tensornetwork.py diff --git a/benchmark/script.sh b/benchmark/comparisons/script.sh similarity index 100% rename from benchmark/script.sh rename to benchmark/comparisons/script.sh diff --git a/benchmark/tenpy_timers.py b/benchmark/comparisons/tenpy_timers.py similarity index 100% rename from benchmark/tenpy_timers.py rename to benchmark/comparisons/tenpy_timers.py diff --git a/benchmark/tensorkit_timers.jl b/benchmark/comparisons/tensorkit_timers.jl similarity index 100% rename from benchmark/tensorkit_timers.jl rename to benchmark/comparisons/tensorkit_timers.jl diff --git a/benchmark/tensornetwork_timers.py b/benchmark/comparisons/tensornetwork_timers.py similarity index 100% rename from benchmark/tensornetwork_timers.py rename to benchmark/comparisons/tensornetwork_timers.py