|
| 1 | +""" |
| 2 | +TimerOutputComparisons |
| 3 | +====================== |
| 4 | +
|
| 5 | +Provides some helper functions to save/load TimerOutput objects, and plot comparisons of |
| 6 | +them. This may be useful to compare performance with different settings, or between |
| 7 | +different versions of some code. |
| 8 | +""" |
| 9 | +module TimerOutputComparisons |
| 10 | + |
| 11 | +export save_timer, load_timer, compare_timers |
| 12 | +# Workaround for failures when JLD is not loaded in Main, see |
| 13 | +# https://github.com/JuliaIO/JLD.jl/issues/252 |
| 14 | +export JLD |
| 15 | + |
| 16 | +using GLMakie |
| 17 | +using JLD |
| 18 | +using TimerOutputs |
| 19 | + |
| 20 | +""" |
| 21 | + save_timer(filename::AbstractString, to::TimerOutput, |
| 22 | + timer_name::AbstractString="to") |
| 23 | +
|
| 24 | +Save `to` to a JLD file called `filename`, in a variable called `timer_name`. `filename` |
| 25 | +should end with ".jld". |
| 26 | +""" |
| 27 | +function save_timer(filename::AbstractString, to::TimerOutput, |
| 28 | + timer_name::AbstractString="to") |
| 29 | + if splitext(filename)[2] != ".jld" |
| 30 | + error("`filename` should end in \".jld\" so that JLD format is used. Otherwise " |
| 31 | + * "a TimerOutput might not be writable and re-loadable.") |
| 32 | + end |
| 33 | + JLD.save(filename, timer_name, to) |
| 34 | +end |
| 35 | + |
| 36 | +""" |
| 37 | + load_timer(filename::AbstractString, |
| 38 | + timer_name::AbstractString="to")::TimerOutput |
| 39 | +
|
| 40 | +Load a TimerOutput called `timer_name` from a JLD file called `filename`. |
| 41 | +""" |
| 42 | +function load_timer(filename::AbstractString, |
| 43 | + timer_name::AbstractString="to")::TimerOutput |
| 44 | + if splitext(filename)[2] != ".jld" |
| 45 | + error("Expected `filename` to end in \".jld\" so that JLD format is used.") |
| 46 | + end |
| 47 | + to = JLD.load(filename, timer_name) |
| 48 | + return to |
| 49 | +end |
| 50 | + |
| 51 | +const possible_includes = (:ncalls, :time, :allocs) |
| 52 | + |
| 53 | +""" |
| 54 | + compare_timers(timers::Union{AbstractString,Tuple{<:AbstractString,<:AbstractString},Tuple{TimerOutput,<:AbstractString}}...; |
| 55 | + flatten=false, save_as=nothing, include=$possible_includes) |
| 56 | +
|
| 57 | +Make a plot comparing `timers`. For `t` in `timers`: |
| 58 | +* if `t` is an AbstractString, load a TimerOutput from the file named `t` using |
| 59 | + `load_timer()`, and label it `t`. |
| 60 | +* if `t` is a `Tuple{<:AbstractString,<:AbstractString}`, load a TimerOutput called `t[2]` |
| 61 | + from the file named `t[1]` using `load_timer()`, and label it `t[1] * ":" * t[2]`. |
| 62 | +* if `t` is a `Tuple{TimerOutput,<:AbstractString}`, use the TimerOutput `t[1]` and label |
| 63 | + it `t[2]`. |
| 64 | +
|
| 65 | +We assume that the TimerOutput objects in `timers` contain (mostly) the same timers, |
| 66 | +otherwise this comparison will not make much sense. |
| 67 | +
|
| 68 | +If `flatten=true`, the TimerOutput objects are flattened with `TimerOutputs.flatten()`. |
| 69 | +
|
| 70 | +If a file-name is passed to `save_as` the plots are saved instead of being displayed |
| 71 | +interactively. For example, `save_as="foo.png"` would result in the plots being saved as |
| 72 | +"foo_ncalls.png", "foo_time.png", and "foo_allocs.png". |
| 73 | +
|
| 74 | +To plot only one or two quantities, pass one or more of `:ncalls`, `:time` and `:allocs` |
| 75 | +to `include`, for example `include=:time`. |
| 76 | +""" |
| 77 | +compare_timers |
| 78 | + |
| 79 | +function compare_timers(timers::Union{AbstractString,<:Tuple{AbstractString,AbstractString},<:Tuple{TimerOutput,AbstractString}}...; kwargs...) |
| 80 | + function get_timer(t)::Tuple{TimerOutput,String} |
| 81 | + if t isa Tuple{TimerOutput,<:AbstractString} |
| 82 | + return (t[1], String(t[2])) |
| 83 | + elseif t isa AbstractString |
| 84 | + return (load_timer(t), splitext(String(t))[1]) |
| 85 | + elseif t isa Tuple{<:AbstractString,<:AbstractString} |
| 86 | + return (load_timer(t...), splitext(String(t[1]))[1] * ":" * String(t[2])) |
| 87 | + else |
| 88 | + error("Unsupported type $(typeof(t)) for t=$t.") |
| 89 | + end |
| 90 | + end |
| 91 | + return compare_timers(Tuple(get_timer(t) for t ∈ timers)...; kwargs...) |
| 92 | +end |
| 93 | +function compare_timers(timers::Tuple{TimerOutput,String}...; |
| 94 | + flatten=false, save_as=nothing, include=possible_includes) |
| 95 | + |
| 96 | + if isa(include, Symbol) |
| 97 | + include = (include,) |
| 98 | + end |
| 99 | + for i ∈ include |
| 100 | + if i ∉ possible_includes |
| 101 | + error("'$i' is not a valid entry in include. Possible values are " |
| 102 | + * "$possible_includes.") |
| 103 | + end |
| 104 | + end |
| 105 | + if flatten |
| 106 | + timers = map(t->(TimerOutputs.flatten(t[1]), t[2]), timers) |
| 107 | + end |
| 108 | + |
| 109 | + to_list = [t[1] for t ∈ timers] |
| 110 | + x_values = [t[2] for t ∈ timers] |
| 111 | + |
| 112 | + # Get names of all timers to plot. |
| 113 | + timer_names = Vector{String}[] |
| 114 | + function extract_names!(t, name) |
| 115 | + if !isempty(name) && name ∉ timer_names |
| 116 | + push!(timer_names, name) |
| 117 | + end |
| 118 | + inner_timers = t.inner_timers |
| 119 | + if !isempty(inner_timers) |
| 120 | + for k ∈ keys(inner_timers) |
| 121 | + new_name = copy(name) |
| 122 | + push!(new_name, k) |
| 123 | + extract_names!(t[k], new_name) |
| 124 | + end |
| 125 | + end |
| 126 | + return nothing |
| 127 | + end |
| 128 | + for t ∈ to_list |
| 129 | + extract_names!(t, String[]) |
| 130 | + end |
| 131 | + |
| 132 | + xticks = (1:length(x_values), x_values) |
| 133 | + if :ncalls ∈ include |
| 134 | + fig_ncalls = Figure() |
| 135 | + ax_ncalls = Axis(fig_ncalls[1,1]; xticks=xticks, ylabel="ncalls") |
| 136 | + else |
| 137 | + fig_ncalls = nothing |
| 138 | + ax_ncalls = nothing |
| 139 | + end |
| 140 | + if :time ∈ include |
| 141 | + fig_time = Figure() |
| 142 | + ax_time = Axis(fig_time[1,1]; xticks=xticks, ylabel="time (ms)") |
| 143 | + else |
| 144 | + fig_time = nothing |
| 145 | + ax_time = nothing |
| 146 | + end |
| 147 | + if :allocs ∈ include |
| 148 | + fig_allocs = Figure() |
| 149 | + ax_allocs = Axis(fig_allocs[1,1]; xticks=xticks, ylabel="allocated (kB)") |
| 150 | + else |
| 151 | + fig_allocs = nothing |
| 152 | + ax_allocs = nothing |
| 153 | + end |
| 154 | + |
| 155 | + for name ∈ timer_names |
| 156 | + plot_single_timer!(ax_ncalls, ax_time, ax_allocs, to_list, name, xticks) |
| 157 | + end |
| 158 | + |
| 159 | + for (fig, ax) in zip((fig_ncalls, fig_time, fig_allocs), (ax_ncalls, ax_time, ax_allocs)) |
| 160 | + if fig !== nothing |
| 161 | + # Ensure the first row width is 3/4 of the column width so that the plot does not get |
| 162 | + # squashed by the legend |
| 163 | + rowsize!(fig.layout, 1, Aspect(1, 3/4)) |
| 164 | + |
| 165 | + Legend(fig[2,1], ax; tellwidth=false, tellheight=true) |
| 166 | + |
| 167 | + resize_to_layout!(fig) |
| 168 | + end |
| 169 | + end |
| 170 | + |
| 171 | + if save_as === nothing |
| 172 | + backend = Makie.current_backend() |
| 173 | + for fig in (fig_ncalls, fig_time, fig_allocs) |
| 174 | + if fig !== nothing |
| 175 | + DataInspector(fig) |
| 176 | + display(backend.Screen(), fig) |
| 177 | + end |
| 178 | + end |
| 179 | + else |
| 180 | + prefix, suffix = splitext(save_as) |
| 181 | + if fig_ncalls !== nothing |
| 182 | + save(prefix * "_ncalls" * suffix, fig_ncalls) |
| 183 | + end |
| 184 | + if fig_time !== nothing |
| 185 | + save(prefix * "_time" * suffix, fig_time) |
| 186 | + end |
| 187 | + if fig_allocs !== nothing |
| 188 | + save(prefix * "_allocs" * suffix, fig_allocs) |
| 189 | + end |
| 190 | + end |
| 191 | + |
| 192 | + return fig_ncalls, fig_time, fig_allocs |
| 193 | +end |
| 194 | + |
| 195 | +function get_single_timer(to::TimerOutput, name::Vector{String}) |
| 196 | + for n ∈ name |
| 197 | + if n ∈ keys(to.inner_timers) |
| 198 | + to = to[n] |
| 199 | + else |
| 200 | + return nothing |
| 201 | + end |
| 202 | + end |
| 203 | + return to |
| 204 | +end |
| 205 | + |
| 206 | +function plot_single_timer!(ax_ncalls, ax_time, ax_allocs, to_list, name::Vector{String}, |
| 207 | + xticks) |
| 208 | + this_timer_list = [get_single_timer(to, name) for to ∈ to_list] |
| 209 | + label = join(name, ":") |
| 210 | + |
| 211 | + xtick_values = xticks[2] |
| 212 | + if ax_ncalls !== nothing |
| 213 | + ncalls_values = [t === nothing ? NaN : TimerOutputs.ncalls(t) for t ∈ this_timer_list] |
| 214 | + lines!(ax_ncalls, ncalls_values; |
| 215 | + label, |
| 216 | + inspector_label=(self,i,p) -> "$(self.label[])\n$(xtick_values[round(Int64, p[1])]): ncalls=$(ncalls_values[round(Int64, p[1])])") |
| 217 | + end |
| 218 | + |
| 219 | + # Convert times from ns to ms. |
| 220 | + if ax_time !== nothing |
| 221 | + time_values = [t === nothing ? NaN : TimerOutputs.time(t) * 1.0e-6 for t ∈ this_timer_list] |
| 222 | + lines!(ax_time, time_values; |
| 223 | + label, |
| 224 | + inspector_label=(self,i,p) -> "$(self.label[])\n$(xtick_values[round(Int64, p[1])]): time=$(time_values[round(Int64, p[1])]) ms") |
| 225 | + end |
| 226 | + |
| 227 | + if ax_allocs !== nothing |
| 228 | + allocs_values = [t === nothing ? NaN : TimerOutputs.allocated(t) / 1024 for t ∈ this_timer_list] |
| 229 | + lines!(ax_allocs, allocs_values; |
| 230 | + label, |
| 231 | + inspector_label=(self,i,p) -> "$(self.label[])\n$(xtick_values[round(Int64, p[1])]): allocs=$(allocs_values[round(Int64, p[1])]) kB") |
| 232 | + end |
| 233 | + |
| 234 | + return nothing |
| 235 | +end |
| 236 | + |
| 237 | +end |
0 commit comments