|
| 1 | +""" |
| 2 | +TimerOutputComparisons |
| 3 | +====================== |
| 4 | +
|
| 5 | +Provides some helper functions to save/load TimerOutput objects, and plot comparisons of |
| 6 | +them. This may be useful to compare performance with different settings, or between |
| 7 | +different versions of some code. |
| 8 | +""" |
1 | 9 | module TimerOutputComparisons |
2 | 10 |
|
3 | | -# Write your package code here. |
| 11 | +export save_timer, load_timer, compare_timers |
| 12 | +# Workaround for failures when JLD is not loaded in Main, see |
| 13 | +# https://github.com/JuliaIO/JLD.jl/issues/252 |
| 14 | +export JLD |
| 15 | + |
| 16 | +using GLMakie |
| 17 | +using JLD |
| 18 | +using TimerOutputs |
| 19 | + |
| 20 | +""" |
| 21 | + save_timer(filename::AbstractString, to::TimerOutput, |
| 22 | + timer_name::AbstractString="to") |
| 23 | +
|
| 24 | +Save `to` to a JLD file called `filename`, in a variable called `timer_name`. `filename` |
| 25 | +should end with ".jld". |
| 26 | +""" |
| 27 | +function save_timer(filename::AbstractString, to::TimerOutput, |
| 28 | + timer_name::AbstractString="to") |
| 29 | + if splitext(filename)[2] != ".jld" |
| 30 | + error("`filename` should end in \".jld\" so that JLD format is used. Otherwise " |
| 31 | + * "a TimerOutput might not be writable and re-loadable.") |
| 32 | + end |
| 33 | + JLD.save(filename, timer_name, to) |
| 34 | +end |
| 35 | + |
| 36 | +""" |
| 37 | + load_timer(filename::AbstractString, |
| 38 | + timer_name::AbstractString="to")::TimerOutput |
| 39 | +
|
| 40 | +Load a TimerOutput called `timer_name` from a JLD file called `filename`. |
| 41 | +""" |
| 42 | +function load_timer(filename::AbstractString, |
| 43 | + timer_name::AbstractString="to")::TimerOutput |
| 44 | + if splitext(filename)[2] != ".jld" |
| 45 | + error("Expected `filename` to end in \".jld\" so that JLD format is used.") |
| 46 | + end |
| 47 | + to = JLD.load(filename, timer_name) |
| 48 | + return to |
| 49 | +end |
| 50 | + |
| 51 | +const possible_use_data = (:ncalls, :time, :allocs) |
| 52 | + |
| 53 | +""" |
| 54 | + compare_timers(timers::Union{AbstractString,Tuple{<:AbstractString,<:AbstractString},Tuple{TimerOutput,<:AbstractString}}...; |
| 55 | + flatten::Bool=false, averages::Bool=false, |
| 56 | + save_as::Union{AbstractString,Nothing}=nothing, |
| 57 | + use_data=possible_use_data, legend::Bool=true, root=nothing) |
| 58 | +
|
| 59 | +Make a plot comparing `timers`. For `t` in `timers`: |
| 60 | +* if `t` is an AbstractString, load a TimerOutput from the file named `t` using |
| 61 | + `load_timer()`, and label it `t`. |
| 62 | +* if `t` is a `Tuple{<:AbstractString,<:AbstractString}`, load a TimerOutput called `t[2]` |
| 63 | + from the file named `t[1]` using `load_timer()`, and label it `t[1] * ":" * t[2]`. |
| 64 | +* if `t` is a `Tuple{TimerOutput,<:AbstractString}`, use the TimerOutput `t[1]` and label |
| 65 | + it `t[2]`. |
| 66 | +
|
| 67 | +We assume that the TimerOutput objects in `timers` contain (mostly) the same timers, |
| 68 | +otherwise this comparison will not make much sense. |
| 69 | +
|
| 70 | +If `flatten=true`, the TimerOutput objects are flattened with `TimerOutputs.flatten()`. |
| 71 | +
|
| 72 | +If `averages=true`, then the time per call and allocs per call will be plotted instead of |
| 73 | +the total time and total allocs for each timer. |
| 74 | +
|
| 75 | +If a file-name is passed to `save_as` the plots are saved instead of being displayed |
| 76 | +interactively. For example, `save_as="foo.png"` would result in the plots being saved as |
| 77 | +"foo_ncalls.png", "foo_time.png", and "foo_allocs.png". |
| 78 | +
|
| 79 | +To plot only one or two quantities, pass one or more of `:ncalls`, `:time` and `:allocs` |
| 80 | +to `use_data`, for example `use_data=:time`. |
| 81 | +
|
| 82 | +To remove the legend, pass `legend=false`. |
| 83 | +
|
| 84 | +If you only want to use_data some (possibly nested) sub-timer, pass its name as a String, |
| 85 | +Tuple or Vector to `root`. For example if the top-level TimerOutput is `to`, passing |
| 86 | +`root=("foo", "bar")` will use_data only the timers in `to["foo"]["bar"]`. |
| 87 | +""" |
| 88 | +compare_timers |
| 89 | + |
| 90 | +function compare_timers(timers::Union{AbstractString,<:Tuple{AbstractString,AbstractString},<:Tuple{TimerOutput,AbstractString}}...; kwargs...) |
| 91 | + function get_timer(t)::Tuple{TimerOutput,String} |
| 92 | + if t isa Tuple{TimerOutput,<:AbstractString} |
| 93 | + return (t[1], String(t[2])) |
| 94 | + elseif t isa AbstractString |
| 95 | + return (load_timer(t), splitext(String(t))[1]) |
| 96 | + elseif t isa Tuple{<:AbstractString,<:AbstractString} |
| 97 | + return (load_timer(t...), splitext(String(t[1]))[1] * ":" * String(t[2])) |
| 98 | + else |
| 99 | + error("Unsupported type $(typeof(t)) for t=$t.") |
| 100 | + end |
| 101 | + end |
| 102 | + return compare_timers(Tuple(get_timer(t) for t ∈ timers)...; kwargs...) |
| 103 | +end |
| 104 | +function compare_timers(timers::Tuple{TimerOutput,String}...; |
| 105 | + flatten::Bool=false, averages::Bool=false, |
| 106 | + save_as::Union{AbstractString,Nothing}=nothing, |
| 107 | + use_data=possible_use_data, legend::Bool=true, root=nothing) |
| 108 | + |
| 109 | + if isa(use_data, Symbol) |
| 110 | + use_data = (use_data,) |
| 111 | + end |
| 112 | + for i ∈ use_data |
| 113 | + if i ∉ possible_use_data |
| 114 | + error("'$i' is not a valid entry in use_data. Possible values are " |
| 115 | + * "$possible_use_data.") |
| 116 | + end |
| 117 | + end |
| 118 | + |
| 119 | + to_list = [t[1] for t ∈ timers] |
| 120 | + x_values = [t[2] for t ∈ timers] |
| 121 | + |
| 122 | + if flatten |
| 123 | + to_list = [TimerOutputs.flatten(t) for t ∈ to_list] |
| 124 | + end |
| 125 | + if root !== nothing |
| 126 | + if isa(root, AbstractString) |
| 127 | + root = [root] |
| 128 | + end |
| 129 | + for r ∈ root |
| 130 | + to_list = [t[r] for t ∈ to_list] |
| 131 | + end |
| 132 | + end |
| 133 | + |
| 134 | + # Get names of all timers to plot. |
| 135 | + timer_names = Vector{String}[] |
| 136 | + function extract_names!(t, name) |
| 137 | + if !isempty(name) && name ∉ timer_names |
| 138 | + push!(timer_names, name) |
| 139 | + end |
| 140 | + inner_timers = t.inner_timers |
| 141 | + if !isempty(inner_timers) |
| 142 | + for k ∈ keys(inner_timers) |
| 143 | + new_name = copy(name) |
| 144 | + push!(new_name, k) |
| 145 | + extract_names!(t[k], new_name) |
| 146 | + end |
| 147 | + end |
| 148 | + return nothing |
| 149 | + end |
| 150 | + for t ∈ to_list |
| 151 | + extract_names!(t, String[]) |
| 152 | + end |
| 153 | + |
| 154 | + xticks = (1:length(x_values), x_values) |
| 155 | + if :ncalls ∈ use_data |
| 156 | + fig_ncalls = Figure() |
| 157 | + ax_ncalls = Axis(fig_ncalls[1,1]; xticks=xticks, ylabel="ncalls") |
| 158 | + else |
| 159 | + fig_ncalls = nothing |
| 160 | + ax_ncalls = nothing |
| 161 | + end |
| 162 | + if :time ∈ use_data |
| 163 | + fig_time = Figure() |
| 164 | + ax_time = Axis(fig_time[1,1]; xticks=xticks, ylabel="time (ms)") |
| 165 | + else |
| 166 | + fig_time = nothing |
| 167 | + ax_time = nothing |
| 168 | + end |
| 169 | + if :allocs ∈ use_data |
| 170 | + fig_allocs = Figure() |
| 171 | + ax_allocs = Axis(fig_allocs[1,1]; xticks=xticks, ylabel="allocated (kB)") |
| 172 | + else |
| 173 | + fig_allocs = nothing |
| 174 | + ax_allocs = nothing |
| 175 | + end |
| 176 | + |
| 177 | + if root !== nothing |
| 178 | + plot_single_timer!(ax_ncalls, ax_time, ax_allocs, to_list, root[end:end], xticks, |
| 179 | + averages, true) |
| 180 | + end |
| 181 | + for name ∈ timer_names |
| 182 | + plot_single_timer!(ax_ncalls, ax_time, ax_allocs, to_list, name, xticks, averages) |
| 183 | + end |
| 184 | + |
| 185 | + if legend |
| 186 | + for (fig, ax) in zip((fig_ncalls, fig_time, fig_allocs), (ax_ncalls, ax_time, ax_allocs)) |
| 187 | + if fig !== nothing |
| 188 | + # Ensure the first row width is 3/4 of the column width so that the plot does not get |
| 189 | + # squashed by the legend |
| 190 | + rowsize!(fig.layout, 1, Aspect(1, 3/4)) |
| 191 | + |
| 192 | + Legend(fig[2,1], ax; tellwidth=false, tellheight=true) |
| 193 | + |
| 194 | + resize_to_layout!(fig) |
| 195 | + end |
| 196 | + end |
| 197 | + end |
| 198 | + |
| 199 | + if save_as === nothing |
| 200 | + backend = Makie.current_backend() |
| 201 | + for fig in (fig_ncalls, fig_time, fig_allocs) |
| 202 | + if fig !== nothing |
| 203 | + DataInspector(fig) |
| 204 | + display(backend.Screen(), fig) |
| 205 | + end |
| 206 | + end |
| 207 | + else |
| 208 | + prefix, suffix = splitext(save_as) |
| 209 | + if fig_ncalls !== nothing |
| 210 | + save(prefix * "_ncalls" * suffix, fig_ncalls) |
| 211 | + end |
| 212 | + if fig_time !== nothing |
| 213 | + save(prefix * "_time" * suffix, fig_time) |
| 214 | + end |
| 215 | + if fig_allocs !== nothing |
| 216 | + save(prefix * "_allocs" * suffix, fig_allocs) |
| 217 | + end |
| 218 | + end |
| 219 | + |
| 220 | + return fig_ncalls, fig_time, fig_allocs |
| 221 | +end |
| 222 | + |
| 223 | +function get_single_timer(to::TimerOutput, name::Vector{String}) |
| 224 | + for n ∈ name |
| 225 | + if n ∈ keys(to.inner_timers) |
| 226 | + to = to[n] |
| 227 | + else |
| 228 | + return nothing |
| 229 | + end |
| 230 | + end |
| 231 | + return to |
| 232 | +end |
| 233 | + |
| 234 | +function plot_single_timer!(ax_ncalls, ax_time, ax_allocs, to_list, name::Vector{String}, |
| 235 | + xticks, averages, is_root::Bool=false) |
| 236 | + if is_root |
| 237 | + this_timer_list = to_list |
| 238 | + else |
| 239 | + this_timer_list = [get_single_timer(to, name) for to ∈ to_list] |
| 240 | + end |
| 241 | + label = join(name, ":") |
| 242 | + |
| 243 | + if averages |
| 244 | + ncalls_values = [t === nothing ? 0 : TimerOutputs.ncalls(t) for t ∈ this_timer_list] |
| 245 | + end |
| 246 | + |
| 247 | + xtick_values = xticks[2] |
| 248 | + if ax_ncalls !== nothing |
| 249 | + if !averages |
| 250 | + ncalls_values = [t === nothing ? 0 : TimerOutputs.ncalls(t) for t ∈ this_timer_list] |
| 251 | + end |
| 252 | + lines!(ax_ncalls, ncalls_values; |
| 253 | + label, |
| 254 | + inspector_label=(self,i,p) -> "$(self.label[])\n$(xtick_values[round(Int64, p[1])]): ncalls=$(ncalls_values[round(Int64, p[1])])") |
| 255 | + end |
| 256 | + |
| 257 | + # Convert times from ns to ms. |
| 258 | + if ax_time !== nothing |
| 259 | + time_values = [t === nothing ? NaN : TimerOutputs.time(t) * 1.0e-6 for t ∈ this_timer_list] |
| 260 | + if averages |
| 261 | + time_values ./= ncalls_values |
| 262 | + end |
| 263 | + lines!(ax_time, time_values; |
| 264 | + label, |
| 265 | + inspector_label=(self,i,p) -> "$(self.label[])\n$(xtick_values[round(Int64, p[1])]): time=$(time_values[round(Int64, p[1])]) ms") |
| 266 | + end |
| 267 | + |
| 268 | + if ax_allocs !== nothing |
| 269 | + allocs_values = [t === nothing ? NaN : TimerOutputs.allocated(t) / 1024 for t ∈ this_timer_list] |
| 270 | + if averages |
| 271 | + allocs_values ./= ncalls_values |
| 272 | + end |
| 273 | + lines!(ax_allocs, allocs_values; |
| 274 | + label, |
| 275 | + inspector_label=(self,i,p) -> "$(self.label[])\n$(xtick_values[round(Int64, p[1])]): allocs=$(allocs_values[round(Int64, p[1])]) kB") |
| 276 | + end |
| 277 | + |
| 278 | + return nothing |
| 279 | +end |
4 | 280 |
|
5 | 281 | end |
0 commit comments