Add synchronization statements to ensure timer output correctness on the GPU (#2892)

* Add synchronization statements to ensure timer output correctness on the GPU

* use trixi_timit macro with a backend

* don't commit piracy

* Update src/Trixi.jl

Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>

* fixup! Add synchronization statements to ensure timer output correctness on the GPU

Co-authored-by: Benedict <135045760+benegee@users.noreply.github.com>

* only synchronize when timers are enabled

---------

Co-authored-by: Hendrik Ranocha <ranocha@users.noreply.github.com>
Co-authored-by: Benedict <135045760+benegee@users.noreply.github.com>

Author: 3 people

src/auxiliary/auxiliary.jl

@@ -82,6 +82,26 @@ end
     return ncalls_first
 end
 
+"""
+    @trixi_timeit_ext backend timer() "some label" expression
+
+This macro is an extension of [`@trixi_timeit`](@ref) that also synchronizes the given `backend` after executing the given `expression`.
+This is useful to get accurate timing measurements for GPU backends, where the execution of kernels is asynchronous.
+The synchronization ensures that all GPU operations are completed before the timer is stopped.
+
+See also [`@trixi_timeit`](@ref).
+"""
+macro trixi_timeit_ext(backend, timer_output, label, expr)
+    expr = quote
+        local val = $(esc(expr))
+        if $(esc(backend)) !== nothing && $(TrixiBase).timeit_debug_enabled()
+            $(KernelAbstractions.synchronize)($(esc(backend)))
+        end
+        val
+    end
+    return :(@trixi_timeit($(esc(timer_output)), $(esc(label)), $(expr)))
+end
+
 """
     examples_dir()
 

src/solvers/dgsem_tree/dg_2d.jl

@@ -113,63 +113,66 @@ function rhs!(du, u, t,
     backend = trixi_backend(u)
 
     # Reset du
-    @trixi_timeit timer() "reset ∂u/∂t" set_zero!(du, dg, cache)
+    @trixi_timeit_ext backend timer() "reset ∂u/∂t" begin
+        set_zero!(du, dg, cache)
+    end
 
     # Calculate volume integral
-    @trixi_timeit timer() "volume integral" begin
+    @trixi_timeit_ext backend timer() "volume integral" begin
         calc_volume_integral!(backend, du, u, mesh,
                               have_nonconservative_terms(equations), equations,
                               dg.volume_integral, dg, cache)
     end
 
     # Prolong solution to interfaces
-    @trixi_timeit timer() "prolong2interfaces" begin
+    @trixi_timeit_ext backend timer() "prolong2interfaces" begin
         prolong2interfaces!(backend, cache, u, mesh, equations, dg)
     end
 
     # Calculate interface fluxes
-    @trixi_timeit timer() "interface flux" begin
+    @trixi_timeit_ext backend timer() "interface flux" begin
         calc_interface_flux!(backend, cache.elements.surface_flux_values, mesh,
                              have_nonconservative_terms(equations), equations,
                              dg.surface_integral, dg, cache)
     end
 
     # Prolong solution to boundaries
-    @trixi_timeit timer() "prolong2boundaries" begin
+    @trixi_timeit_ext backend timer() "prolong2boundaries" begin
         prolong2boundaries!(cache, u, mesh, equations, dg)
     end
 
     # Calculate boundary fluxes
-    @trixi_timeit timer() "boundary flux" begin
+    @trixi_timeit_ext backend timer() "boundary flux" begin
         calc_boundary_flux!(cache, t, boundary_conditions, mesh, equations,
                             dg.surface_integral, dg)
     end
 
     # Prolong solution to mortars
-    @trixi_timeit timer() "prolong2mortars" begin
+    @trixi_timeit_ext backend timer() "prolong2mortars" begin
         prolong2mortars!(cache, u, mesh, equations,
                          dg.mortar, dg)
     end
 
     # Calculate mortar fluxes
-    @trixi_timeit timer() "mortar flux" begin
+    @trixi_timeit_ext backend timer() "mortar flux" begin
         calc_mortar_flux!(cache.elements.surface_flux_values, mesh,
                           have_nonconservative_terms(equations), equations,
                           dg.mortar, dg.surface_integral, dg, cache)
     end
 
     # Calculate surface integrals
-    @trixi_timeit timer() "surface integral" begin
+    @trixi_timeit_ext backend timer() "surface integral" begin
         calc_surface_integral!(backend, du, u, mesh, equations,
                                dg.surface_integral, dg, cache)
     end
 
     # Apply Jacobian from mapping to reference element
-    @trixi_timeit timer() "Jacobian" apply_jacobian!(backend, du, mesh, equations, dg,
-                                                     cache)
+    @trixi_timeit_ext backend timer() "Jacobian" begin
+        apply_jacobian!(backend, du, mesh, equations, dg, cache)
+    end
 
     # Calculate source terms
-    @trixi_timeit timer() "source terms" begin
+    @trixi_timeit_ext backend timer() "source terms" begin
         calc_sources!(du, u, t, source_terms, equations, dg, cache)
     end