Fix GPU tests, GPU interpolation and GPU postprocessing (#912)

* fix gpu

* rm saving callback

* implement suggestions

* rm solution_saving=nothing

---------

Co-authored-by: LasNikas <niklas.nehe@web.de>
Co-authored-by: Erik Faulhaber <44124897+efaulhaber@users.noreply.github.com>

Author: 3 people

src/callbacks/post_process.jl

@@ -252,8 +252,10 @@ function (pp::PostprocessCallback)(integrator)
             system_index = system_indices(system, semi)
 
             for (key, f) in pp.func
-                result = custom_quantity(f, system, dv_ode, du_ode, v_ode, u_ode, semi, t)
-                if result !== nothing
+                result_ = custom_quantity(f, system, dv_ode, du_ode, v_ode, u_ode, semi, t)
+                if result_ !== nothing
+                    # Transfer to CPU if data is on the GPU. Do nothing if already on CPU.
+                    result = transfer2cpu(result_)
                     add_entry!(pp, string(key), t, result, filenames[system_index])
                     new_data = true
                 end

src/general/interpolation.jl

@@ -628,6 +628,11 @@ end
     return (; velocity, jacobian, von_mises_stress, cauchy_stress)
 end
 
+function interpolate_system!(cache, v, neighbor_system,
+                             point, neighbor, volume_b, W_ab, clip_negative_pressure)
+    return cache
+end
+
 @inline function interpolate_system!(cache, v, system::AbstractFluidSystem,
                                      point, neighbor, volume_b, W_ab,
                                      clip_negative_pressure)

src/io/io.jl

@@ -135,12 +135,8 @@ function add_system_data!(system_data, system::OpenBoundarySystem)
     system_data["system_type"] = type2string(system)
     system_data["fluid_system_index"] = system.fluid_system_index[]
     system_data["smoothing_length"] = system.smoothing_length
+    system_data["number_of_boundary_zones"] = length(system.boundary_zones)
     add_system_data!(system_data, system.boundary_model)
-
-    system_data["boundary_zones"] = Dict{String, Any}()
-    for (indice, boundary_zone) in enumerate(system.boundary_zones)
-        add_system_data!(system_data["boundary_zones"], boundary_zone, indice)
-    end
 end
 
 function add_system_data!(system_data, system::ParticlePackingSystem)
@@ -299,8 +295,6 @@ function add_system_data!(system_data, motion::PrescribedMotion)
     system_data["prescribed_motion"] = Dict{String, Any}()
     system_data["prescribed_motion"]["model"] = type2string(motion)
     system_data["prescribed_motion"]["movement_function"] = type2string(motion.movement_function)
-    system_data["prescribed_motion"]["is_moving"] = type2string(motion.is_moving)
-    system_data["prescribed_motion"]["moving_particles"] = motion.moving_particles
 end
 
 function add_system_data!(system_data, penalty_force::PenaltyForceGanzenmueller)

src/io/write_vtk.jl

@@ -140,8 +140,8 @@ function trixi2vtk(system_, dvdu_ode_, vu_ode_, semi_, t, periodic_box;
 
         # Extract custom quantities for this system
         if !isempty(custom_quantities)
-            dv_ode, du_ode = dvdu_ode_.x
-            dv_ode, du_ode = transfer2cpu(dv_ode, du_ode)
+            dv_ode_, du_ode_ = dvdu_ode_.x
+            dv_ode, du_ode = transfer2cpu(dv_ode_, du_ode_)
 
             for (key, quantity) in custom_quantities
                 value = custom_quantity(quantity, system, dv_ode, du_ode, v_ode, u_ode,
@@ -173,8 +173,8 @@ function transfer2cpu(v_, u_, system_, semi_)
 end
 
 function transfer2cpu(v_::AbstractGPUArray, u_)
-    v = Adapt.adapt(Array, v_)
-    u = Adapt.adapt(Array, u_)
+    v = transfer2cpu(v_)
+    u = transfer2cpu(u_)
 
     return v, u
 end
@@ -183,6 +183,14 @@ function transfer2cpu(v_, u_)
     return v_, u_
 end
 
+function transfer2cpu(a_::AbstractGPUArray)
+    return Adapt.adapt(Array, a_)
+end
+
+function transfer2cpu(a_)
+    return a_
+end
+
 function custom_quantity(quantity::AbstractArray, system, dv_ode, du_ode, v_ode, u_ode,
                          semi, t)
     return quantity

test/examples/gpu.jl

@@ -453,7 +453,7 @@ end
             # Neighborhood search with `FullGridCellList` for GPU compatibility
             min_corner = minimum(tank.boundary.coordinates, dims=2)
             max_corner = maximum(tank.boundary.coordinates, dims=2)
-            max_corner[2] = gate_height + movement_function(0.1)[2]
+            max_corner[2] = gate_height + movement_function([0, 0], 0.1f0)[2]
             # We need a very high `max_points_per_cell` because the plate resolution
             # is much finer than the fluid resolution.
             cell_list = FullGridCellList(; min_corner, max_corner)