Skip to content

Commit 004d8d2

Browse files
committed
Add runtime call tracing
1 parent 462cac7 commit 004d8d2

13 files changed

Lines changed: 1407 additions & 51 deletions

CMakeLists.txt

Lines changed: 99 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ option(MFC_OpenMP "Build with OpenMP" OFF
2323
option(MFC_GCov "Build with GCov" OFF)
2424
option(MFC_Unified "Build with unified CPU & GPU memory (GH-200 only)" OFF)
2525
option(MFC_Fastmath "Build with -gpu=fastmath on NV GPUs" OFF)
26+
option(MFC_TRACE_CALLS "Build with generated runtime call tracing" OFF)
2627
option(MFC_PRE_PROCESS "Build pre_process" OFF)
2728
option(MFC_SIMULATION "Build simulation" OFF)
2829
option(MFC_POST_PROCESS "Build post_process" OFF)
@@ -108,6 +109,17 @@ endif()
108109

109110
find_program(FYPP_EXE fypp REQUIRED)
110111

112+
set(MFC_TRACE_SUPPORTED_COMPILERS GNU Intel IntelLLVM NVHPC PGI)
113+
114+
if (MFC_TRACE_CALLS)
115+
if (NOT CMAKE_Fortran_COMPILER_ID IN_LIST MFC_TRACE_SUPPORTED_COMPILERS)
116+
message(FATAL_ERROR "ERROR: Runtime call tracing is supported with GNU, Intel/IntelLLVM, and NVHPC/PGI Fortran compilers. CMake detected ${CMAKE_Fortran_COMPILER_ID}.")
117+
endif()
118+
119+
find_package(Python3 REQUIRED COMPONENTS Interpreter)
120+
find_package(Threads REQUIRED)
121+
endif()
122+
111123

112124
# Miscellaneous Configuration:
113125
# * Explicitly link to -ldl (or system equivalent)
@@ -133,6 +145,16 @@ if (CMAKE_Fortran_COMPILER_ID STREQUAL "GNU")
133145
$<$<COMPILE_LANGUAGE:Fortran>:-ffree-line-length-none>
134146
)
135147

148+
if (MFC_TRACE_CALLS)
149+
add_compile_options(
150+
$<$<COMPILE_LANGUAGE:Fortran>:-finstrument-functions>
151+
$<$<COMPILE_LANGUAGE:Fortran>:-g>
152+
$<$<COMPILE_LANGUAGE:Fortran>:-fno-inline>
153+
$<$<COMPILE_LANGUAGE:Fortran>:-fno-optimize-sibling-calls>
154+
)
155+
add_link_options("-rdynamic")
156+
endif()
157+
136158
if (MFC_GCov)
137159

138160
# Warning present in gcc versions >= 12 that is treated as an error
@@ -241,14 +263,23 @@ elseif (CMAKE_Fortran_COMPILER_ID STREQUAL "Flang")
241263
if (CMAKE_BUILD_TYPE STREQUAL "Debug" OR CMAKE_BUILD_TYPE STREQUAL "RelDebug")
242264
add_compile_options($<$<COMPILE_LANGUAGE:Fortran>:-O1> $<$<COMPILE_LANGUAGE:Fortran>:-g>)
243265
endif()
244-
elseif (CMAKE_Fortran_COMPILER_ID STREQUAL "Intel")
266+
elseif ((CMAKE_Fortran_COMPILER_ID STREQUAL "Intel") OR (CMAKE_Fortran_COMPILER_ID STREQUAL "IntelLLVM"))
245267
add_compile_options($<$<COMPILE_LANGUAGE:Fortran>:-free>)
246268

247269
if (CMAKE_BUILD_TYPE STREQUAL "Debug")
248270
add_compile_options(-g -Og -traceback -debug -check all)
249271
elseif (CMAKE_BUILD_TYPE STREQUAL "RelDebug")
250272
add_compile_options(-g -Og -traceback -check bounds)
251273
endif()
274+
275+
if (MFC_TRACE_CALLS)
276+
add_compile_options(
277+
$<$<COMPILE_LANGUAGE:Fortran>:-finstrument-functions>
278+
$<$<COMPILE_LANGUAGE:Fortran>:-g>
279+
$<$<COMPILE_LANGUAGE:Fortran>:-O0>
280+
)
281+
add_link_options("-Wl,-export-dynamic")
282+
endif()
252283
elseif ((CMAKE_Fortran_COMPILER_ID STREQUAL "NVHPC") OR (CMAKE_Fortran_COMPILER_ID STREQUAL "PGI"))
253284
add_compile_options(
254285
$<$<COMPILE_LANGUAGE:Fortran>:-Mfreeform>
@@ -275,12 +306,28 @@ elseif ((CMAKE_Fortran_COMPILER_ID STREQUAL "NVHPC") OR (CMAKE_Fortran_COMPILER_
275306
)
276307
endif()
277308

309+
if (MFC_TRACE_CALLS)
310+
add_compile_options(
311+
$<$<COMPILE_LANGUAGE:Fortran>:-Minstrument=functions>
312+
$<$<COMPILE_LANGUAGE:Fortran>:-g>
313+
)
314+
add_link_options("-Wl,-export-dynamic")
315+
endif()
316+
278317
if (DEFINED ENV{MFC_CUDA_CC})
279318
string(REGEX MATCHALL "[0-9]+" MFC_CUDA_CC $ENV{MFC_CUDA_CC})
280319
message(STATUS "Found $MFC_CUDA_CC specified. GPU code will be generated for compute capability(ies) ${MFC_CUDA_CC}.")
281320
endif()
282321
endif()
283322

323+
if (MFC_TRACE_CALLS)
324+
if (CMAKE_Fortran_COMPILER_ID STREQUAL "GNU")
325+
set_source_files_properties("${CMAKE_SOURCE_DIR}/src/common/m_trace.f90" PROPERTIES COMPILE_OPTIONS "-fno-instrument-functions")
326+
elseif ((CMAKE_Fortran_COMPILER_ID STREQUAL "NVHPC") OR (CMAKE_Fortran_COMPILER_ID STREQUAL "PGI"))
327+
set_source_files_properties("${CMAKE_SOURCE_DIR}/src/common/m_trace.f90" PROPERTIES COMPILE_OPTIONS "-Minstrument-exclude-func-list=s_trace_point_begin;-Minstrument-exclude-func-list=s_trace_point_end")
328+
endif()
329+
endif()
330+
284331
if (CMAKE_BUILD_TYPE STREQUAL "Release")
285332
# Processor tuning: Check if we can target the host's native CPU's ISA.
286333
# Skip for gcov builds — -march=native on newer CPUs (e.g. Granite Rapids)
@@ -306,7 +353,9 @@ if (CMAKE_BUILD_TYPE STREQUAL "Release")
306353

307354
# Enable LTO/IPO if supported (skip for gcov — LTO interferes with coverage
308355
# instrumentation and can trigger assembler errors on newer architectures).
309-
if (MFC_GCov)
356+
if (MFC_TRACE_CALLS)
357+
message(STATUS "LTO/IPO disabled for trace build")
358+
elseif (MFC_GCov)
310359
message(STATUS "LTO/IPO disabled for gcov build")
311360
elseif (CMAKE_Fortran_COMPILER_ID STREQUAL "NVHPC")
312361
if (MFC_Unified)
@@ -396,6 +445,7 @@ macro(HANDLE_SOURCES target useCommon)
396445
set(${target}_SRCs ${${target}_F90s})
397446
if (${useCommon})
398447
file(GLOB common_F90s CONFIGURE_DEPENDS "${common_DIR}/*.f90")
448+
list(FILTER common_F90s EXCLUDE REGEX ".*/m_trace\.f90$")
399449
list(APPEND ${target}_SRCs ${common_F90s})
400450
endif()
401451

@@ -458,6 +508,48 @@ macro(HANDLE_SOURCES target useCommon)
458508

459509
list(APPEND ${target}_SRCs ${f90})
460510
endforeach()
511+
512+
if (MFC_TRACE_CALLS)
513+
list(FIND ${target}_SRCs "${common_DIR}/m_trace.f90" _trace_module_index)
514+
if (_trace_module_index EQUAL -1)
515+
list(APPEND ${target}_SRCs "${common_DIR}/m_trace.f90")
516+
endif()
517+
list(APPEND ${target}_SRCs "${common_DIR}/m_trace_runtime.c")
518+
519+
set(_trace_SRCs "")
520+
set(_trace_instrument_options "")
521+
if (MFC_OpenACC OR MFC_OpenMP)
522+
list(APPEND _trace_instrument_options "--no-point-traces")
523+
endif()
524+
file(MAKE_DIRECTORY "${CMAKE_BINARY_DIR}/trace/${target}")
525+
526+
foreach(src ${${target}_SRCs})
527+
cmake_path(GET src FILENAME src_filename)
528+
529+
if(src_filename STREQUAL "m_trace.f90" OR src_filename STREQUAL "m_trace_runtime.c" OR src_filename STREQUAL "m_thermochem.f90")
530+
list(APPEND _trace_SRCs "${src}")
531+
else()
532+
set(trace_src "${CMAKE_BINARY_DIR}/trace/${target}/${src_filename}")
533+
534+
add_custom_command(
535+
OUTPUT "${trace_src}"
536+
COMMAND "${Python3_EXECUTABLE}"
537+
"${CMAKE_SOURCE_DIR}/toolchain/instrument_fortran_trace.py"
538+
${_trace_instrument_options}
539+
"${src}"
540+
"${trace_src}"
541+
DEPENDS "${src}"
542+
"${CMAKE_SOURCE_DIR}/toolchain/instrument_fortran_trace.py"
543+
COMMENT "Instrumenting (trace) ${src_filename}"
544+
VERBATIM
545+
)
546+
547+
list(APPEND _trace_SRCs "${trace_src}")
548+
endif()
549+
endforeach()
550+
551+
set(${target}_SRCs ${_trace_SRCs})
552+
endif()
461553
endmacro()
462554

463555

@@ -559,6 +651,10 @@ exit 0
559651
MFC_${${ARGS_TARGET}_UPPER}
560652
)
561653

654+
if (MFC_TRACE_CALLS)
655+
target_link_libraries(${a_target} PRIVATE Threads::Threads)
656+
endif()
657+
562658
if (MFC_MPI AND ARGS_MPI)
563659
find_package(MPI COMPONENTS Fortran REQUIRED)
564660

@@ -1039,4 +1135,4 @@ site_name(SITE_NAME)
10391135

10401136
configure_file(
10411137
"${CMAKE_CURRENT_SOURCE_DIR}/toolchain/cmake/configuration.cmake.in"
1042-
"${CMAKE_CURRENT_BINARY_DIR}/configuration.txt")
1138+
"${CMAKE_CURRENT_BINARY_DIR}/configuration.txt")

docs/module_categories.json

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -71,7 +71,8 @@
7171
"m_checker",
7272
"m_checker_common",
7373
"m_sim_helpers",
74-
"m_derived_variables"
74+
"m_derived_variables",
75+
"m_trace"
7576
]
7677
},
7778
{

src/common/m_trace.f90

Lines changed: 176 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,176 @@
1+
!>
2+
!! @file
3+
!! @brief Contains module m_trace
4+
5+
!> @brief Simple runtime call tracing helpers.
6+
module m_trace
7+
8+
use iso_fortran_env, only: output_unit
9+
10+
implicit none
11+
12+
private
13+
public :: s_trace_enter, s_trace_call, s_trace_global_call, s_trace_point_begin, s_trace_point_end
14+
15+
logical, private :: trace_initialized = .false.
16+
logical, private :: trace_enabled = .false.
17+
logical, private :: trace_point_enabled = .false.
18+
logical, private :: trace_point_middle = .false.
19+
integer, private :: trace_j = 0
20+
integer, private :: trace_k = 0
21+
integer, private :: trace_l = 0
22+
integer, private :: trace_point_depth = 0
23+
character(len=64), private :: trace_point_vars = ''
24+
25+
interface
26+
subroutine c_trace_point_begin() bind(C, name="mfc_trace_point_begin")
27+
28+
end subroutine c_trace_point_begin
29+
30+
subroutine c_trace_point_end() bind(C, name="mfc_trace_point_end")
31+
32+
end subroutine c_trace_point_end
33+
end interface
34+
35+
contains
36+
37+
!> Initialize trace settings from environment variables.
38+
subroutine s_initialize_trace
39+
40+
character(len=64) :: env_value
41+
integer :: env_len
42+
integer :: first_comma
43+
integer :: second_comma
44+
integer :: read_status
45+
46+
if (trace_initialized) return
47+
48+
call get_environment_variable('MFC_TRACE', env_value, length=env_len)
49+
trace_enabled = env_len > 0 .and. trim(env_value) /= '0'
50+
51+
call get_environment_variable('MFC_TRACE_POINT', env_value, length=env_len)
52+
if (env_len > 0) then
53+
if (trim(env_value(:env_len)) == 'middle') then
54+
trace_point_enabled = .true.
55+
trace_point_middle = .true.
56+
else
57+
first_comma = index(env_value, ',')
58+
second_comma = 0
59+
if (first_comma > 0) second_comma = index(env_value(first_comma + 1:), ',')
60+
if (second_comma > 0) second_comma = second_comma + first_comma
61+
62+
if (first_comma > 0 .and. second_comma > first_comma) then
63+
read (env_value(:first_comma - 1), *, iostat=read_status) trace_j
64+
trace_point_enabled = read_status == 0
65+
read (env_value(first_comma + 1:second_comma - 1), *, iostat=read_status) trace_k
66+
trace_point_enabled = trace_point_enabled .and. read_status == 0
67+
read (env_value(second_comma + 1:env_len), *, iostat=read_status) trace_l
68+
trace_point_enabled = trace_point_enabled .and. read_status == 0
69+
end if
70+
end if
71+
end if
72+
73+
call get_environment_variable('MFC_TRACE_POINT_VARS', env_value, length=env_len)
74+
if (env_len > 0) trace_point_vars = trim(env_value(:env_len))
75+
76+
trace_initialized = .true.
77+
78+
end subroutine s_initialize_trace
79+
80+
!> Emit a live call-trace line when MFC_TRACE is enabled.
81+
subroutine s_trace_enter(name)
82+
83+
character(len=*), intent(in) :: name
84+
85+
call s_initialize_trace()
86+
87+
if (.not. trace_enabled) return
88+
if (trace_point_enabled .and. trace_point_depth <= 0) return
89+
90+
write (output_unit, '(A,A)') 'TRACE ', trim(name)
91+
call flush (output_unit)
92+
93+
end subroutine s_trace_enter
94+
95+
!> Emit a live call-site trace when the current trace scope is active.
96+
subroutine s_trace_call(name)
97+
98+
character(len=*), intent(in) :: name
99+
100+
call s_initialize_trace()
101+
102+
if (.not. trace_enabled) return
103+
if (trace_point_enabled .and. trace_point_depth <= 0) return
104+
105+
write (output_unit, '(A,A)') 'TRACE ', trim(name)
106+
call flush (output_unit)
107+
108+
end subroutine s_trace_call
109+
110+
!> Emit a call-site trace even when point tracing has not entered a cell loop yet.
111+
subroutine s_trace_global_call(name)
112+
113+
character(len=*), intent(in) :: name
114+
115+
call s_initialize_trace()
116+
117+
if (.not. trace_enabled) return
118+
119+
write (output_unit, '(A,A)') 'TRACE ', trim(name)
120+
call flush (output_unit)
121+
122+
end subroutine s_trace_global_call
123+
124+
!> Enable nested routine-entry tracing while a call at MFC_TRACE_POINT executes.
125+
subroutine s_trace_point_begin(j, k, l, vars, mid_j, mid_k, mid_l)
126+
127+
integer, intent(in) :: j
128+
integer, intent(in) :: k
129+
integer, intent(in) :: l
130+
character(len=*), intent(in) :: vars
131+
integer, intent(in) :: mid_j
132+
integer, intent(in) :: mid_k
133+
integer, intent(in) :: mid_l
134+
integer :: target_j
135+
integer :: target_k
136+
integer :: target_l
137+
138+
call s_initialize_trace()
139+
140+
if (.not. trace_enabled) return
141+
if (.not. trace_point_enabled) return
142+
if (len_trim(trace_point_vars) > 0 .and. trim(trace_point_vars) /= 'any') then
143+
if (trim(vars) /= trim(trace_point_vars)) return
144+
end if
145+
if (trace_point_middle) then
146+
target_j = mid_j
147+
target_k = mid_k
148+
target_l = mid_l
149+
else
150+
target_j = trace_j
151+
target_k = trace_k
152+
target_l = trace_l
153+
end if
154+
155+
if (j /= target_j .or. k /= target_k .or. l /= target_l) return
156+
157+
trace_point_depth = trace_point_depth + 1
158+
call c_trace_point_begin()
159+
160+
end subroutine s_trace_point_begin
161+
162+
!> Disable nested point tracing after a grid-point call returns.
163+
subroutine s_trace_point_end
164+
165+
call s_initialize_trace()
166+
167+
if (.not. trace_enabled) return
168+
if (.not. trace_point_enabled) return
169+
if (trace_point_depth > 0) then
170+
call c_trace_point_end()
171+
trace_point_depth = trace_point_depth - 1
172+
end if
173+
174+
end subroutine s_trace_point_end
175+
176+
end module m_trace

0 commit comments

Comments
 (0)