Makefile.coverage

# Dear emacs, this is a -*- Makefile -*-
#  This behaves kind of like a SUBDIRS makefile.
#  It is meant to be run from the top level ob directory.

SUBDIRS:=$(shell cat ./scripts/testlist.v$(ARCHV) | perl -p -e 's/\s*\#.*//;')
#SUBDIRS_ALL:=$(shell cat ./scripts/testlist.v* | sort | uniq | perl -p -e 's/^\s*\#.*//;')

TESTOUT ?= $(INSTALLPATH)/test.out

# TEST_BUILD_ROOT must be defined here (before the pattern rule and RESULT_FILES) so
# that $(TEST_BUILD_ROOT) expands correctly at parse time.  INSTALLPATH is exported by
# the parent make (h2_test) so it is already available when this file is first read.
TEST_BUILD_ROOT ?= $(patsubst %/install,%/build/tests,$(INSTALLPATH))

# File-based list of per-test result targets.  Used by tst and test_report.html
# so make can track what is up-to-date and skip unnecessary work.
RESULT_FILES = $(patsubst %, $(TEST_BUILD_ROOT)/%/results.txt, $(SUBDIRS))

# Coverage result files — per-test cov.txt generated alongside gmon data.
COV_FILES = $(patsubst %, $(TEST_BUILD_ROOT)/%/cov.txt, $(SUBDIRS))

# Committed list of functions to omit from coverage tracking.  gen_cov_fns.pl
# drops these; the cov.html "Omit" button produces an updated copy to commit.
COV_OMIT = $(H2DIR)/scripts/cov_omit_functions

# Minimum packet count threshold: functions with <= N packets are excluded from
# the HTML report (they are too small to produce meaningful coverage signal).
# Override on the command line:  make cov TARGET=ref_cov COV_MIN_PACKETS=3
# Default is 1 (only single-packet stubs are filtered).

all:
	$(MAKE) -f scripts/Makefile.coverage $(SUBDIRS) TARG=all

# tst is phony but depends on real file targets so make tracks what needs re-running.
.PHONY: tst
tst: $(RESULT_FILES)

# Build and run one test, producing its results.txt.
# Depends on $(INSTALLPATH)/manifest (sha256 of key install files, updated only when
# content changes) so tests re-run iff the build actually changed something.
# The leading '-' on the inner make line makes the rule succeed even when the
# test fails; PASS/FAIL is captured in results.txt content (read by check-fail
# and gen_test_results.py).  Without this, .DELETE_ON_ERROR would remove the
# FAIL details and abort tst, blocking unified report generation.
# The find whitelists only source-file extensions (so stale outputs from
# in-source builds — *.elf, *.o, *.log, results.txt, gmon*, etc. — are NOT
# symlinked into the per-variant build dir).  Without this, every variant
# would write through the same symlink to the source tree and clobber each
# other's results, leaving only the last writer's result in the report.
# Named exclusions cover generator outputs whose extension matches a source
# extension (scenarios.h from scenarios.py, generated_tests.dat from
# gentests.py, threadmap.py from a perf tool).
$(TEST_BUILD_ROOT)/%/results.txt: $(INSTALLPATH)/manifest
	mkdir -p "$(TEST_BUILD_ROOT)/$*"
	find "$(H2DIR)/$*" -maxdepth 1 -type f \
	    \( -name 'Makefile' -o -name 'tested_functions' \
	       -o -name '*.c' -o -name '*.h' \
	       -o -name '*.S' -o -name '*.s' \
	       -o -name '*.cpp' -o -name '*.cc' \
	       -o -name '*.py' -o -name '*.dat' \
	       -o -name '*.cfg' \
	    \) \
	    ! -name 'scenarios.h' \
	    ! -name 'generated_tests.dat' \
	    ! -name 'threadmap.py' \
	    -exec ln -sf '{}' "$(TEST_BUILD_ROOT)/$*/" ';'
	printf 'H2DIR=$(H2DIR)\ninclude $$(H2DIR)/scripts/Makefile.inc.test\n' \
	    > "$(TEST_BUILD_ROOT)/$*/Makefile.inc"
	(echo; echo; echo "##########   $*   ##########") > "$(TEST_BUILD_ROOT)/$*/make.log"
	-$(MAKE) -C "$(TEST_BUILD_ROOT)/$*" tst TEST="$*" 2>&1 | \
	    tee -a "$(TEST_BUILD_ROOT)/$*/make.log"; \
	    TSTAT=$${PIPESTATUS[0]}; \
	    if [ $$TSTAT -eq 0 ]; then touch "$@"; fi; \
	    exit $$TSTAT

# Build and run one test for coverage (TARG=all), producing per-test cov.txt.
# Also produces results.txt as a side effect of running the simulator with --profile.
# Depends on manifest so tests only re-run when the build actually changes, and on
# cov_fns so the function list exists before a multi-gmon test invokes merge_cov.py.
$(TEST_BUILD_ROOT)/%/cov.txt: $(INSTALLPATH)/manifest $(INSTALLPATH)/cov/cov_fns
	mkdir -p "$(TEST_BUILD_ROOT)/$*"
	find "$(H2DIR)/$*" -maxdepth 1 -type f ! -name Makefile.inc ! -name 'cov.txt' ! -name 'cov-*.txt' \
	    -exec ln -sf '{}' "$(TEST_BUILD_ROOT)/$*/" ';'
	printf 'H2DIR=$(H2DIR)\ninclude $$(H2DIR)/scripts/Makefile.inc.test\n' \
	    > "$(TEST_BUILD_ROOT)/$*/Makefile.inc"
	(echo; echo; echo "##########   $*   ##########") > "$(TEST_BUILD_ROOT)/$*/make.log"
	$(MAKE) -C "$(TEST_BUILD_ROOT)/$*" all TEST="$*" 2>&1 | \
	    tee -a "$(TEST_BUILD_ROOT)/$*/make.log"; \
	    TSTAT=$${PIPESTATUS[0]}; \
	    if [ $$TSTAT -eq 0 ]; then touch "$@"; fi; \
	    exit $$TSTAT

# Generate the coverage function list from the just-built kernel libraries so the
# set of tracked functions always matches the binaries under test (no stale,
# checked-in cov_fns snapshot).  gen_cov_fns.pl scans libh2kernel.a and libh2.a
# for every F-type H2K_*/h2_* symbol.  Depends on manifest (not the libs
# directly) because manifest carries the build rule and changes whenever the
# install is rebuilt.  Also depends on COV_OMIT so editing the omit list
# regenerates the function list.
$(INSTALLPATH)/cov/cov_fns: $(INSTALLPATH)/manifest $(COV_OMIT)
	mkdir -p $(INSTALLPATH)/cov
	${H2DIR}/scripts/gen_cov_fns.pl $(INSTALLPATH) $(COV_OMIT) > $@

# Generate the function -> source-file map for the HTML report's file-tree view.
# Derived from the same just-built libraries (objdump symbols + DWARF compile-unit
# paths), so it always matches the binaries under test.  Depends on manifest for
# the same reason as cov_fns.
$(INSTALLPATH)/cov/cov_files: $(INSTALLPATH)/manifest
	mkdir -p $(INSTALLPATH)/cov
	$(PYTHON) ${H2DIR}/scripts/gen_cov_files.py \
	    --installpath $(INSTALLPATH) --repo $(H2DIR) \
	    --objdump "$(OBJDUMP)" --dwarfdump "$(DWARFDUMP)" > $@

# Merge all per-test cov.txt files into a single file under the install directory.
# .DELETE_ON_ERROR ensures a failed merge_cov.py run does not leave a stale output.
.DELETE_ON_ERROR:
$(INSTALLPATH)/cov/cov.txt: $(COV_FILES) $(INSTALLPATH)/cov/cov_fns
	find $(TEST_BUILD_ROOT) -name "cov.txt" | $(PYTHON) ${H2DIR}/scripts/merge_cov.py \
	    --offset_pc --dir $(H2DIR) --installpath $(INSTALLPATH) > $@

$(INSTALLPATH)/cov/cov.rpt: $(INSTALLPATH)/cov/cov.txt
	${H2DIR}/scripts/cov_rpt_tool $< > $@

$(INSTALLPATH)/cov.html: $(INSTALLPATH)/cov/cov.rpt $(INSTALLPATH)/cov/cov_files $(COV_OMIT)
	$(PYTHON) ${H2DIR}/scripts/cov_html_report.py \
	    --rpt $< --cov $(INSTALLPATH)/cov/cov.txt \
	    --fn-files $(INSTALLPATH)/cov/cov_files --omit-file $(COV_OMIT) --output $@ \
	    $(patsubst %,--test-cov %,$(COV_FILES)) \
	    $(if $(COV_MIN_PACKETS),--min-packets $(COV_MIN_PACKETS))

# Collect per-test results.txt into a clean PASS/FAIL summary and append to TESTOUT.
# Called after tst completes; each test's full output is in <test-dir>/make.log.
.PHONY: test_summary
test_summary:
	@PASS=0; FAIL=0; \
	for SUBDIR in ${SUBDIRS}; do \
	    if grep -q "TEST PASSED" "$(TEST_BUILD_ROOT)/$$SUBDIR/results.txt" 2>/dev/null; then \
	        PASS=$$((PASS+1)); \
	    else \
	        FAIL=$$((FAIL+1)); \
	    fi; \
	done; \
	echo "SUMMARY: $$PASS passed, $$FAIL failed" | tee -a $(TESTOUT)

clean:
	rm -rf "$(TEST_BUILD_ROOT)"

$(INSTALLPATH)/merged.profraw: tst
	echo '[COVERAGE] Merging:' $(shell find ${H2DIR}/kernel ${H2DIR}/libs -name "default.profraw")
	$(LLVM_PROFDATA) merge -o $@ $(shell find ${H2DIR}/kernel ${H2DIR}/libs -name "default.profraw")

$(INSTALLPATH)/coverage_report.txt: $(INSTALLPATH)/merged.profraw
	echo '[COVERAGE] Creating text report'
	$(LLVM_COV) report $(INSTALLPATH)/bin/booter --instr-profile=$< --format=text | tee $@

$(INSTALLPATH)/coverage_report.lcov: $(INSTALLPATH)/merged.profraw
	echo '[COVERAGE] Creating lcov report'
	$(LLVM_COV) export $(INSTALLPATH)/bin/booter --instr-profile=$< --format=lcov > $@

$(INSTALLPATH)/index.html: $(INSTALLPATH)/merged.profraw
	echo '[COVERAGE] Creating html report'
	$(LLVM_COV) report $(INSTALLPATH)/bin/booter --instr-profile=$< --format=html

#TODO: enable html reports
coverage_report: $(INSTALLPATH)/coverage_report.txt $(INSTALLPATH)/coverage_report.lcov #$(INSTALLPATH)/index.html

$(SUBDIRS)::
	mkdir -p "$(TEST_BUILD_ROOT)/$@"
	find "$(H2DIR)/$@" -maxdepth 1 -type f ! -name Makefile.inc \
	    -exec ln -sf '{}' "$(TEST_BUILD_ROOT)/$@/" ';'
	printf 'H2DIR=$(H2DIR)\ninclude $$(H2DIR)/scripts/Makefile.inc.test\n' \
	    > "$(TEST_BUILD_ROOT)/$@/Makefile.inc"
	(echo; echo; echo "##########   $@   ##########") > "$(TEST_BUILD_ROOT)/$@/make.log"
	$(MAKE) -C "$(TEST_BUILD_ROOT)/$@" $(TARG) TEST=$@ 2>&1 | \
	    tee -a "$(TEST_BUILD_ROOT)/$@/make.log"; \
	    TSTAT=$${PIPESTATUS[0]}; if [ $$TSTAT -ne 0 ]; then STAT=$$TSTAT; fi

# $(SUBDIRS_ALL)::
# 	cd $@;\
# 	$(MAKE) $(TARG) TEST=$@ 2>&1 | tee -a make.log; TSTAT=$${PIPESTATUS[0]}; \
# 	if [ $$TSTAT -ne 0 ]; then STAT=$$TSTAT; fi;

# .PHONY: $(SUBDIRS) $(SUBDIRS_ALL) tst coverage_report



include ./scripts/Makefile.inc.test