diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 06c2a50..5e1cefc 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -45,6 +45,16 @@ jobs: run: cd test-workspace && rustup target add riscv64imac-unknown-none-elf && rustup component add clippy - name: Run all checks run: cd test-workspace && make build test check clippy + - name: Generate native simulators for coverage + if: matrix.os == 'ubuntu-latest' + run: cd test-workspace && + make generate-native-simulator CRATE=c1 TEMPLATE_TYPE=--path TEMPLATE_REPO=.. + - name: Install llvm-tools for coverage + if: matrix.os == 'ubuntu-latest' + run: cd test-workspace && make coverage-install + - name: Run coverage tests + if: matrix.os == 'ubuntu-latest' + run: cd test-workspace && make coverage - name: Reproducible build runs run: cd test-workspace && ./scripts/reproducible_build_docker --update && ./scripts/reproducible_build_docker --no-clean - name: Generate standalone contract diff --git a/README.md b/README.md index fad96b9..32214e2 100644 --- a/README.md +++ b/README.md @@ -255,3 +255,47 @@ make generate-native-simulator CRATE=example_crate - The `CRATE` parameter must refer to a subproject. - Missing subprojects will cause the command to fail. + +### Code Coverage + +The templates include built-in support for code coverage reporting using LLVM's coverage tools. + +**Requirements:** +- Only contracts created with the `contract` template support coverage (pure Rust contracts with `native-simulator` feature) +- Other templates (`atomics-contract`, `stack-reorder-contract`, `contract-without-simulator`, `standalone-contract`) do not support coverage +- Native simulators must be generated for each contract you want coverage for +- Only works on x86_64 Linux (not ARM) + +**Setup:** + +1. Install llvm-tools: +```bash +make coverage-install +``` + +2. Generate native simulators for your contracts: +```bash +make generate-native-simulator CRATE= +``` + +**Generate Reports:** +```bash +make coverage # Text report to console +make coverage-html # HTML report in target/coverage/html/ +make coverage-lcov # LCOV format for CI integration +``` + +**Example:** +```bash +# Create a contract using the default 'contract' template +make generate CRATE=my-contract + +# Generate native simulator for it +make generate-native-simulator CRATE=my-contract + +# Install tools and run coverage +make coverage-install +make coverage +``` + +Coverage works by running tests with the `native-simulator` feature, which executes contract code natively on x86_64 instead of in CKB-VM. This allows LLVM's coverage instrumentation to track which lines of code are executed. diff --git a/atomics-contract/.cargo-generate/tests.rs b/atomics-contract/.cargo-generate/tests.rs index b763d6b..f19ca2f 100644 --- a/atomics-contract/.cargo-generate/tests.rs +++ b/atomics-contract/.cargo-generate/tests.rs @@ -4,8 +4,7 @@ fn test_{{crate_name}}() { // deploy contract let mut context = Context::default(); - let contract_bin: Bytes = Loader::default().load_binary("{{project-name}}"); - let out_point = context.deploy_cell(contract_bin); + let out_point = context.deploy_cell_by_name("{{project-name}}"); // prepare scripts let lock_script = context diff --git a/contract-without-simulator/.cargo-generate/tests.rs b/contract-without-simulator/.cargo-generate/tests.rs index b763d6b..f19ca2f 100644 --- a/contract-without-simulator/.cargo-generate/tests.rs +++ b/contract-without-simulator/.cargo-generate/tests.rs @@ -4,8 +4,7 @@ fn test_{{crate_name}}() { // deploy contract let mut context = Context::default(); - let contract_bin: Bytes = Loader::default().load_binary("{{project-name}}"); - let out_point = context.deploy_cell(contract_bin); + let out_point = context.deploy_cell_by_name("{{project-name}}"); // prepare scripts let lock_script = context diff --git a/contract/.cargo-generate/tests.rs b/contract/.cargo-generate/tests.rs index b763d6b..f19ca2f 100644 --- a/contract/.cargo-generate/tests.rs +++ b/contract/.cargo-generate/tests.rs @@ -4,8 +4,7 @@ fn test_{{crate_name}}() { // deploy contract let mut context = Context::default(); - let contract_bin: Bytes = Loader::default().load_binary("{{project-name}}"); - let out_point = context.deploy_cell(contract_bin); + let out_point = context.deploy_cell_by_name("{{project-name}}"); // prepare scripts let lock_script = context diff --git a/native-simulator/Cargo.toml b/native-simulator/Cargo.toml index 203488c..847813c 100644 --- a/native-simulator/Cargo.toml +++ b/native-simulator/Cargo.toml @@ -3,10 +3,13 @@ name = "{{project-name}}" version = "0.1.0" edition = "2024" -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] {{project-name | append: "@@SIMULATOR_PLACEHOLDER@@" | remove: "-sim@@SIMULATOR_PLACEHOLDER@@"}} = { path = "../../contracts/{{project-name | append: "@@SIMULATOR_PLACEHOLDER@@" | remove: "-sim@@SIMULATOR_PLACEHOLDER@@"}}", features = ["native-simulator"] } ckb-std = { version = "1.0", features = ["native-simulator"] } +ctor = "0.6" [lib] crate-type = ["cdylib"] + +[lints.rust] +unexpected_cfgs = { level = "warn", check-cfg = ['cfg(coverage)'] } diff --git a/native-simulator/src/lib.rs b/native-simulator/src/lib.rs index 26128b7..a66475c 100644 --- a/native-simulator/src/lib.rs +++ b/native-simulator/src/lib.rs @@ -1 +1,18 @@ ckb_std::entry_simulator!({{project-name | append: "@@SIMULATOR_PLACEHOLDER@@" | remove: "-sim@@SIMULATOR_PLACEHOLDER@@" | replace: "-", "_"}}::program_entry); + +// Flush LLVM coverage data when the shared library is unloaded. +// This is necessary because coverage data from dynamically loaded libraries +// is not automatically written when using cargo-llvm-cov. +#[cfg(coverage)] +mod coverage_flush { + unsafe extern "C" { + fn __llvm_profile_write_file() -> i32; + } + + #[ctor::dtor] + fn flush_coverage() { + unsafe { + __llvm_profile_write_file(); + } + } +} diff --git a/stack-reorder-contract/.cargo-generate/tests.rs b/stack-reorder-contract/.cargo-generate/tests.rs index b763d6b..f19ca2f 100644 --- a/stack-reorder-contract/.cargo-generate/tests.rs +++ b/stack-reorder-contract/.cargo-generate/tests.rs @@ -4,8 +4,7 @@ fn test_{{crate_name}}() { // deploy contract let mut context = Context::default(); - let contract_bin: Bytes = Loader::default().load_binary("{{project-name}}"); - let out_point = context.deploy_cell(contract_bin); + let out_point = context.deploy_cell_by_name("{{project-name}}"); // prepare scripts let lock_script = context diff --git a/workspace/Makefile b/workspace/Makefile index 36a9c7e..c0dfbe8 100644 --- a/workspace/Makefile +++ b/workspace/Makefile @@ -150,4 +150,139 @@ CHECKSUM_FILE := build/checksums-$(MODE).txt checksum: build shasum -a 256 build/$(MODE)/* > $(CHECKSUM_FILE) -.PHONY: build test check clippy fmt cargo clean prepare checksum +# ============================================================================ +# Coverage targets (using native-simulator mode) +# ============================================================================ +# +# Coverage works by: +# 1. Building simulator .so files with LLVM coverage instrumentation +# 2. Running tests with native-simulator feature (executes contracts natively) +# 3. Simulator .so files call __llvm_profile_write_file() on unload to flush coverage +# 4. Using llvm-cov to generate reports from the profraw files +# +# Requirements: rustup component add llvm-tools-preview +# +# Usage: +# make coverage # Generate text coverage report +# make coverage-html # Generate HTML coverage report +# make coverage-lcov # Generate LCOV format for CI integration + +LLVM_TOOLS_DIR := $(shell rustc --print sysroot)/lib/rustlib/$(shell rustc -vV | grep host | cut -d' ' -f2)/bin +LLVM_PROFDATA := $(LLVM_TOOLS_DIR)/llvm-profdata +LLVM_COV := $(LLVM_TOOLS_DIR)/llvm-cov +COVERAGE_DIR := target/coverage +PROFRAW_PATTERN := $(COVERAGE_DIR)/profraw + +# Dynamically find all simulator .so files +SIMULATOR_OBJECTS = $(wildcard build/debug/lib*_sim.so) +SIMULATOR_OBJECT_ARGS = $(foreach obj,$(SIMULATOR_OBJECTS),-object $(obj)) + +# Build simulators with coverage instrumentation +build-sim-cov: + @echo "Building native simulators with coverage instrumentation..." + @# First build the RISC-V contract binaries (these don't need coverage) + $(MAKE) build MODE=debug CLEAN_BUILD_DIR_FIRST=true + @# Clean simulator artifacts to force rebuild with coverage + @rm -f target/debug/deps/lib*_sim*.rlib + @rm -f target/debug/lib*_sim.so + @# Rebuild simulators with LLVM coverage instrumentation + @echo "Rebuilding simulators with LLVM coverage instrumentation..." + @for sim in $(wildcard native-simulators/*); do \ + simname=$$(basename $$sim); \ + RUSTFLAGS="-C instrument-coverage --cfg=coverage" \ + cargo build -p $$simname; \ + done + @# Copy the instrumented simulators to build/debug/ + @echo "Copying instrumented simulators to build/debug/..." + @for sim in $(wildcard native-simulators/*); do \ + simname=$$(basename $$sim); \ + libname=$$(echo $$simname | sed 's/-/_/g'); \ + if [ -f "target/debug/lib$${libname}.so" ]; then \ + cp "target/debug/lib$${libname}.so" build/debug/; \ + fi; \ + done + @echo "Done! Instrumented simulators are in build/debug/" + +# Build simulators for coverage (debug mode for better coverage data) +build-sim: + @echo "Building native simulators..." + $(MAKE) build MODE=debug CLEAN_BUILD_DIR_FIRST=true + +# Internal target: run tests and collect coverage data +coverage-run-tests: + @echo "Running tests with coverage (native-simulator mode)..." + @mkdir -p $(COVERAGE_DIR) + @rm -f $(PROFRAW_PATTERN)-*.profraw + @# Run all tests with native-simulator feature and coverage instrumentation + LLVM_PROFILE_FILE="$(CURDIR)/$(PROFRAW_PATTERN)-%p-%8m.profraw" \ + RUSTFLAGS="-C instrument-coverage" \ + MODE=debug cargo test --features native-simulator --package tests \ + -- --test-threads=1 2>&1 || true + @echo "Merging coverage data..." + @if ls $(PROFRAW_PATTERN)-*.profraw 1> /dev/null 2>&1; then \ + $(LLVM_PROFDATA) merge -sparse $(PROFRAW_PATTERN)-*.profraw -o $(COVERAGE_DIR)/coverage.profdata; \ + rm -f $(PROFRAW_PATTERN)-*.profraw; \ + else \ + echo "Warning: No profraw files found. Tests may have failed or no coverage data was generated."; \ + fi + @# Clean up stray default_*.profraw files created by child processes + @rm -f $(CURDIR)/default_*.profraw + +# Run tests with coverage and generate text report +coverage: build-sim-cov coverage-run-tests + @echo "" + @echo "=== Coverage Report ===" + @if [ -f "$(COVERAGE_DIR)/coverage.profdata" ]; then \ + $(LLVM_COV) report \ + $(SIMULATOR_OBJECT_ARGS) \ + -instr-profile=$(COVERAGE_DIR)/coverage.profdata \ + --ignore-filename-regex='\.cargo|rustc|rustlib'; \ + else \ + echo "Error: No coverage data found. Run 'make coverage-install' to install llvm-tools."; \ + fi + +# Generate HTML coverage report +coverage-html: build-sim-cov coverage-run-tests + @echo "Generating HTML coverage report..." + @mkdir -p $(COVERAGE_DIR)/html + @if [ -f "$(COVERAGE_DIR)/coverage.profdata" ]; then \ + $(LLVM_COV) show \ + $(SIMULATOR_OBJECT_ARGS) \ + -instr-profile=$(COVERAGE_DIR)/coverage.profdata \ + --ignore-filename-regex='\.cargo|rustc|rustlib' \ + --format=html --output-dir=$(COVERAGE_DIR)/html \ + --show-line-counts-or-regions --show-instantiations; \ + echo "Coverage report generated at: $(COVERAGE_DIR)/html/index.html"; \ + else \ + echo "Error: No coverage data found."; \ + fi + +# Generate LCOV format for CI integration +coverage-lcov: build-sim-cov coverage-run-tests + @echo "Generating LCOV coverage report..." + @if [ -f "$(COVERAGE_DIR)/coverage.profdata" ]; then \ + $(LLVM_COV) export \ + $(SIMULATOR_OBJECT_ARGS) \ + -instr-profile=$(COVERAGE_DIR)/coverage.profdata \ + --ignore-filename-regex='\.cargo|rustc|rustlib' \ + --format=lcov > $(COVERAGE_DIR)/lcov.info; \ + echo "LCOV report generated at: $(COVERAGE_DIR)/lcov.info"; \ + else \ + echo "Error: No coverage data found."; \ + fi + +# Install llvm-tools if not present +coverage-install: + @if ! rustup component list --installed | grep -q llvm-tools; then \ + echo "Installing llvm-tools-preview..."; \ + rustup component add llvm-tools-preview; \ + else \ + echo "llvm-tools-preview is already installed"; \ + fi + +# Clean coverage artifacts +coverage-clean: + rm -rf $(COVERAGE_DIR) + +.PHONY: build test check clippy fmt cargo clean prepare checksum \ + build-sim build-sim-cov coverage-run-tests coverage coverage-html coverage-lcov coverage-install coverage-clean diff --git a/workspace/tests/src/tests.rs b/workspace/tests/src/tests.rs index e6353c3..32e0c04 100644 --- a/workspace/tests/src/tests.rs +++ b/workspace/tests/src/tests.rs @@ -1,10 +1,4 @@ -use crate::Loader; -use ckb_testtool::ckb_types::{ - bytes::Bytes, - core::TransactionBuilder, - packed::*, - prelude::*, -}; +use ckb_testtool::ckb_types::{bytes::Bytes, core::TransactionBuilder, packed::*, prelude::*}; use ckb_testtool::context::Context; // Include your tests here