adds support for test coverage for all test types

Author: ag

.gitignore

@@ -40,3 +40,6 @@ tags
 
 # ProxySQL log files
 test/proxysql/*.log*
+
+# Coverage files
+test/coverage

README.md

@@ -133,6 +133,12 @@ To run all integration tests:
 docker compose run --use-aliases boulder ./test.sh --integration
 ```
 
+To run unit tests and integration tests with coverage:
+
+```shell
+docker compose run --use-aliases boulder ./test.sh --unit --integration --coverage --coverage-dir=./test/coverage/mytestrun
+```
+
 To run specific integration tests (example runs TestAkamaiPurgerDrainQueueFails and TestWFECORS):
 
 ```shell

test.sh

@@ -19,6 +19,8 @@ UNIT_PACKAGES=()
 UNIT_FLAGS=()
 INTEGRATION_FLAGS=()
 FILTER=()
+COVERAGE="false"
+COVERAGE_DIR="test/coverage/$(date +%Y-%m-%d_%H-%M-%S)"
 
 #
 # Cleanup Functions
@@ -105,6 +107,9 @@ With no options passed, runs standard battery of tests (lint, unit, and integrat
     -i, --integration                     Adds integration to the list of tests to run
     -s, --start-py                        Adds start to the list of tests to run
     -g, --generate                        Adds generate to the list of tests to run
+    -c, --coverage                        Enables coverage for tests
+    -d <DIR>, --coverage-directory=<DIR>  Directory to store coverage files in
+                                          Default: test/coverage/<timestamp>
     -f <REGEX>, --filter=<REGEX>          Run only those tests matching the regular expression
 
                                           Note:
@@ -120,7 +125,7 @@ With no options passed, runs standard battery of tests (lint, unit, and integrat
 EOM
 )"
 
-while getopts luvwecismgnhp:f:-: OPT; do
+while getopts luvwecisgnhd:p:f:-: OPT; do
   if [ "$OPT" = - ]; then     # long option: reformulate OPT and OPTARG
     OPT="${OPTARG%%=*}"       # extract long option name
     OPTARG="${OPTARG#$OPT}"   # extract long option argument (may be empty)
@@ -138,6 +143,8 @@ while getopts luvwecismgnhp:f:-: OPT; do
     s | start-py )                   RUN+=("start") ;;
     g | generate )                   RUN+=("generate") ;;
     n | config-next )                BOULDER_CONFIG_DIR="test/config-next" ;;
+    c | coverage )                   COVERAGE="true" ;;
+    d | coverage-dir )               check_arg; COVERAGE_DIR="${OPTARG}" ;;
     h | help )                       print_usage_exit ;;
     ??* )                            exit_msg "Illegal option --$OPT" ;;  # bad long option
     ? )                              exit 2 ;;  # bad short option (error reported via getopts)
@@ -197,10 +204,15 @@ settings="$(cat -- <<-EOM
     UNIT_PACKAGES:      ${UNIT_PACKAGES[@]}
     UNIT_FLAGS:         ${UNIT_FLAGS[@]}
     FILTER:             ${FILTER[@]}
-
+    COVERAGE:           $COVERAGE
+    COVERAGE_DIR:       $COVERAGE_DIR
 EOM
 )"
 
+if [ "${COVERAGE}" == "true" ]; then
+  mkdir -p "$COVERAGE_DIR"
+fi
+
 echo "$settings"
 print_heading "Starting..."
 
@@ -226,6 +238,12 @@ STAGE="unit"
 if [[ "${RUN[@]}" =~ "$STAGE" ]] ; then
   print_heading "Running Unit Tests"
   flush_redis
+
+  if [ "${COVERAGE}" == "true" ]; then
+    UNIT_CSV=$(IFS=,; echo "${UNIT_PACKAGES[*]}")
+    UNIT_FLAGS+=("-cover" "-covermode=atomic" "-coverprofile=${COVERAGE_DIR}/unit.coverprofile" "-coverpkg=${UNIT_CSV}")
+  fi
+
   run_unit_tests
 fi
 
@@ -236,11 +254,27 @@ STAGE="integration"
 if [[ "${RUN[@]}" =~ "$STAGE" ]] ; then
   print_heading "Running Integration Tests"
   flush_redis
+
+  # Set up test parameters
+  INTEGRATION_ARGS=("--chisel")
+
+  # Add verbose flag if requested
   if [[ "${INTEGRATION_FLAGS[@]}" =~ "-v" ]] ; then
-    python3 test/integration-test.py --chisel --gotestverbose "${FILTER[@]}"
+    INTEGRATION_ARGS+=("--gotestverbose")
   else
-    python3 test/integration-test.py --chisel --gotest "${FILTER[@]}"
+    INTEGRATION_ARGS+=("--gotest")
+  fi
+
+  # Add coverage settings if enabled
+  if [ "${COVERAGE}" == "true" ]; then
+    INTEGRATION_ARGS+=("--coverage" "--coverage-dir=${COVERAGE_DIR}")
   fi
+
+  # Add any filters
+  INTEGRATION_ARGS+=("${FILTER[@]}")
+
+  # Run the integration tests with all collected arguments
+  python3 test/integration-test.py "${INTEGRATION_ARGS[@]}"
 fi
 
 # Test that just ./start.py works, which is a proxy for testing that

test/integration-test.py

@@ -10,23 +10,15 @@
 import argparse
 import datetime
 import inspect
-import json
 import os
-import random
 import re
-import requests
 import subprocess
-import shlex
-import signal
-import time
 
+import requests
 import startservers
-
 import v2_integration
 from helpers import *
 
-from acme import challenges
-
 # Set the environment variable RACE to anything other than 'true' to disable
 # race detection. This significantly speeds up integration testing cycles
 # locally.
@@ -55,6 +47,11 @@ def main():
     parser = argparse.ArgumentParser(description='Run integration tests')
     parser.add_argument('--chisel', dest="run_chisel", action="store_true",
                         help="run integration tests using chisel")
+    parser.add_argument('--coverage', dest="coverage", action="store_true",
+                        help="run integration tests with coverage")
+    parser.add_argument('--coverage-dir', dest="coverage_dir", action="store",
+                        default=f"test/coverage/{datetime.datetime.now().strftime('%Y-%m-%d_%H-%M-%S')}",
+                        help="directory to store coverage data")
     parser.add_argument('--gotest', dest="run_go", action="store_true",
                         help="run Go integration tests")
     parser.add_argument('--gotestverbose', dest="run_go_verbose", action="store_true",
@@ -64,24 +61,45 @@ def main():
     # allow any ACME client to run custom command for integration
     # testing (without having to implement its own busy-wait loop)
     parser.add_argument('--custom', metavar="CMD", help="run custom command")
-    parser.set_defaults(run_chisel=False, test_case_filter="", skip_setup=False)
+    parser.set_defaults(run_chisel=False, test_case_filter="", skip_setup=False, coverage=False, coverage_dir=None)
     args = parser.parse_args()
 
+    if args.coverage and args.coverage_dir:
+        if not os.path.exists(args.coverage_dir):
+            os.makedirs(args.coverage_dir)
+        if not os.path.isdir(args.coverage_dir):
+            raise(Exception("coverage-dir must be a directory"))
+
     if not (args.run_chisel or args.custom  or args.run_go is not None):
         raise(Exception("must run at least one of the letsencrypt or chisel tests with --chisel, --gotest, or --custom"))
 
-    if not startservers.install(race_detection=race_detection):
+    if not startservers.install(race_detection=race_detection, coverage=args.coverage):
         raise(Exception("failed to build"))
 
-    if not startservers.start():
+    if not args.test_case_filter:
+        now = datetime.datetime.utcnow()
+
+        six_months_ago = now+datetime.timedelta(days=-30*6)
+        if not startservers.start(fakeclock=fakeclock(six_months_ago), coverage_dir=args.coverage_dir):
+            raise(Exception("startservers failed (mocking six months ago)"))
+        setup_six_months_ago()
+        startservers.stop()
+
+        twenty_days_ago = now+datetime.timedelta(days=-20)
+        if not startservers.start(fakeclock=fakeclock(twenty_days_ago), coverage_dir=args.coverage_dir):
+            raise(Exception("startservers failed (mocking twenty days ago)"))
+        setup_twenty_days_ago()
+        startservers.stop()
+
+    if not startservers.start(fakeclock=None, coverage_dir=args.coverage_dir):
         raise(Exception("startservers failed"))
 
     if args.run_chisel:
         run_chisel(args.test_case_filter)
 
     if args.run_go:
         run_go_tests(args.test_case_filter, False)
-    
+
     if args.run_go_verbose:
         run_go_tests(args.test_case_filter, True)
 
@@ -94,6 +112,10 @@ def main():
         run_cert_checker()
         check_balance()
 
+    # If coverage is enabled, process the coverage data
+    if args.coverage:
+        process_covdata(args.coverage_dir)
+
     if not startservers.check():
         raise(Exception("startservers.check failed"))
 
@@ -128,12 +150,24 @@ def check_balance():
     ]
     for address in addresses:
         metrics = requests.get("http://%s/metrics" % address)
-        if not "grpc_server_handled_total" in metrics.text:
+        if "grpc_server_handled_total" not in metrics.text:
             raise(Exception("no gRPC traffic processed by %s; load balancing problem?")
                 % address)
 
 def run_cert_checker():
     run(["./bin/boulder", "cert-checker", "-config", "%s/cert-checker.json" % config_dir])
 
+def process_covdata(coverage_dir):
+    """Process coverage data and generate reports."""
+    if not os.path.exists(coverage_dir):
+        raise(Exception("Coverage directory does not exist: %s" % coverage_dir))
+
+    # Generate text report
+    coverage_dir = os.path.abspath(coverage_dir)
+    cov_text = os.path.join(coverage_dir, "integration.coverprofile")
+    # this works, but if it takes a long time consider merging with `go tool covdata merge` first
+    # https://go.dev/blog/integration-test-coverage#merging-raw-profiles-with-go-tool-covdata-merge
+    run(["go", "tool", "covdata", "textfmt", "-i", coverage_dir, "-o", cov_text])
+
 if __name__ == "__main__":
     main()

test/startservers.py

@@ -1,16 +1,11 @@
 import atexit
 import collections
 import os
-import shutil
 import signal
 import socket
 import subprocess
-import sys
-import tempfile
-import threading
-import time
 
-from helpers import waithealth, waitport, config_dir, CONFIG_NEXT
+from helpers import config_dir, waithealth, waitport
 
 Service = collections.namedtuple('Service', ('name', 'debug_port', 'grpc_port', 'host_override', 'cmd', 'deps'))
 
@@ -130,7 +125,7 @@
         None),
     Service('pardot-test-srv',
         # Uses port 9601 to mock Salesforce OAuth2 token API and 9602 to mock
-        # the Pardot API. 
+        # the Pardot API.
         9601, None, None,
         ('./bin/pardot-test-srv', '--config', os.path.join(config_dir, 'pardot-test-srv.json'),),
         None),
@@ -186,24 +181,33 @@ def _service_toposort(services):
 # to run the load-generator).
 challSrvProcess = None
 
-def install(race_detection):
+def install(race_detection, coverage=False):
     # Pass empty BUILD_TIME and BUILD_ID flags to avoid constantly invalidating the
     # build cache with new BUILD_TIMEs, or invalidating it on merges with a new
     # BUILD_ID.
     go_build_flags='-tags "integration"'
     if race_detection:
         go_build_flags += ' -race'
 
+    if coverage:
+        go_build_flags += ' -cover' # https://go.dev/blog/integration-test-coverage
+
     return subprocess.call(["/usr/bin/make", "GO_BUILD_FLAGS=%s" % go_build_flags]) == 0
 
-def run(cmd):
+def run(cmd, fakeclock, coverage_dir=None):
     e = os.environ.copy()
     e.setdefault("GORACE", "halt_on_error=1")
+    if coverage_dir:
+        abs_coverage_dir = os.path.abspath(coverage_dir)
+        e.setdefault("GOCOVERDIR", abs_coverage_dir)
+        e.setdefault("GOCOVERMODE", "atomic")
+    if fakeclock:
+        e.setdefault("FAKECLOCK", fakeclock)
     p = subprocess.Popen(cmd, env=e)
     p.cmd = cmd
     return p
 
-def start():
+def start(fakeclock, coverage_dir=None):
     """Return True if everything builds and starts.
 
     Give up and return False if anything fails to build, or dies at
@@ -232,7 +236,7 @@ def start():
         print("Starting service", service.name)
         try:
             global processes
-            p = run(service.cmd)
+            p = run(service.cmd, fakeclock, coverage_dir)
             processes.append(p)
             if service.grpc_port is not None:
                 waithealth(' '.join(p.args), service.grpc_port, service.host_override)