Include gpu and example tests also in codecov coverage reporting and enable omitted folder coverage (#1154)

So far, we only measured unit test coverage but we also have gpu test
and example tests which needed to be setup differently to track in
overall codecov coverage so we get accurate coverage reporting

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

* **Chores**
* Strengthened code coverage measurement with parallelized collection
across unit, GPU, and example test suites
* Enhanced continuous integration workflow configuration with improved
coverage reporting and threshold management
* Updated testing infrastructure dependencies and settings to support
more robust quality assurance processes
<!-- end of auto-generated comment: release notes by coderabbit.ai -->

Signed-off-by: Keval Morabia <28916987+kevalmorabia97@users.noreply.github.com>

Author: kevalmorabia97

.github/codecov.yml

@@ -1,8 +1,13 @@
-# Allow atmost 5% coverage drop from main branch.
+# Flags partition coverage by test suite. carryforward ensures that if GPU tests are skipped
+# on a PR (no relevant file changes), their coverage from the last nightly run is reused so
+# the comparison is not penalized for the missing upload.
+flag_management:
+  default_rules:
+    carryforward: true
 coverage:
   status:
     project:
       default:
         target: auto
-        threshold: 5%
+        threshold: 2% # Allow atmost 2% coverage drop from main branch.
     patch: false

.github/workflows/_example_tests_runner.yml

@@ -61,6 +61,19 @@ jobs:
 
           find examples/${{ inputs.example }} -name "requirements.txt" | while read req_file; do python -m pip install -r "$req_file" || exit 1; done
       - name: Run tests
+        env:
+          # Absolute paths so subprocesses running from different working directories
+          # all find the config and write .coverage.* files to the same location.
+          COVERAGE_PROCESS_START: ${{ github.workspace }}/pyproject.toml
+          COVERAGE_FILE: ${{ github.workspace }}/.coverage
         run: |
           echo "Running tests for: ${{ inputs.example }}"
-          pytest tests/examples/${{ inputs.example }}
+          pytest tests/examples/${{ inputs.example }} --cov
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v5
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          files: coverage.xml
+          flags: examples
+          fail_ci_if_error: false # test may be skipped if relevant file changes are not detected
+          verbose: true

.github/workflows/gpu_tests.yml

@@ -86,7 +86,20 @@ jobs:
         run: |
           echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/include:/usr/lib/x86_64-linux-gnu" >> $GITHUB_ENV
       - name: Run gpu tests
-        run: pip install tox-current-env && tox -e cuda13-${{ matrix.example }} --current-env
+        env:
+          COVERAGE_PROCESS_START: ${{ github.workspace }}/pyproject.toml
+          COVERAGE_FILE: ${{ github.workspace }}/.coverage
+        run: |
+          pip install tox-current-env
+          COV_ARGS="--cov" tox -e cuda13-${{ matrix.example }} --current-env
+      - name: Upload GPU coverage to Codecov
+        uses: codecov/codecov-action@v5
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          files: coverage.xml
+          flags: gpu
+          fail_ci_if_error: false # test may be skipped if relevant file changes are not detected
+          verbose: true
   gpu-tests-non-pr:
     if: ${{ !startsWith(github.ref, 'refs/heads/pull-request/') }}
     strategy: *gpu_strategy

.github/workflows/unit_tests.yml

@@ -43,6 +43,7 @@ jobs:
         uses: codecov/codecov-action@v5
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
+          flags: unit
           fail_ci_if_error: true
           verbose: true
   windows:

pyproject.toml

@@ -101,6 +101,7 @@ dev-docs = [
     "sphinx-togglebutton>=0.3.2",
 ]
 dev-test = [
+    "coverage[toml]>=7.13.0",  # a1_coverage.pth for subprocess tracking requires this
     "pytest",
     "pytest-cov",
     "pytest-instafail",
@@ -132,7 +133,7 @@ managed = true
 
 
 ####################################################################################################
-###############################  LINTING, FORMATTING AND TESTING CONFIGURATION  ####################
+###############################  LINTING, FORMATTING CONFIGURATION  ################################
 ####################################################################################################
 [tool.ruff]
 target-version = "py310"
@@ -256,6 +257,22 @@ ignore_errors = true
 module = ["examples.*"]
 disable_error_code = ["attr-defined"]
 
+[tool.bandit]
+exclude_dirs = [".github/", "examples/", "tests/"]
+# Do not change `skips`. It should be consistent with NVIDIA's Wheel-CI-CD bandit.yml config.
+# Use of `# nosec BXXX` requires special approval
+skips = [
+    "B101", # assert_used
+    "B110", # try_except_pass
+    "B112", # try_except_continue
+    "B303", # MD2, MD4, MD5, or SHA1
+    "B311", # random
+]
+
+
+####################################################################################################
+###############################  TESTING CONFIGURATION  ############################################
+####################################################################################################
 [tool.pytest.ini_options]
 # Default additional options
 # Show a short test summary info for all except passed tests with -ra flag
@@ -270,12 +287,12 @@ markers = [
 
 [tool.coverage.run]
 branch = false
-include = ["modelopt/*"]
-omit = ["*/plugins/*", "*/export/*"]
+parallel = true
+concurrency = ["multiprocessing", "thread"]
+source_pkgs = ["modelopt", "modelopt_recipes"]
 
 
 [tool.coverage.report]
-fail_under = 70
 skip_covered = true
 ignore_errors = true
 exclude_lines = [
@@ -297,16 +314,3 @@ exclude_lines = [
     # Don't complain about abstract methods, they aren't run
     "@(abc\\.)?abstractmethod",
 ]
-
-
-[tool.bandit]
-exclude_dirs = [".github/", "examples/", "tests/"]
-# Do not change `skips`. It should be consistent with NVIDIA's Wheel-CI-CD bandit.yml config.
-# Use of `# nosec BXXX` requires special approval
-skips = [
-    "B101", # assert_used
-    "B110", # try_except_pass
-    "B112", # try_except_continue
-    "B303", # MD2, MD4, MD5, or SHA1
-    "B311", # random
-]

tox.ini

@@ -67,8 +67,7 @@ commands_pre =
     pip uninstall -y cupy-cuda12x
     pip install cupy-cuda13x
 commands =
-    # Coverage fails with "Can't combine line data with arc data" error so not using "--cov"
-    python -m pytest tests/gpu
+    python -m pytest tests/gpu {env:COV_ARGS:}
 
 [testenv:cuda13-gpu-megatron]
 commands_pre =
@@ -78,16 +77,15 @@ commands_pre =
     pip install --no-build-isolation git+https://github.com/Dao-AILab/causal-conv1d.git
     pip install -e .[hf,dev-test]
 commands =
-    # Coverage fails with "Can't combine line data with arc data" error so not using "--cov"
-    python -m pytest tests/gpu_megatron
+    python -m pytest tests/gpu_megatron {env:COV_ARGS:}
 
 [testenv:cuda13-gpu-trtllm]
 # Expected to be run in TRT-LLM container
 commands_pre =
     # Install deps here so that it gets installed even in --current-env
     pip install -e .[hf,dev-test]
 commands =
-    python -m pytest tests/gpu_trtllm
+    python -m pytest tests/gpu_trtllm {env:COV_ARGS:}
 
 #############################################
 # Code quality checks on all files or on diff