diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 21c0fe8..0716aa5 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -7,35 +7,53 @@ on: pull_request: schedule: - cron: '0 0 * * *' + # Allow release.yml to call this workflow + workflow_call: jobs: # - # Verify the build and installation of SDB. + # Consolidated linting job: pylint + ruff + yapf # - install: + # All linters run on a single Python version since they check code quality, + # not runtime behavior. + # + lint: runs-on: ubuntu-24.04 - strategy: - matrix: - python-version: ['3.10', '3.12'] steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: - python-version: ${{ matrix.python-version }} - - run: python3 -m pip install --upgrade pip setuptools wheel + python-version: '3.10' + - run: ./.github/scripts/install-drgn.sh + - run: python3 -m pip install pylint pytest ruff yapf - run: python3 -m pip install . - # - # The statement below is used for debugging the Github job. - # - - run: python3 --version - # - # Verify "pylint" runs successfully. - # - # Note, we need to have "drgn" installed in order to run "pylint". - # Thus, prior to running "pylint" we have to clone, build, and install + # + # Pylint checks + # + - name: Run pylint on sdb + run: pylint -d duplicate-code -d invalid-name -d missing-docstring -d import-outside-toplevel -d too-many-branches -d missing-module-docstring -d missing-function-docstring sdb + - name: Run pylint on tests + run: pylint -d duplicate-code -d invalid-name -d missing-docstring -d import-outside-toplevel -d too-many-branches -d missing-module-docstring -d missing-function-docstring tests + # + # Ruff checks (fast Python linter) + # + - name: Run ruff + run: ruff check sdb tests + # + # YAPF formatting check + # + - name: Check formatting with yapf (sdb) + run: yapf --diff --style google --recursive sdb + - name: Check formatting with yapf (tests) + run: yapf --diff --style google --recursive tests + # + # Type checking with mypy and pyright + # + # Note, we need to have "drgn" installed in order to run type checkers. + # Thus, prior to running them we have to clone, build, and install # the "drgn" from source (there's no package currently available). # - pylint: + type-check: runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v4 @@ -43,10 +61,23 @@ jobs: with: python-version: '3.10' - run: ./.github/scripts/install-drgn.sh - - run: python3 -m pip install pylint pytest + - run: python3 -m pip install mypy pyright pytest - run: python3 -m pip install . - - run: pylint -d duplicate-code -d invalid-name -d missing-docstring -d import-outside-toplevel -d too-many-branches -d missing-module-docstring -d missing-function-docstring sdb - - run: pylint -d duplicate-code -d invalid-name -d missing-docstring -d import-outside-toplevel -d too-many-branches -d missing-module-docstring -d missing-function-docstring tests + # + # Mypy checks + # Note: --ignore-missing-imports for tests because pytest doesn't provide stubs + # + - name: Run mypy on sdb + run: python3 -m mypy --strict --show-error-codes -p sdb + - name: Run mypy on tests + run: python3 -m mypy --strict --ignore-missing-imports --show-error-codes -p tests + # + # Pyright checks + # + - name: Run pyright on sdb + run: pyright sdb + - name: Run pyright on tests + run: pyright tests # # Verify "pytest" runs successfully on unit tests. # @@ -72,9 +103,6 @@ jobs: uses: codecov/codecov-action@v5 with: token: ${{ secrets.CODECOV_TOKEN }} - files: ./coverage.xml - fail_ci_if_error: false - verbose: true # # Verify "pytest" runs successfully on integration tests with crash dumps. # @@ -100,58 +128,27 @@ jobs: - run: ./.github/scripts/download-dumps-from-gdrive.sh - run: ./.github/scripts/extract-dump.sh dump.201912060006.tar.lzma - run: ./.github/scripts/extract-dump.sh dump.202303131823.tar.gz - - run: pytest -v --cov sdb --cov-report xml tests/integration + # + # Run integration tests. Use -s flag to show size comparison output + # for record-replay tests. Use set -o pipefail to ensure test failures + # are properly reported even when piping output. + # + - name: Run integration tests + run: | + set -o pipefail + pytest -v -s --cov sdb --cov-report xml tests/integration 2>&1 | tee test_output.txt + # + # Extract and display record-replay size comparison in GitHub summary + # + - name: Add size comparison to summary + if: always() + run: | + echo "## Record-Replay Size Comparison" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + grep -A 8 "DUMP SIZE COMPARISON" test_output.txt >> $GITHUB_STEP_SUMMARY || echo "No size comparison data found" >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY - name: Upload coverage to Codecov uses: codecov/codecov-action@v5 with: token: ${{ secrets.CODECOV_TOKEN }} - files: ./coverage.xml - fail_ci_if_error: false - verbose: true - # - # Verify "yapf" runs successfully. - # - yapf: - runs-on: ubuntu-24.04 - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: '3.10' - - run: python3 -m pip install yapf - - run: yapf --diff --style google --recursive sdb - - run: yapf --diff --style google --recursive tests - # - # Verify "ruff" runs successfully. - # - ruff: - runs-on: ubuntu-24.04 - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: '3.10' - - run: python3 -m pip install ruff - - run: ruff check sdb tests - # - # Verify "mypy" runs successfully. - # - # Note, we need to have "drgn" installed in order to run "mypy". - # Thus, prior to running "mypy" we have to clone, build, and install - # the "drgn" from source (there's no package currently available). - # - # Also note that we supply --ignore-missing-imports to the tests package - # because pytest doesn't provide stubs on typeshed. - # - mypy: - runs-on: ubuntu-24.04 - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: '3.10' - - run: ./.github/scripts/install-drgn.sh - - run: python3 -m pip install mypy pytest - - run: python3 -m pip install . - - run: python3 -m mypy --strict --show-error-codes -p sdb - - run: python3 -m mypy --strict --ignore-missing-imports --show-error-codes -p tests diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 168bf59..23441c9 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,8 +1,8 @@ # -# Release workflow - triggered on version tag push (e.g., v0.2.0) or manually +# Release workflow - triggered on version tag push (e.g., v0.2.0) # # This workflow: -# 1. Runs all CI checks (lint, type-check, tests) +# 1. Runs all CI checks via the main workflow # 2. Builds the package (sdist + wheel) # 3. Publishes to PyPI using Trusted Publishing (OIDC) # 4. Creates a GitHub Release with auto-generated notes @@ -12,18 +12,7 @@ name: Release on: push: tags: - - 'v[0-9]+.[0-9]+.[0-9]+' - - 'v[0-9]+.[0-9]+.[0-9]+-*' # Allow pre-release tags like v0.2.0-beta1 - workflow_dispatch: - inputs: - tag: - description: 'Tag to release (e.g., v0.2.0)' - required: true - type: string - -env: - # Use input tag for workflow_dispatch, or ref_name for tag push - RELEASE_TAG: ${{ inputs.tag || github.ref_name }} + - 'v*' # Required for PyPI Trusted Publishing (OIDC) permissions: @@ -32,87 +21,21 @@ permissions: jobs: # - # Run all CI checks before releasing + # Run all CI checks by calling the main workflow # - lint: - runs-on: ubuntu-24.04 - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ inputs.tag || github.ref_name }} - - uses: actions/setup-python@v5 - with: - python-version: '3.10' - - run: ./.github/scripts/install-drgn.sh - - run: python3 -m pip install pylint pytest ruff yapf - - run: pylint -d duplicate-code -d invalid-name -d missing-docstring -d import-outside-toplevel -d too-many-branches -d missing-module-docstring -d missing-function-docstring sdb - - run: pylint -d duplicate-code -d invalid-name -d missing-docstring -d import-outside-toplevel -d too-many-branches -d missing-module-docstring -d missing-function-docstring tests - - run: ruff check sdb tests - - run: yapf --diff --style google --recursive sdb - - run: yapf --diff --style google --recursive tests - - type-check: - runs-on: ubuntu-24.04 - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ inputs.tag || github.ref_name }} - - uses: actions/setup-python@v5 - with: - python-version: '3.10' - - run: ./.github/scripts/install-drgn.sh - - run: python3 -m pip install mypy pytest - - run: python3 -m mypy --strict --show-error-codes -p sdb - - run: python3 -m mypy --strict --ignore-missing-imports --show-error-codes -p tests - - test-unit: - runs-on: ubuntu-24.04 - strategy: - matrix: - python-version: ['3.10', '3.12'] - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ inputs.tag || github.ref_name }} - - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - run: python3 -m pip install pytest pytest-cov - - run: ./.github/scripts/install-drgn.sh - - run: pytest -v --cov sdb --cov-report xml tests/unit - - test-integration: - runs-on: ubuntu-24.04 - strategy: - matrix: - python-version: ['3.10', '3.12'] - env: - AWS_DEFAULT_REGION: 'us-west-2' - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ inputs.tag || github.ref_name }} - - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - run: python3 -m pip install pytest pytest-cov - - run: ./.github/scripts/install-libkdumpfile.sh - - run: ./.github/scripts/install-drgn.sh - - run: ./.github/scripts/download-dumps-from-gdrive.sh - - run: ./.github/scripts/extract-dump.sh dump.201912060006.tar.lzma - - run: ./.github/scripts/extract-dump.sh dump.202303131823.tar.gz - - run: pytest -v --cov sdb --cov-report xml tests/integration + ci: + uses: ./.github/workflows/main.yml + secrets: inherit # # Build the package # build: - needs: [lint, type-check, test-unit, test-integration] + needs: ci runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v4 with: - ref: ${{ inputs.tag || github.ref_name }} fetch-depth: 0 # Required for setuptools_scm to get version from tags - uses: actions/setup-python@v5 with: @@ -154,18 +77,19 @@ jobs: steps: - uses: actions/checkout@v4 with: - ref: ${{ inputs.tag || github.ref_name }} fetch-depth: 0 # Required for generating release notes - name: Download build artifacts uses: actions/download-artifact@v4 with: name: dist path: dist/ + - name: Get version from tag + id: version + run: echo "VERSION=${GITHUB_REF#refs/tags/v}" >> $GITHUB_OUTPUT - name: Create GitHub Release + uses: softprops/action-gh-release@v2 + with: + name: sdb ${{ steps.version.outputs.VERSION }} + generate_release_notes: true env: - GITHUB_TOKEN: ${{ github.token }} - run: | - gh release create '${{ env.RELEASE_TAG }}' \ - --title 'sdb ${{ env.RELEASE_TAG }}' \ - --generate-notes \ - dist/* + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/pyproject.toml b/pyproject.toml index 1efc2ca..fdb052c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -126,6 +126,20 @@ ignore_missing_imports = true module = "sdb._version" ignore_missing_imports = true +# Pyright configuration +[tool.pyright] +pythonVersion = "3.10" +typeCheckingMode = "basic" +# drgn.Object is designed to be used interchangeably with int in many contexts +# These are intentional design patterns, not bugs +reportArgumentType = "warning" +reportReturnType = "warning" +# Dynamic attributes added by decorators (e.g., input_typename_handled) +reportFunctionMemberAccess = "none" +reportAttributeAccessIssue = "warning" +# Operator type issues in tests (drgn.Type.size can be None but is always int for our types) +reportOperatorIssue = "warning" + # Pytest configuration [tool.pytest.ini_options] testpaths = ["tests"] diff --git a/sdb/__init__.py b/sdb/__init__.py index 29dca16..7b6c1d0 100644 --- a/sdb/__init__.py +++ b/sdb/__init__.py @@ -34,6 +34,10 @@ # the modules are imported and attempt to have a cleaner # separation of concerns between modules. # +# Export submodules for direct access (e.g., sdb.target.create_object()) +from sdb import error +from sdb import target + from sdb.error import ( Error, CommandNotFoundError, @@ -88,6 +92,8 @@ '__version__', 'Address', 'All', + 'error', + 'target', 'Cast', 'Command', 'CommandArgumentsError', diff --git a/sdb/command.py b/sdb/command.py index fb4d44f..52cd4f5 100644 --- a/sdb/command.py +++ b/sdb/command.py @@ -171,9 +171,9 @@ def _init_parser(cls, name: str) -> argparse.ArgumentParser: # may be calling splitlines() for None. # if cls.__doc__: - summary = ( - inspect.getdoc( # type: ignore[union-attr] - cls).splitlines()[0].strip()) + docstring = inspect.getdoc(cls) + assert docstring is not None # guaranteed by if cls.__doc__ + summary = docstring.splitlines()[0].strip() else: summary = None return argparse.ArgumentParser(prog=name, description=summary) @@ -293,8 +293,9 @@ def help(cls, name: str) -> None: # already be included in the parser description. The second # line should be empty. Thus, we skip these two lines. # - for line in inspect.getdoc( # type: ignore[union-attr] - cls).splitlines()[2:]: + docstring = inspect.getdoc(cls) + assert docstring is not None # guaranteed by if cls.__doc__ + for line in docstring.splitlines()[2:]: print(f"{line}") print() diff --git a/sdb/commands/spl/internal/kmem_helpers.py b/sdb/commands/spl/internal/kmem_helpers.py index 7eda7ca..ea9b84f 100644 --- a/sdb/commands/spl/internal/kmem_helpers.py +++ b/sdb/commands/spl/internal/kmem_helpers.py @@ -71,8 +71,9 @@ def slab_linux_cache_source(cache: drgn.Object) -> str: def for_each_slab_flag_in_cache(cache: drgn.Object) -> Iterable[str]: assert sdb.type_canonical_name(cache.type_) == 'struct spl_kmem_cache *' flag = cache.skc_flags.value_() - for enum_entry, enum_entry_bit in cache.prog_.type( - 'enum kmc_bit').enumerators: + enum_type = cache.prog_.type('enum kmc_bit') + assert enum_type.enumerators is not None + for enum_entry, enum_entry_bit in enum_type.enumerators: if flag & (1 << enum_entry_bit): yield enum_entry.replace('_BIT', '') diff --git a/sdb/commands/zfs/arc.py b/sdb/commands/zfs/arc.py index abe0008..253a8b1 100644 --- a/sdb/commands/zfs/arc.py +++ b/sdb/commands/zfs/arc.py @@ -28,7 +28,9 @@ class ARCStats(sdb.Locator, sdb.PrettyPrinter): @staticmethod def print_stats(obj: drgn.Object) -> None: - names = [memb.name for memb in sdb.get_type('struct arc_stats').members] + arc_stats_type = sdb.get_type('struct arc_stats') + assert arc_stats_type.members is not None + names = [memb.name for memb in arc_stats_type.members] for name in names: print(f"{name:32} = {int(obj.member_(name).value.ui64)}") diff --git a/sdb/commands/zfs/metaslab.py b/sdb/commands/zfs/metaslab.py index 9546be3..3c69725 100644 --- a/sdb/commands/zfs/metaslab.py +++ b/sdb/commands/zfs/metaslab.py @@ -187,10 +187,11 @@ def from_vdev(self, vdev: drgn.Object) -> Iterable[drgn.Object]: for i in self.args.metaslab_ids: if i >= vdev.vdev_ms_count: ms_count = int(vdev.vdev_ms_count) - vdev = int(vdev.vdev_id) + vdev_id = int(vdev.vdev_id) raise sdb.CommandError( self.name, f"metaslab id {i} not valid; " - f"there are only {ms_count} metaslabs in vdev {vdev}") + f"there are only {ms_count} metaslabs in vdev {vdev_id}" + ) yield vdev.vdev_ms[i] else: for i in range(int(vdev.vdev_ms_count)): diff --git a/sdb/commands/zfs/range_tree.py b/sdb/commands/zfs/range_tree.py index 001ec56..dfdd6bb 100644 --- a/sdb/commands/zfs/range_tree.py +++ b/sdb/commands/zfs/range_tree.py @@ -73,7 +73,9 @@ class RangeSeg(sdb.Locator): @sdb.InputHandler('range_tree_t *') def from_range_tree(self, rt: drgn.Object) -> Iterable[drgn.Object]: - enum_dict = dict(sdb.get_type('enum range_seg_type').enumerators) + enum_type = sdb.get_type('enum range_seg_type') + assert enum_type.enumerators is not None + enum_dict: dict[str, int] = dict(enum_type.enumerators) # pyright: ignore range_seg_type_to_type = { enum_dict['RANGE_SEG32']: 'range_seg32_t*', enum_dict['RANGE_SEG64']: 'range_seg64_t*', diff --git a/sdb/commands/zfs/spa.py b/sdb/commands/zfs/spa.py index 4b19057..f050a22 100644 --- a/sdb/commands/zfs/spa.py +++ b/sdb/commands/zfs/spa.py @@ -78,7 +78,7 @@ def pretty_print(self, objs: Iterable[drgn.Object]) -> None: vdevs = sdb.execute_pipeline([spa], [Vdev()]) Vdev(self.arg_list).print_indented(vdevs, 5) - def no_input(self) -> drgn.Object: + def no_input(self) -> Iterable[drgn.Object]: spas = sdb.execute_pipeline( [sdb.get_object("spa_namespace_avl").address_of_()], [Avl(), sdb.Cast(["spa_t *"])], diff --git a/sdb/commands/zfs/zio.py b/sdb/commands/zfs/zio.py index 1891053..83e5114 100644 --- a/sdb/commands/zfs/zio.py +++ b/sdb/commands/zfs/zio.py @@ -128,7 +128,7 @@ def zio_has_parents(zio: drgn.Object) -> bool: return True return False - def no_input(self) -> drgn.Object: + def no_input(self) -> Iterable[drgn.Object]: if self.args.parents: raise sdb.CommandInvalidInputError( self.name, "command argument -p is not applicable " + diff --git a/sdb/session.py b/sdb/session.py index d143c3b..b261500 100644 --- a/sdb/session.py +++ b/sdb/session.py @@ -412,6 +412,8 @@ def _capture_pointer(self, obj: drgn.Object, depth: int) -> None: def _capture_struct_pointers(self, obj: drgn.Object, depth: int) -> None: """Capture pointer members of a struct.""" + if obj.type_.members is None: + return for member in obj.type_.members: if member.name: try: