diff --git a/.cargo/config.toml b/.cargo/config.toml index fe6d2a7f4..1c6481dcd 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -3,4 +3,4 @@ pipelining = true [target.aarch64-metta-none-eabi] -runner = "cargo make qemu-test-runner" +runner = "just _test-runner" diff --git a/.github/dependabot.yaml b/.github/dependabot.yaml new file mode 100644 index 000000000..cadefb93c --- /dev/null +++ b/.github/dependabot.yaml @@ -0,0 +1,7 @@ +version: 2 +updates: + - package-ecosystem: cargo + directory: "/" + schedule: + interval: daily + open-pull-requests-limit: 10 diff --git a/.github/dependabot.yml b/.github/dependabot.yml deleted file mode 100644 index 20153e95b..000000000 --- a/.github/dependabot.yml +++ /dev/null @@ -1,21 +0,0 @@ -version: 2 -updates: -- package-ecosystem: cargo - directory: "/" - schedule: - interval: daily - open-pull-requests-limit: 10 - ignore: - - dependency-name: cortex-a - versions: - - 5.1.2 - - 5.1.3 - - 5.1.4 - - 5.1.5 - - 5.1.6 - - dependency-name: qemu-exit - versions: - - 1.0.2 - - dependency-name: register - versions: - - 1.0.2 diff --git a/.github/shared/qemu/action.yaml b/.github/shared/qemu/action.yaml new file mode 100644 index 000000000..08a362d30 --- /dev/null +++ b/.github/shared/qemu/action.yaml @@ -0,0 +1,43 @@ +name: "QEMU" +description: "Install QEMU for particular platform" +runs: + using: "composite" + steps: + - name: Install QEMU (Linux) + if: runner.os == 'Linux' + shell: "bash" + run: | + sudo apt-get update + sudo apt-get install --fix-missing qemu-system-aarch64 + + - name: Install QEMU (macOS) + if: runner.os == 'macOS' + shell: "bash" + run: brew install qemu + env: + HOMEBREW_NO_AUTO_UPDATE: 1 + HOMEBREW_NO_BOTTLE_SOURCE_FALLBACK: 1 + HOMEBREW_NO_INSTALL_CLEANUP: 1 + + - name: Install Scoop (Windows) + if: runner.os == 'Windows' + shell: pwsh + run: | + Invoke-WebRequest -UseBasicParsing get.scoop.sh -outfile 'install.ps1' + .\install.ps1 -RunAsAdmin + echo "$HOME\scoop\shims" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append + + - name: Add custom Scoop bucket for QEMU (Windows) + if: runner.os == 'Windows' + shell: pwsh + run: | + scoop bucket add scoop-for-ci https://github.com/metta-systems/scoop-for-ci + + - name: Install QEMU (Windows) + if: runner.os == 'Windows' + shell: pwsh + run: scoop install qemu-1010 + + - name: "Print QEMU Version" + shell: ${{ runner.os == 'Windows' && 'pwsh' || 'bash' }} + run: qemu-system-aarch64 --version diff --git a/.github/shared/setup/action.yaml b/.github/shared/setup/action.yaml new file mode 100644 index 000000000..3dc435db0 --- /dev/null +++ b/.github/shared/setup/action.yaml @@ -0,0 +1,30 @@ +name: "Prepare" +description: "Prepare Rust build environment and dependencies" +runs: + using: "composite" + steps: + - name: "Install nightly Rust" + uses: actions-rust-lang/setup-rust-toolchain@v1.15.2 + with: + rustflags: "" + - name: "Install Just" + uses: "taiki-e/install-action@just" + - name: "Print Rust Version" + shell: ${{ runner.os == 'Windows' && 'pwsh' || 'bash' }} + run: | + rustc -Vv + cargo -Vv + - name: "Install build tools" + shell: ${{ runner.os == 'Windows' && 'pwsh' || 'bash' }} + run: cargo install cargo-binutils + - name: "Validate rust-lld" + if: runner.os == 'macOS' + shell: "bash" + run: | + which rust-lld || echo "Not found" + otool -L ~/.cargo/bin/rust-lld + - name: "Print Tools Versions" + shell: ${{ runner.os == 'Windows' && 'pwsh' || 'bash' }} + run: | + just --version + cargo objcopy --version diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml new file mode 100644 index 000000000..fd75fff1e --- /dev/null +++ b/.github/workflows/build.yaml @@ -0,0 +1,52 @@ +name: Build + +on: + push: + branches: + - "*" + pull_request: + +jobs: + check_formatting: + name: "Check Formatting" + runs-on: ubuntu-latest + timeout-minutes: 2 + steps: + - name: "Checkout Repository" + uses: actions/checkout@v6.0.2 + - uses: "./.github/shared/setup" + - run: just fmt-check + + clippy: + name: "Clippy" + needs: check_formatting + strategy: + fail-fast: false + matrix: + platform: [ubuntu-latest, macos-latest, windows-latest] + runs-on: ${{ matrix.platform }} + timeout-minutes: 10 + steps: + - name: "Checkout Repository" + uses: actions/checkout@v6.0.2 + - uses: "./.github/shared/setup" + - run: just clippy + + test: + name: Test + needs: check_formatting + strategy: + fail-fast: false + matrix: + platform: [ubuntu-latest, macos-latest, windows-latest] + runs-on: ${{ matrix.platform }} + timeout-minutes: 30 + steps: + - name: "Checkout Repository" + uses: actions/checkout@v6.0.2 + - uses: "./.github/shared/setup" + - uses: "./.github/shared/qemu" + - name: "Build kernel" + run: just build + - name: "Run tests" + run: just test diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml deleted file mode 100644 index 5b4998b4f..000000000 --- a/.github/workflows/build.yml +++ /dev/null @@ -1,130 +0,0 @@ -name: Build - -on: - push: - branches: - - "*" - pull_request: - -jobs: - check_formatting: - name: "Check Formatting" - runs-on: ubuntu-latest - timeout-minutes: 2 - steps: - - uses: actions/checkout@v1 - - uses: actions-rust-lang/setup-rust-toolchain@v1 - - run: cargo +nightly fmt -- --check - - clippy: - name: "Clippy" - needs: check_formatting - strategy: - fail-fast: false - matrix: - features: [ - "", - "noserial", - "qemu", - "noserial,qemu", - "jtag", - "noserial,jtag", - # jtag and qemu together don't make much sense - ] - runs-on: ubuntu-latest - timeout-minutes: 10 - steps: - - uses: actions/checkout@v1 - - uses: actions-rust-lang/setup-rust-toolchain@v1 - with: - rustflags: "" - - - run: cargo install cargo-make # TODO cache this tool - - run: env CLIPPY_FEATURES=${{ matrix.features }} cargo make clippy - - test: - name: Test - needs: check_formatting - - strategy: - fail-fast: false - matrix: - platform: [ubuntu-latest, macos-latest, windows-latest] - - runs-on: ${{ matrix.platform }} - timeout-minutes: 30 - - steps: - - name: "Checkout Repository" - uses: actions/checkout@v1 - - - name: "Install nightly Rust" - uses: actions-rust-lang/setup-rust-toolchain@v1 - with: - rustflags: "" - - - name: "Print Rust Version" - run: | - rustc -Vv - cargo -Vv - - - name: "Install build tools" - run: cargo install cargo-make cargo-binutils # TODO cache these tools - - - name: "Validate rust-lld" - if: runner.os == 'macOS' - run: | - which rust-lld || echo "Not found" - otool -L ~/.cargo/bin/rust-lld - - - name: "Print Tools Version" - run: | - cargo make --version - cargo objcopy --version - - - name: "Deny Warnings" - run: cargo make build - env: - RUSTFLAGS: "-D warnings" - - - name: Install QEMU (Linux) - if: runner.os == 'Linux' - run: | - sudo apt-get update - sudo apt-get install --fix-missing qemu-system-aarch64 - - - name: Install QEMU (macOS) - if: runner.os == 'macOS' - run: brew install qemu - env: - HOMEBREW_NO_AUTO_UPDATE: 1 - HOMEBREW_NO_BOTTLE_SOURCE_FALLBACK: 1 - HOMEBREW_NO_INSTALL_CLEANUP: 1 - - - name: Install Scoop (Windows) - if: runner.os == 'Windows' - shell: pwsh - run: | - Invoke-WebRequest -UseBasicParsing get.scoop.sh -outfile 'install.ps1' - .\install.ps1 -RunAsAdmin - echo "$HOME\scoop\shims" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append - - - name: Add custom Scoop bucket for QEMU (Windows) - if: runner.os == 'Windows' - shell: pwsh - run: | - scoop bucket add scoop-for-ci https://github.com/metta-systems/scoop-for-ci - - - name: Install QEMU (Windows) - if: runner.os == 'Windows' - shell: pwsh - run: scoop install qemu-1010 - - - name: "Print QEMU Version" - run: qemu-system-aarch64 --version - - - name: "Build kernel" - run: cargo make build - - - name: "Run tests" - run: cargo make test diff --git a/.github/workflows/cog.yaml b/.github/workflows/cog.yaml new file mode 100644 index 000000000..50546c920 --- /dev/null +++ b/.github/workflows/cog.yaml @@ -0,0 +1,23 @@ +name: Cocogitto check + +on: + pull_request: + +jobs: + cog: + name: "Check conventional commits" + runs-on: ubuntu-latest + timeout-minutes: 2 + steps: + - name: "Checkout Repository" + uses: actions/checkout@v6.0.2 + with: + fetch-tags: true + fetch-depth: 0 + # pick the pr HEAD instead of the merge commit + ref: ${{github.event.pull_request.head.sha}} + - name: Conventional commit check + uses: cocogitto/cocogitto-action@v4.1.0 + with: + command: check + args: ${{github.event.pull_request.base.sha}}..${{github.event.pull_request.head.sha}} diff --git a/.zed/settings.json b/.zed/settings.json index 42597fe60..c1d49f549 100644 --- a/.zed/settings.json +++ b/.zed/settings.json @@ -2,17 +2,34 @@ "lsp": { "rust-analyzer": { "initialization_options": { - "cargo": { - "target": "/Users/berkus/Projects/Metta/vesper/targets/aarch64-metta-none-eabi.json", // $ZED_WORKTREE_ROOT - "extraArgs": [ - "-Zbuild-std=compiler_builtins,core,alloc", - "-Zbuild-std-features=compiler-builtins-mem" - ] + "rust": { + "analyzerTargetDir": true, }, + // "checkOnSave": { + // "enable": true, + // "command": "clippy", + // "target": "/Users/berkus/Projects/Metta/vesper/targets/aarch64-metta-none-eabi.json", // $ZED_WORKTREE_ROOT + // "extraArgs": [ + // "-Zbuild-std=compiler_builtins,core,alloc", + // "-Zbuild-std-features=compiler-builtins-mem", + // "-Zmacro-backtrace", + // "--cfg board_rpi3", + // "-Ctarget-cpu=cortex-a53", + // "--features qemu", + // ], + // }, + // "cargo": { + // "target": "/Users/berkus/Projects/Metta/vesper/targets/aarch64-metta-none-eabi.json", // $ZED_WORKTREE_ROOT + // "allTargets": false, + // "extraArgs": [ + // "-Zbuild-std=compiler_builtins,core,alloc", + // "-Zbuild-std-features=compiler-builtins-mem", + // ], + // }, "rustfmt": { - "extraArgs": ["+nightly"] - } - } - } - } + "extraArgs": ["+nightly"], + }, + }, + }, + }, } diff --git a/Cargo.lock b/Cargo.lock index cd7c105dc..1f3c0ee5d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,27 +2,6 @@ # It is not intended for manual editing. version = 4 -[[package]] -name = "CoreFoundation-sys" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0e9889e6db118d49d88d84728d0e964d973a5680befb5f85f55141beea5c20b" -dependencies = [ - "libc", - "mach 0.1.2", -] - -[[package]] -name = "IOKit-sys" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99696c398cbaf669d2368076bdb3d627fb0ce51a26899d7c61228c5c0af3bf4a" -dependencies = [ - "CoreFoundation-sys", - "libc", - "mach 0.1.2", -] - [[package]] name = "aarch64-cpu" version = "11.2.0" @@ -33,48 +12,44 @@ dependencies = [ ] [[package]] -name = "aho-corasick" -version = "1.1.3" +name = "anyhow" +version = "1.0.102" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" -dependencies = [ - "memchr", -] +checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" [[package]] -name = "anyhow" -version = "1.0.100" +name = "arbitrary-int" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61" +checksum = "993a810118f8f37e9c4411c86f1c4c940a09a7ab34b7bf2d88d06f50c553fab7" [[package]] name = "argh" -version = "0.1.13" +version = "0.1.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34ff18325c8a36b82f992e533ece1ec9f9a9db446bd1c14d4f936bac88fcd240" +checksum = "42a66b74feaa2a7eddc2a74d273f151b545742ef375581c5a566607df992565e" dependencies = [ "argh_derive", "argh_shared", - "rust-fuzzy-search", ] [[package]] name = "argh_derive" -version = "0.1.13" +version = "0.1.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adb7b2b83a50d329d5d8ccc620f5c7064028828538bdf5646acd60dc1f767803" +checksum = "0ce8f59ed4ec07742760cf3593727c8caff28ef4537eeb33be25e5fb454850d1" dependencies = [ "argh_shared", "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] name = "argh_shared" -version = "0.1.13" +version = "0.1.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a464143cc82dedcdc3928737445362466b7674b5db4e2eb8e869846d6d84f4f6" +checksum = "284b476f7c2b6af9aa49f059466ebfa725b0f53c33d271a691e613e9187bf507" dependencies = [ "serde", ] @@ -86,10 +61,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" [[package]] -name = "bit_field" -version = "0.10.3" +name = "bitbybit" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e4b40c7323adcfc0a41c4b88143ed58346ff65a288fc144329c5c45e05d70c6" +checksum = "71d2a3353d70ac1091a33cbf31fc7e77b19091538a7e306e3740712af19807ca" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] [[package]] name = "bitflags" @@ -99,31 +79,31 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.10.0" +version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "812e12b5285cc515a9c72a5c1d3b6d46a19dac5acfef5265968c166106e31dd3" +checksum = "843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af" [[package]] -name = "buddy-alloc" -version = "0.6.0" -source = "git+https://github.com/metta-systems/buddy-alloc?branch=feature%2Fallocator-api#e59420e20e8aa862b4fb9c1b0a676d53722329b1" - -[[package]] -name = "bytes" -version = "1.11.1" +name = "build-print" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" +checksum = "d8e6738dfb11354886f890621b4a34c0b177f75538023f7100b608ab9adbd66b" [[package]] -name = "cc" -version = "1.2.41" +name = "build-rs" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac9fe6cdbb24b6ade63616c0a0688e45bb56732262c158df3c0c4bea4ca47cb7" +checksum = "ffc87f52297187fb5d25bde3d368f0480f88ac1d8f3cf4c80ac5575435511114" dependencies = [ - "find-msvc-tools", - "shlex", + "unicode-ident", ] +[[package]] +name = "bytes" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" + [[package]] name = "cfg-if" version = "1.0.4" @@ -141,20 +121,13 @@ name = "chainboot" version = "0.0.1" dependencies = [ "aarch64-cpu", - "bit_field", - "bitflags 2.10.0", - "cfg-if", - "libconsole", - "liblog", - "libmachine", - "libplatform", - "libqemu", - "libtest", "seahash", - "snafu", - "tock-registers", - "usize_conversions", - "ux", + "vesper-console", + "vesper-log", + "vesper-machines", + "vesper-platforms", + "vesper-qemu", + "vesper-tests", ] [[package]] @@ -166,30 +139,44 @@ dependencies = [ "bytes", "crossterm", "futures", - "futures-util", "seahash", "tokio", "tokio-serial", "tokio-stream", - "tokio-util", ] [[package]] name = "convert_case" -version = "0.7.1" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb402b8d4c85569410425650ce3eddc7d698ed96d39a73f941b08fb63082f1e7" +checksum = "633458d4ef8c78b72454de2d54fd6ab2e60f9e02be22f3c6104cdc8a4e0fceb9" dependencies = [ "unicode-segmentation", ] +[[package]] +name = "core-foundation" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b55271e5c8c478ad3f38ad24ef34923091e0548492a266d19b3c0b4d82574c63" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + [[package]] name = "crossterm" version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d8b9f2e4c67f833b660cdb0a3523065869fb35570177239812ed4c905aeff87b" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.11.0", "crossterm_winapi", "derive_more", "document-features", @@ -213,34 +200,47 @@ dependencies = [ [[package]] name = "derive_more" -version = "2.0.1" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "093242cf7570c207c83073cf82f79706fe7b8317e98620a47d5be7c3d8497678" +checksum = "d751e9e49156b02b44f9c1815bcb94b984cdcc4396ecc32521c739452808b134" dependencies = [ "derive_more-impl", ] [[package]] name = "derive_more-impl" -version = "2.0.1" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bda628edc44c4bb645fbe0f758797143e4e07926f7ebf4e9bdfbd3d2ce621df3" +checksum = "799a97264921d8623a957f6c3b9011f3b5492f557bbb7a5a19b7fa6d06ba8dcb" dependencies = [ "convert_case", "proc-macro2", "quote", - "syn", + "rustc_version 0.4.1", + "syn 2.0.117", ] [[package]] name = "document-features" -version = "0.2.11" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95249b50c6c185bee49034bcb378a49dc2b5dff0be90ff6616d31d64febab05d" +checksum = "d4b8a88685455ed29a21542a33abd9cb6510b6b129abadabdcef0f4c55bc8f61" dependencies = [ "litrs", ] +[[package]] +name = "either" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" + +[[package]] +name = "endian-type-rs" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6419a5c75e40011b9fe0174db3fe24006ab122fbe1b7e9cc5974b338a755c76" + [[package]] name = "errno" version = "0.3.14" @@ -252,16 +252,32 @@ dependencies = [ ] [[package]] -name = "find-msvc-tools" -version = "0.1.4" +name = "fallible-iterator" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" + +[[package]] +name = "fdt-rs" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52051878f80a721bb68ebfbc930e07b65ba72f2da88968ea5c06fd6ca3d3a127" +checksum = "ee31ca45a6defbba7504c84c7dac4c68379b59366d01e8c9cf64ab7b11ba4816" +dependencies = [ + "endian-type-rs", + "fallible-iterator", + "memoffset", + "num-derive", + "num-traits", + "rustc_version 0.2.3", + "static_assertions", + "unsafe_unwrap", +] [[package]] name = "futures" -version = "0.3.31" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" +checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d" dependencies = [ "futures-channel", "futures-core", @@ -274,9 +290,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.31" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" +checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" dependencies = [ "futures-core", "futures-sink", @@ -284,15 +300,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.31" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" +checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" [[package]] name = "futures-executor" -version = "0.3.31" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" +checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d" dependencies = [ "futures-core", "futures-task", @@ -301,38 +317,38 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.31" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" +checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" [[package]] name = "futures-macro" -version = "0.3.31" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" +checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] name = "futures-sink" -version = "0.3.31" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" +checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" [[package]] name = "futures-task" -version = "0.3.31" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" +checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" [[package]] name = "futures-util" -version = "0.3.31" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" +checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" dependencies = [ "futures-channel", "futures-core", @@ -342,220 +358,90 @@ dependencies = [ "futures-task", "memchr", "pin-project-lite", - "pin-utils", "slab", ] [[package]] -name = "heck" -version = "0.5.0" +name = "goblin" +version = "0.10.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" - -[[package]] -name = "libboot" -version = "0.0.1" +checksum = "983a6aafb3b12d4c41ea78d39e189af4298ce747353945ff5105b54a056e5cd9" dependencies = [ - "aarch64-cpu", - "libcpu", - "libplatform", - "tock-registers", + "log", + "plain", + "scroll", ] [[package]] -name = "libc" -version = "0.2.177" +name = "heck" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2874a2af47a2325c2001a6e6fad9b16a53b802102b528163885171cf92b15976" - -[[package]] -name = "libconsole" -version = "0.0.1" -dependencies = [ - "aarch64-cpu", - "liblocking", - "liblog", - "libqemu", - "libtime", -] - -[[package]] -name = "libcpu" -version = "0.0.1" -dependencies = [ - "aarch64-cpu", - "bit_field", - "bitflags 2.10.0", - "cfg-if", - "once_cell", - "snafu", - "tock-registers", - "usize_conversions", - "ux", -] - -[[package]] -name = "libdriver" -version = "0.0.1" -dependencies = [ - "liblocking", - "liblog", -] - -[[package]] -name = "libexception" -version = "0.0.1" -dependencies = [ - "aarch64-cpu", - "liblocal-irq", - "liblocking", - "liblog", - "libprimitives", - "snafu", - "tock-registers", -] - -[[package]] -name = "libkernel-state" -version = "0.0.1" - -[[package]] -name = "liblocal-irq" -version = "0.0.1" -dependencies = [ - "aarch64-cpu", - "liblog", - "tock-registers", -] - -[[package]] -name = "liblocking" -version = "0.0.1" -dependencies = [ - "libkernel-state", - "liblocal-irq", -] - -[[package]] -name = "liblog" -version = "0.0.1" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" [[package]] -name = "libmachine" -version = "0.0.1" +name = "io-kit-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "617ee6cf8e3f66f3b4ea67a4058564628cde41901316e19f559e14c7c72c5e7b" dependencies = [ - "aarch64-cpu", - "bit_field", - "bitflags 2.10.0", - "buddy-alloc", - "cfg-if", - "libconsole", - "libcpu", - "libexception", - "libkernel-state", - "liblocal-irq", - "liblocking", - "liblog", - "libmemory", - "libplatform", - "libprimitives", - "libqemu", - "libtime", - "once_cell", - "snafu", - "tock-registers", - "usize_conversions", - "ux", + "core-foundation-sys", + "mach2", ] [[package]] -name = "libmemory" -version = "0.0.1" +name = "itertools" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f56a2d0bc861f9165be4eb3442afd3c236d8a98afd426f65d92324ae1091a484" dependencies = [ - "aarch64-cpu", - "bit_field", - "bitflags 2.10.0", - "buddy-alloc", - "cfg-if", - "liblocking", - "liblog", - "once_cell", - "snafu", - "tock-registers", - "usize_conversions", - "ux", + "either", ] [[package]] -name = "libplatform" +name = "kickstart" version = "0.0.1" dependencies = [ "aarch64-cpu", - "bit_field", - "bitflags 2.10.0", - "buddy-alloc", + "build-print", + "build-rs", "cfg-if", - "libconsole", - "libcpu", - "libdriver", - "libexception", - "libkernel-state", - "liblocal-irq", - "liblocking", - "liblog", - "libmemory", - "libprimitives", - "libtest", - "libtime", - "once_cell", - "qemu-exit", + "fdt-rs", + "goblin", + "minijinja", + "shrinkwraprs", "snafu", - "tock-registers", - "usize_conversions", - "ux", + "vesper-address", + "vesper-boot", + "vesper-cpu", + "vesper-exceptions", + "vesper-locking", + "vesper-log", + "vesper-machines", + "vesper-mapping", + "vesper-objects", + "vesper-print", + "vesper-qemu", + "vesper-syscalls", + "vesper-tests", ] [[package]] -name = "libprimitives" -version = "0.0.1" - -[[package]] -name = "libqemu" -version = "0.0.1" -dependencies = [ - "qemu-exit", -] - -[[package]] -name = "libtest" -version = "0.0.1" -dependencies = [ - "liblog", - "libqemu", -] - -[[package]] -name = "libtime" -version = "0.0.1" -dependencies = [ - "aarch64-cpu", - "liblocking", - "liblog", - "once_cell", - "tock-registers", -] +name = "libc" +version = "0.2.183" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5b646652bf6661599e1da8901b3b9522896f01e736bad5f723fe7a3a27f899d" [[package]] name = "linux-raw-sys" -version = "0.11.0" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df1d3c3b53da64cf5760482273a98e575c651a67eec7f77df96b5b642de8f039" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" [[package]] name = "litrs" -version = "0.4.2" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5e54036fe321fd421e10d732f155734c4e4afd610dd556d9a82833ab3ee0bed" +checksum = "11d3d7f243d5c5a8b9bb5d6dd2b1602c0cb0b9db1621bafc7ed66e35ff9fe092" [[package]] name = "lock_api" @@ -568,53 +454,61 @@ dependencies = [ [[package]] name = "log" -version = "0.4.28" +version = "0.4.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34080505efa8e45a4b816c349525ebe327ceaa8559756f0356cba97ef3bf7432" +checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" [[package]] -name = "mach" -version = "0.1.2" +name = "mach2" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fd13ee2dd61cc82833ba05ade5a30bb3d63f7ced605ef827063c63078302de9" +checksum = "d640282b302c0bb0a2a8e0233ead9035e3bed871f0b7e81fe4a1ec829765db44" dependencies = [ "libc", ] [[package]] -name = "mach" -version = "0.2.3" +name = "memchr" +version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86dd2487cdfea56def77b88438a2c915fb45113c5319bfe7e14306ca4cd0b0e1" -dependencies = [ - "libc", -] +checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" [[package]] -name = "memchr" -version = "2.7.6" +name = "memo-map" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273" +checksum = "38d1115007560874e373613744c6fba374c17688327a71c1476d1a5954cc857b" [[package]] name = "memoffset" -version = "0.6.5" +version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5aa361d4faea93603064a027415f07bd8e1d5c88c9fbf68bf56a285428fd79ce" +checksum = "043175f069eda7b85febe4a74abbaeff828d9f8b448515d3151a14a3542811aa" dependencies = [ "autocfg", ] +[[package]] +name = "minijinja" +version = "2.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ea5ea1e90055f200af6b8e52a4a34e05e77e7fee953a9fb40c631efdc43cab1" +dependencies = [ + "memo-map", + "self_cell", + "serde", +] + [[package]] name = "mio" -version = "1.0.4" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78bed444cc8a2160f01cbcf811ef18cac863ad68ae8ca62092e8db51d51c761c" +checksum = "a69bcab0ad47271a0234d9422b131806bf3968021e5dc9328caf2d4cd58557fc" dependencies = [ "libc", "log", "wasi", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -632,15 +526,13 @@ dependencies = [ [[package]] name = "nix" -version = "0.23.2" +version = "0.26.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f3790c00a0150112de0f4cd161e3d7fc4b2d8a5542ffc35f099a2562aecb35c" +checksum = "598beaf3cc6fdd9a5dfb1630c2800c7acd31df7aaf0f565796fba2b53ca1af1b" dependencies = [ "bitflags 1.3.2", - "cc", "cfg-if", "libc", - "memoffset", ] [[package]] @@ -649,7 +541,7 @@ version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.11.0", "cfg-if", "cfg_aliases", "libc", @@ -660,26 +552,92 @@ name = "nucleus" version = "0.0.1" dependencies = [ "aarch64-cpu", - "bit_field", - "bitflags 2.10.0", + "bitflags 2.11.0", + "build-rs", "cfg-if", - "libboot", - "libconsole", - "libcpu", - "libexception", - "libkernel-state", - "liblocking", - "liblog", - "libmachine", - "libmemory", - "libplatform", - "libqemu", - "libtest", - "libtime", - "snafu", - "tock-registers", - "usize_conversions", - "ux", + "vesper-address", + "vesper-cpu", + "vesper-exceptions", + "vesper-locking", + "vesper-log", + "vesper-machines", + "vesper-mapping", + "vesper-objects", + "vesper-print", + "vesper-qemu", + "vesper-tests", +] + +[[package]] +name = "num" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" +dependencies = [ + "num-complex", + "num-integer", + "num-iter", + "num-rational", + "num-traits", +] + +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-derive" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "876a53fff98e03a936a674b29568b0e605f06b29372c2489ff4de23f1949743d" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-iter" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", ] [[package]] @@ -713,36 +671,30 @@ dependencies = [ [[package]] name = "pin-project-lite" -version = "0.2.16" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" [[package]] -name = "pin-utils" -version = "0.1.0" +name = "plain" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6" [[package]] name = "proc-macro2" -version = "1.0.101" +version = "1.0.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89ae43fd86e4158d6db51ad8e2b80f313af9cc74f5c0e03ccb87de09998732de" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" dependencies = [ "unicode-ident", ] -[[package]] -name = "qemu-exit" -version = "3.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bb0fd6580eeed0103c054e3fba2c2618ff476943762f28a645b63b8692b21c9" - [[package]] name = "quote" -version = "1.0.41" +version = "1.0.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce25767e7b499d1b604768e7cde645d14cc8584231ea6b295e9c9eb22c02e1d1" +checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d" dependencies = [ "proc-macro2", ] @@ -753,51 +705,34 @@ version = "0.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.11.0", ] [[package]] -name = "regex" -version = "1.12.2" +name = "rustc_version" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "843bc0191f75f3e22651ae5f1e72939ab2f72a4bc30fa80a066bd66edefc24d4" +checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" dependencies = [ - "aho-corasick", - "memchr", - "regex-automata", - "regex-syntax", + "semver 0.9.0", ] [[package]] -name = "regex-automata" -version = "0.4.13" +name = "rustc_version" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5276caf25ac86c8d810222b3dbb938e512c55c6831a10f3e6ed1c93b84041f1c" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" dependencies = [ - "aho-corasick", - "memchr", - "regex-syntax", + "semver 1.0.27", ] -[[package]] -name = "regex-syntax" -version = "0.8.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a2d987857b319362043e95f5353c0535c1f58eec5336fdfcf626430af7def58" - -[[package]] -name = "rust-fuzzy-search" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a157657054ffe556d8858504af8a672a054a6e0bd9e8ee531059100c0fa11bb2" - [[package]] name = "rustix" -version = "1.1.2" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd15f8a2c5551a84d56efdc1cd049089e409ac19a3072d5037a17fd70719ff3e" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.11.0", "errno", "libc", "linux-raw-sys", @@ -810,12 +745,59 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "scroll" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1257cd4248b4132760d6524d6dda4e053bc648c9070b960929bf50cfb1e7add" +dependencies = [ + "scroll_derive", +] + +[[package]] +name = "scroll_derive" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed76efe62313ab6610570951494bdaa81568026e0318eaa55f167de70eeea67d" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "seahash" version = "4.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b" +[[package]] +name = "self_cell" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b12e76d157a900eb52e81bc6e9f3069344290341720e9178cde2407113ac8d89" + +[[package]] +name = "semver" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" +dependencies = [ + "semver-parser", +] + +[[package]] +name = "semver" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2" + +[[package]] +name = "semver-parser" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" + [[package]] name = "serde" version = "1.0.228" @@ -843,29 +825,40 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] name = "serialport" -version = "4.0.1" -source = "git+https://github.com/metta-systems/serialport-rs?branch=macos-ENOTTY-fix#7fec572529ec35b82bd4e3636d897fe2f1c2233f" +version = "4.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21f60a586160667241d7702c420fc223939fb3c0bb8d3fac84f78768e8970dee" dependencies = [ - "CoreFoundation-sys", - "IOKit-sys", - "bitflags 1.3.2", + "bitflags 2.11.0", "cfg-if", - "mach 0.2.3", - "nix 0.23.2", - "regex", - "winapi", + "core-foundation", + "core-foundation-sys", + "io-kit-sys", + "mach2", + "nix 0.26.4", + "quote", + "scopeguard", + "unescaper", + "windows-sys 0.52.0", ] [[package]] -name = "shlex" -version = "1.3.0" +name = "shrinkwraprs" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +checksum = "e63e6744142336dfb606fe2b068afa2e1cca1ee6a5d8377277a92945d81fa331" +dependencies = [ + "bitflags 1.3.2", + "itertools", + "proc-macro2", + "quote", + "syn 1.0.109", +] [[package]] name = "signal-hook" @@ -879,9 +872,9 @@ dependencies = [ [[package]] name = "signal-hook-mio" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34db1a06d485c9142248b7a054f034b349b212551f3dfd19c94d45a754a217cd" +checksum = "b75a19a7a740b25bc7944bdee6172368f988763b744e3d4dfe753f6b4ece40cc" dependencies = [ "libc", "mio", @@ -890,18 +883,19 @@ dependencies = [ [[package]] name = "signal-hook-registry" -version = "1.4.6" +version = "1.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2a4719bff48cee6b39d12c020eeb490953ad2443b7055bd0b21fca26bd8c28b" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" dependencies = [ + "errno", "libc", ] [[package]] name = "slab" -version = "0.4.11" +version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a2ae44ef20feb57a68b23d846850f861394c2e02dc425a50098ae8c90267589" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" [[package]] name = "smallvec" @@ -911,46 +905,83 @@ checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" [[package]] name = "snafu" -version = "0.8.9" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e84b3f4eacbf3a1ce05eac6763b4d629d60cbc94d632e4092c54ade71f1e1a2" +checksum = "d1d4bced6a69f90b2056c03dcff2c4737f98d6fb9e0853493996e1d253ca29c6" dependencies = [ "snafu-derive", ] [[package]] name = "snafu-derive" -version = "0.8.9" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1c97747dbf44bb1ca44a561ece23508e99cb592e862f22222dcf42f51d1e451" +checksum = "54254b8531cafa275c5e096f62d48c81435d1015405a91198ddb11e967301d40" dependencies = [ "heck", "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] name = "socket2" -version = "0.6.1" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17129e116933cf371d018bb80ae557e889637989d8638274fb25622827b03881" +checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" dependencies = [ "libc", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + [[package]] name = "syn" -version = "2.0.106" +version = "1.0.109" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ede7c438028d4436d71104916910f5bb611972c5cfd7f89b8300a8186e6fada6" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" dependencies = [ "proc-macro2", "quote", "unicode-ident", ] +[[package]] +name = "syn" +version = "2.0.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "thiserror" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "tock-registers" version = "0.10.1" @@ -959,9 +990,9 @@ checksum = "8d2d250f87fb3fb6f225c907cf54381509f47b40b74b1d1f12d2dccbc915bdfe" [[package]] name = "tokio" -version = "1.49.0" +version = "1.50.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72a2903cd7736441aac9df9d7688bd0ce48edccaadf181c3b90be801e81d3d86" +checksum = "27ad5e34374e03cfffefc301becb44e9dc3c17584f414349ebe29ed26661822d" dependencies = [ "bytes", "libc", @@ -976,13 +1007,13 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "2.6.0" +version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af407857209536a95c8e56f8231ef2c2e2aff839b22e07a1ffcbc617e9db9fa5" +checksum = "5c55a2eff8b69ce66c84f85e1da1c233edc36ceb85a2058d11b0d6a3c7e7569c" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -1011,23 +1042,19 @@ dependencies = [ ] [[package]] -name = "tokio-util" -version = "0.7.18" +name = "unescaper" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" +checksum = "4064ed685c487dbc25bd3f0e9548f2e34bab9d18cefc700f9ec2dba74ba1138e" dependencies = [ - "bytes", - "futures-core", - "futures-sink", - "pin-project-lite", - "tokio", + "thiserror", ] [[package]] name = "unicode-ident" -version = "1.0.19" +version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f63a545481291138910575129486daeaf8ac54aee4387fe7906919f7830c7d9d" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" [[package]] name = "unicode-segmentation" @@ -1035,6 +1062,12 @@ version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" +[[package]] +name = "unsafe_unwrap" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1230ec65f13e0f9b28d789da20d2d419511893ea9dac2c1f4ef67b8b14e5da80" + [[package]] name = "usize_conversions" version = "0.2.0" @@ -1042,10 +1075,206 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f70329e2cbe45d6c97a5112daad40c34cd9a4e18edb5a2a18fefeb584d8d25e5" [[package]] -name = "ux" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b59fc5417e036e53226bbebd90196825d358624fd5577432c4e486c95b1b096" +name = "vesper-address" +version = "0.0.1" +dependencies = [ + "arbitrary-int", + "bitbybit", + "num", + "usize_conversions", + "vesper-boot", + "vesper-console", + "vesper-log", + "vesper-qemu", + "vesper-tests", +] + +[[package]] +name = "vesper-alloc" +version = "0.0.1" +dependencies = [ + "vesper-address", + "vesper-log", +] + +[[package]] +name = "vesper-boot" +version = "0.0.1" +dependencies = [ + "aarch64-cpu", + "tock-registers", + "vesper-cpu", + "vesper-platforms", +] + +[[package]] +name = "vesper-console" +version = "0.0.1" +dependencies = [ + "vesper-locking", + "vesper-log", + "vesper-print", + "vesper-qemu", +] + +[[package]] +name = "vesper-cpu" +version = "0.0.1" +dependencies = [ + "aarch64-cpu", +] + +[[package]] +name = "vesper-drivers" +version = "0.0.1" +dependencies = [ + "vesper-locking", + "vesper-log", +] + +[[package]] +name = "vesper-exceptions" +version = "0.0.1" +dependencies = [ + "aarch64-cpu", + "tock-registers", + "vesper-local-irq", + "vesper-log", +] + +[[package]] +name = "vesper-kernel-state" +version = "0.0.1" + +[[package]] +name = "vesper-local-irq" +version = "0.0.1" +dependencies = [ + "aarch64-cpu", + "tock-registers", + "vesper-log", +] + +[[package]] +name = "vesper-locking" +version = "0.0.1" +dependencies = [ + "vesper-kernel-state", + "vesper-local-irq", +] + +[[package]] +name = "vesper-log" +version = "0.0.1" + +[[package]] +name = "vesper-machines" +version = "0.0.1" +dependencies = [ + "vesper-cpu", + "vesper-exceptions", + "vesper-local-irq", + "vesper-log", + "vesper-qemu", +] + +[[package]] +name = "vesper-mapping" +version = "0.0.1" +dependencies = [ + "vesper-address", + "vesper-log", + "vesper-mmio", +] + +[[package]] +name = "vesper-memory" +version = "0.0.1" +dependencies = [ + "aarch64-cpu", + "snafu", + "tock-registers", + "vesper-address", + "vesper-log", + "vesper-mapping", +] + +[[package]] +name = "vesper-mmio" +version = "0.0.1" +dependencies = [ + "vesper-address", +] + +[[package]] +name = "vesper-objects" +version = "0.0.1" +dependencies = [ + "vesper-address", + "vesper-print", + "vesper-qemu", + "vesper-syscalls", +] + +[[package]] +name = "vesper-platforms" +version = "0.0.1" +dependencies = [ + "aarch64-cpu", + "snafu", + "tock-registers", + "vesper-address", + "vesper-console", + "vesper-cpu", + "vesper-drivers", + "vesper-exceptions", + "vesper-kernel-state", + "vesper-locking", + "vesper-log", + "vesper-mapping", + "vesper-mmio", + "vesper-primitives", + "vesper-qemu", + "vesper-time", +] + +[[package]] +name = "vesper-primitives" +version = "0.0.1" + +[[package]] +name = "vesper-print" +version = "0.0.1" + +[[package]] +name = "vesper-qemu" +version = "0.0.1" +dependencies = [ + "vesper-print", +] + +[[package]] +name = "vesper-syscalls" +version = "0.0.1" + +[[package]] +name = "vesper-tests" +version = "0.0.1" +dependencies = [ + "vesper-log", + "vesper-qemu", +] + +[[package]] +name = "vesper-time" +version = "0.0.1" +dependencies = [ + "aarch64-cpu", + "once_cell", + "tock-registers", + "vesper-locking", + "vesper-log", +] [[package]] name = "wasi" @@ -1083,20 +1312,11 @@ checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" [[package]] name = "windows-sys" -version = "0.59.0" +version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets 0.52.6", -] - -[[package]] -name = "windows-sys" -version = "0.60.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" -dependencies = [ - "windows-targets 0.53.5", + "windows-targets", ] [[package]] @@ -1114,31 +1334,14 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ - "windows_aarch64_gnullvm 0.52.6", - "windows_aarch64_msvc 0.52.6", - "windows_i686_gnu 0.52.6", - "windows_i686_gnullvm 0.52.6", - "windows_i686_msvc 0.52.6", - "windows_x86_64_gnu 0.52.6", - "windows_x86_64_gnullvm 0.52.6", - "windows_x86_64_msvc 0.52.6", -] - -[[package]] -name = "windows-targets" -version = "0.53.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" -dependencies = [ - "windows-link", - "windows_aarch64_gnullvm 0.53.1", - "windows_aarch64_msvc 0.53.1", - "windows_i686_gnu 0.53.1", - "windows_i686_gnullvm 0.53.1", - "windows_i686_msvc 0.53.1", - "windows_x86_64_gnu 0.53.1", - "windows_x86_64_gnullvm 0.53.1", - "windows_x86_64_msvc 0.53.1", + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", ] [[package]] @@ -1147,92 +1350,44 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" - [[package]] name = "windows_aarch64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" -[[package]] -name = "windows_aarch64_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" - [[package]] name = "windows_i686_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" -[[package]] -name = "windows_i686_gnu" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" - [[package]] name = "windows_i686_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" -[[package]] -name = "windows_i686_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" - [[package]] name = "windows_i686_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" -[[package]] -name = "windows_i686_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" - [[package]] name = "windows_x86_64_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" -[[package]] -name = "windows_x86_64_gnu" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" - [[package]] name = "windows_x86_64_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" - [[package]] name = "windows_x86_64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" - -[[package]] -name = "windows_x86_64_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" diff --git a/Cargo.toml b/Cargo.toml index 6854a6661..2cdef8af1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,9 +1,12 @@ [workspace] members = [ - "nucleus", + "kernel/nucleus", + "kernel/kickstart", "bin/chainboot", "bin/chainofcommand", # Libraries + "libs/address", + "libs/alloc", "libs/boot", "libs/console", "libs/cpu", @@ -14,14 +17,19 @@ members = [ "libs/locking", "libs/log", "libs/machine", + "libs/mapping", "libs/memory", + "libs/mmio", + "libs/object", "libs/platform", "libs/primitives", + "libs/print", "libs/qemu", + "libs/syscall", "libs/test", "libs/time", ] -resolver = "2" +resolver = "3" [workspace.package] authors = ["Berkus Decker "] @@ -40,32 +48,41 @@ version = "0.0.1" # target #======== aarch64-cpu = { version = "11.2" } -bit_field = { version = "0.10" } +arbitrary-int = { version = "2.1", default-features = false } +bitbybit = { version = "2.0" } bitflags = { version = "2.10" } buddy-alloc = { version = "0.6.0", git = "https://github.com/metta-systems/buddy-alloc", branch = "feature/allocator-api" } cfg-if = { version = "1.0" } -libboot = { path = "libs/boot" } -libconsole = { path = "libs/console" } -libcpu = { path = "libs/cpu" } -libdriver = { path = "libs/driver" } -libexception = { path = "libs/exception" } -libkernel-state = { path = "libs/kernel-state" } -liblocal-irq = { path = "libs/local-irq" } -liblocking = { path = "libs/locking" } -liblog = { path = "libs/log" } -libmachine = { path = "libs/machine" } -libmemory = { path = "libs/memory" } -libplatform = { path = "libs/platform" } -libprimitives = { path = "libs/primitives" } -libqemu = { path = "libs/qemu" } -libtest = { path = "libs/test" } -libtime = { path = "libs/time" } +libaddress = { path = "libs/address", package = "vesper-address" } +liballoc = { path = "libs/alloc", package = "vesper-alloc" } +libboot = { path = "libs/boot", package = "vesper-boot" } +libconsole = { path = "libs/console", package = "vesper-console" } +libcpu = { path = "libs/cpu", package = "vesper-cpu" } +libdriver = { path = "libs/driver", package = "vesper-drivers" } +libexception = { path = "libs/exception", package = "vesper-exceptions" } +libkernel-state = { path = "libs/kernel-state", package = "vesper-kernel-state" } +liblocal-irq = { path = "libs/local-irq", package = "vesper-local-irq" } +liblocking = { path = "libs/locking", package = "vesper-locking" } +liblog = { path = "libs/log", package = "vesper-log" } +libmachine = { path = "libs/machine", package = "vesper-machines" } +libmapping = { path = "libs/mapping", package = "vesper-mapping" } +libmemory = { path = "libs/memory", package = "vesper-memory" } +libmmio = { path = "libs/mmio", package = "vesper-mmio" } +libobject = { path = "libs/object", package = "vesper-objects" } +libplatform = { path = "libs/platform", package = "vesper-platforms" } +libprimitives = { path = "libs/primitives", package = "vesper-primitives" } +libprint = { path = "libs/print", package = "vesper-print" } +libqemu = { path = "libs/qemu", package = "vesper-qemu" } +libsyscall = { path = "libs/syscall", package = "vesper-syscalls" } +libtest = { path = "libs/test", package = "vesper-tests" } +libtime = { path = "libs/time", package = "vesper-time" } +num = { version = "0.4", default-features = false } once_cell = { version = "1.21", default-features = false, features = ["unstable"] } -qemu-exit = { version = "3.0" } -snafu = { version = "0.8", default-features = false, features = ["unstable-core-error"] } +safe-mmio = { version = "0.3" } +snafu = { version = "0.9", default-features = false } +static_assertions = { version = "1.1.0" } tock-registers = { version = "0.10" } usize_conversions = { version = "0.2" } -ux = { version = "0.1", default-features = false } #====== # host #====== @@ -75,18 +92,23 @@ bytes = { version = "1.11" } crossterm = { version = "0.29", features = ["event-stream"] } futures = { version = "0.3" } futures-util = { version = "0.3", features = ["io"] } +serialport = { version = "4.0" } tokio = { version = "1.49", features = ["full"] } tokio-serial = { version = "5.4" } tokio-stream = { version = "0.1" } tokio-util = { version = "0.7", features = ["codec", "io"] } +#========== +# build.rs +#========== +goblin = { version = "0.10" } +build-rs = { version = "0.3" } +build-print = { version = "1.0" } +minijinja = { version = "2", features = ["loader"] } #======= # mixed #======= seahash = { version = "4.1" } -[patch.crates-io] -serialport = { git = "https://github.com/metta-systems/serialport-rs", branch = "macos-ENOTTY-fix" } - [profile.dev] # See https://github.com/rust-lang/cargo/issues/7359 about why panic=abort is not working here. # It is still defined in the target JSON so not stricly necessary to specify it here anyway. @@ -98,7 +120,7 @@ codegen-units = 1 debug = true lto = true opt-level = 's' -panic = "abort" +# panic = "abort" # - debug in qemu - wanna see panics [profile.release-fast] codegen-units = 16 diff --git a/Justfile b/Justfile index eebe28c35..1763fa2a5 100644 --- a/Justfile +++ b/Justfile @@ -1,162 +1,342 @@ +# === Configuration === + +target := 'aarch64-metta-none-eabi' +# ⚠️ Target path must be 'escaped' to work on Windows +target_json := "-Zjson-target-spec --target='" + justfile_directory() / 'targets' / target + ".json'" +rust_std := '-Zbuild-std=compiler_builtins,core,alloc -Zbuild-std-features=compiler-builtins-mem' + +# Board presets: rustflags, dtb, qemu-machine +board_rpi3_flags := '-C target-cpu=cortex-a53 --cfg board_rpi3' +board_rpi4_flags := '-C target-cpu=cortex-a73 --cfg board_rpi4' +rpi3_dtb := justfile_directory() / 'targets/bcm2710-rpi-3-b-plus.dtb' +rpi4_dtb := justfile_directory() / 'targets/bcm2711-rpi-4-b.dtb' + +nucleus_link := 'libs/platform/src/raspberrypi/linker/nucleus.ld' +init_link := 'libs/platform/src/raspberrypi/linker/kickstart.ld' +test_link := 'libs/platform/src/raspberrypi/linker/test.ld' +chainboot_link := 'bin/chainboot/src/link.ld' + +fixed_rustflags := '-D warnings -Z macro-backtrace' + +qemu := env('QEMU', 'qemu-system-aarch64') +qemu_machine := env('QEMU_MACHINE', 'raspi3b') +gdb := env('GDB', 'aarch64-elf-gdb') # An aarch64-enabled GDB (brew install aarch64-elf-gdb) +objcopy := 'rust-objcopy' +nm := 'rust-nm' +volume := env('VOLUME', '/Volumes/BOOT') + +kernel_elf := justfile_directory() / 'target' / target / 'release/kickstart' +kernel_bin := justfile_directory() / 'target/kernel.bin' +chainboot_elf := justfile_directory() / 'target' / target / 'release/chainboot' +chainboot_bin := justfile_directory() / 'target/chainboot.bin' + +chainboot_serial := '/dev/tty.SLAB_USBtoUART' +chainboot_baud := '115200' + +# QEMU option fragments +qemu_base_opts := '-M ' + qemu_machine + ' -chardev stdio,mux=on,id=char0,logfile=qemu.log,signal=off -mon chardev=char0 -serial chardev:char0 -semihosting-config enable=on,chardev=char0' +qemu_disasm := '-d in_asm,unimp,int,mmu,cpu_reset,guest_errors,nochain,plugin' +qemu_gdb_opts := '-gdb tcp::5555 -S' +qemu_test_opts := '-nographic' +qemu_disasm_gdb := qemu_disasm + ' ' + qemu_gdb_opts + +gdb_connect := justfile_directory() / 'target' / target / 'gdb-connect' + +openocd_bin := env('OPENOCD', '/usr/local/opt/openocd/4d6519593-rtt/bin/openocd') + +ok_label := '✅' +copy_label := '🔄' + _default: @just --list -make-opts := '--time-summary --hide-uninteresting' -# make-opts := '--quiet' +# === Low-level: cross-compile a single crate === -# Update all dependencies -[group("maintenance")] -deps-up: - cargo update +# Cross-build a crate for a given board with a given linker script and features +[private] +_cross-build crate board='rpi4' linker_script='' features='': + RUSTFLAGS="{{ fixed_rustflags }} {{ if board == 'rpi3' { board_rpi3_flags } else { board_rpi4_flags } }}{{ if linker_script != '' { ' -C link-arg=--script=' + linker_script } else { '' } }}" \ + cargo build {{ target_json }} \ + {{ if features != '' { '--features=' + features } else { '' } }} \ + {{ rust_std }} \ + --release -p {{ crate }} + +# === Kernel (nucleus + kickstart -> kernel.bin) === -# Build default hw kernel and run chainofcommand to boot this kernel onto the board +# Build kernel (features: '' for hw, 'qemu' for emulation) [group("hw")] -boot: chainofcommand - cargo make {{ make-opts }} chainboot # make boot-kernel ? +build board='rpi4' features='': (_cross-build 'nucleus' board nucleus_link features) (_cross-build 'kickstart' board init_link features) + {{ objcopy }} --strip-all -O binary "{{ kernel_elf }}" "{{ kernel_bin }}" + @# TODO: print final binary size! + @echo "{{ok_label}} kernel built for {{ board }}{{ if features != '' { ' [' + features + ']' } else { '' } }}" -# Build and run kernel in QEMU with serial port emulation -[group("emu")] -zellij: - cargo make {{ make-opts }} --makefile $(pwd)/nucleus/Makefile.toml --cwd nucleus zellij-nucleus - zellij --layout emulation/layout.zellij +alias b := build -# Build and run chainboot in QEMU with serial port emulation -[group("emu")] -cb-zellij: - # Connect to it via chainofcommand to load an actual kernel - # TODO: actually run chainofcommand in a zellij session too - cargo make {{ make-opts }} --makefile $(pwd)/bin/chainboot/Makefile.toml --cwd bin/chainboot zellij - zellij --layout emulation/layout.zellij +# === Chainboot === -# Run chainboot with GDB in zellij window -cb-zellij-gdb: - cargo make {{ make-opts }} --makefile $(pwd)/bin/chainboot/Makefile.toml --cwd bin/chainboot zellij-gdb - zellij --layout emulation/layout.zellij +# Build chainboot bootloader (features: '' for hw, 'qemu' for emulation) +[group("hw")] +build-chainboot board='rpi4' features='': (_cross-build 'chainboot' board chainboot_link features) + {{ objcopy }} --strip-all -O binary "{{ chainboot_elf }}" "{{ chainboot_bin }}" + @echo "{{ok_label}} chainboot built for {{ board }}{{ if features != '' { ' [' + features + ']' } else { '' } }}" + +# === Chainofcommand (host tool) === # Build chainofcommand serial loader [group("hw")] chainofcommand: - cargo make {{ make-opts }} --makefile $(pwd)/bin/chainofcommand/Makefile.toml --cwd bin/chainofcommand build + @cargo build -p chainofcommand alias coc := chainofcommand +# === QEMU runners === + # Build and run kernel in QEMU [group("emu")] -qemu: - cargo make {{ make-opts }} --makefile $(pwd)/nucleus/Makefile.toml --cwd nucleus qemu +qemu: (build 'rpi3' 'qemu') + @echo "🚜 Run QEMU {{ qemu_base_opts }} with {{ kernel_bin }}" + @echo "🚜 .. on {{ rpi3_dtb }}" + @rm -f qemu.log + {{ qemu }} {{ qemu_base_opts }} -dtb "{{ rpi3_dtb }}" -kernel "{{ kernel_bin }}" -# Build and run kernel in QEMU with GDB port enabled +# Build and run kernel in QEMU with GDB port [group("emu")] -qemu-gdb: - cargo make {{ make-opts }} --makefile $(pwd)/nucleus/Makefile.toml --cwd nucleus qemu-gdb +qemu-gdb: (build 'rpi3' 'qemu') + @echo "🚜 Run QEMU {{ qemu_base_opts }} {{ qemu_disasm_gdb }} with {{ kernel_bin }}" + @echo "🚜 .. on {{ rpi3_dtb }}" + @rm -f qemu.log + {{ qemu }} {{ qemu_base_opts }} {{ qemu_disasm_gdb }} -dtb "{{ rpi3_dtb }}" -kernel "{{ kernel_bin }}" # Build and run chainboot in QEMU [group("emu")] -cb-qemu: - # Connect to it via chainofcommand to load an actual kernel - cargo make {{ make-opts }} --makefile $(pwd)/bin/chainboot/Makefile.toml --cwd bin/chainboot qemu +cb-qemu: (build-chainboot 'rpi3' 'qemu') + @echo "🚜 Run QEMU {{ qemu_base_opts }} {{ qemu_disasm }} with {{ chainboot_bin }}" + @echo "🚜 .. on {{ rpi3_dtb }}" + @rm -f qemu.log + {{ qemu }} {{ qemu_base_opts }} {{ qemu_disasm }} -serial pty -dtb "{{ rpi3_dtb }}" -kernel "{{ chainboot_bin }}" + +# Build and run chainboot in QEMU with GDB port +[group("emu")] +cb-qemu-gdb: (build-chainboot 'rpi3' 'qemu') + @echo "🚜 Run QEMU {{ qemu_base_opts }} {{ qemu_disasm_gdb }} with {{ chainboot_bin }}" + @echo "🚜 .. on {{ rpi3_dtb }}" + @rm -f qemu.log + {{ qemu }} {{ qemu_base_opts }} {{ qemu_disasm_gdb }} -serial pty -dtb "{{ rpi3_dtb }}" -kernel "{{ chainboot_bin }}" -# Build and run chainboot in QEMU with GDB port enabled +# === Zellij (QEMU in split terminal) === + +[private] +_write-zellij-config bin runner_opts dtb: + #!/usr/bin/env bash + cat > emulation/zellij-config.sh < "{{ gdb_connect }}" < target/{{ MOD }}.dot \ - && dot -Tpng target/{{ MOD }}.dot -o target/{{ MOD }}.png +_gen-deps-graph mod: + cargo modules dependencies --max-depth 5 --no-sysroot --no-externs -p {{ mod }} > target/{{ mod }}.dot \ + && dot -Tpng target/{{ mod }}.dot -o target/{{ mod }}.png # Render modules' usage graph [group("modules")] [macos] -deps-graph MOD: (gen-deps-graph MOD) - open target/{{ MOD }}.png +deps-graph mod: (_gen-deps-graph mod) + open target/{{ mod }}.png # Render modules' usage graph [group("modules")] [windows] -deps-graph MOD: (gen-deps-graph MOD) - start target/{{ MOD }}.png +deps-graph mod: (_gen-deps-graph mod) + start target/{{ mod }}.png # Render modules' usage graph [group("modules")] [linux] -deps-graph MOD: (gen-deps-graph MOD) - xdg-open target/{{ MOD }}.png +deps-graph mod: (_gen-deps-graph mod) + xdg-open target/{{ mod }}.png # Render modules symbol visibility [group("modules")] -exports MOD: - cargo modules structure -p {{ MOD }} +exports mod: + cargo modules structure -p {{ mod }} -# Find orphan files in the module sources +# Find orphan files [group("modules")] -orphans MOD: - cargo modules orphans -p {{ MOD }} - -# Modules dependency visualization end -#============================================================================== - -# Generate and open documentation -[group("maintenance")] -doc: - cargo make {{ make-opts }} docs-flow - -# Check formatting -[group("maintenance")] -fmt-check: - cargo fmt -- --check - -# Run lint tasks -[group("maintenance")] -lint: fmt-check clippy - -# Run pre-push local checks -[group("ci")] -pre-push: fmt-check clippy-pre-push test - -# Run CI tasks -[group("ci")] -ci: clean lint build test +orphans mod: + cargo modules orphans -p {{ mod }} # Prepare local dev tools and set-up git hooks [group("maintenance")] setup-local-dev: - commit-emoji --help || cargo install commit-emoji + which cargo-binstall || cargo install cargo-binstall + commit-emoji --help || cargo binstall -y commit-emoji commit-emoji -i - # Run local shortened clippy before pushing to remote - cp .hooks/pre-push .git/hooks/pre-push + cargo binstall -y cargo-binutils + # todo install rustfilt, what else? + # install pre-push git hook with `just pre-push` diff --git a/Makefile.toml b/Makefile.toml deleted file mode 100644 index 84a9d45b7..000000000 --- a/Makefile.toml +++ /dev/null @@ -1,422 +0,0 @@ -# @todo: replace this with xtask? -# - Kinda tired of the maintenance pains here. - -# -# SPDX-License-Identifier: BlueOak-1.0.0 -# -# Copyright (c) Berkus Decker -# - -# -# Global workspace configuration -# -[config] -min_version = "0.37.0" -default_to_workspace = false -skip_core_tasks = true - -[env] -DEFAULT_TARGET = "aarch64-metta-none-eabi" -# -# === User-configurable === -# -# Pass TARGET env var if it does not match the default target above. -TARGET = { value = "${DEFAULT_TARGET}", condition = { env_not_set = ["TARGET"] } } -# Name of the target board "rpi3" or "rpi4" -# Will be converted by nucleus/build.rs into a board configuration flag like board_rpi3. -TARGET_BOARD = { value = "rpi4", condition = { env_not_set = ["TARGET_BOARD"] } } -# Name of the DTB file for target board configuration, use bcm2710-rpi-3-b-plus.dtb for RasPi3B+ -TARGET_DTB = { value = "${CARGO_MAKE_WORKSPACE_WORKING_DIRECTORY}/targets/bcm2711-rpi-4-b.dtb", condition = { env_not_set = [ - "TARGET_DTB", -] } } -# -# === RUSTFLAGS === -# -FIXED_RUSTFLAGS = "-D warnings -Z macro-backtrace" -BOARD_rpi3_RUSTFLAGS = "-C target-cpu=cortex-a53 --cfg board_rpi3" -BOARD_rpi4_RUSTFLAGS = "-C target-cpu=cortex-a73 --cfg board_rpi4" -# -# === QEMU === -# -# AArch64 QEMU binary -QEMU = { value = "qemu-system-aarch64", condition = { env_not_set = ["QEMU"] } } -# QEMU machine type, defaults to raspi3b but CI runners override it due to ancient QEMU versions they use. -# Supported as of qemu 9.0: -# raspi3ap Raspberry Pi 3A+ (revision 1.0) -# raspi3b Raspberry Pi 3B (revision 1.2) -# raspi4b Raspberry Pi 4B (revision 1.5) -QEMU_MACHINE = { value = "raspi3b", condition = { env_not_set = ["QEMU_MACHINE"] } } - -# An aarch64-enabled GDB -# brew install aarch64-elf-gdb -GDB = { value = "aarch64-elf-gdb", condition = { env_not_set = ["GDB"] } } - -# @todo: Replace it with probe-rs.. which can be installed by `cargo make`! -# OpenOCD with JLink support -# (RTT patch from http://openocd.zylin.com/#/c/4055/11 has already been merged into main line) -OPENOCD = { value = "/usr/local/opt/openocd/4d6519593-rtt/bin/openocd", condition = { env_not_set = [ - "OPENOCD", -] } } -# Mounted sdcard partition path -VOLUME = { value = "/Volumes/BOOT", condition = { env_not_set = ["VOLUME"] } } -# -# === Automatic === -# -# When running `cargo make` for modules which are part of a workspace, you can automatically -# have the member crates makefile (*even if doesn't exist*) extend the workspace level makefile. -# This allows you to maintain a single makefile for the entire workspace but -# have access to those custom tasks in every member crate. -# This is only relevant for workspace builds which are triggered in the workspace root. -# Flows that start directly in the member crate, must manually extend the workspace -# level makefile using the `extend` keyword. -CARGO_MAKE_EXTEND_WORKSPACE_MAKEFILE = true -# Build rust stdlib locally -RUST_STD = "-Zbuild-std=compiler_builtins,core,alloc -Zbuild-std-features=compiler-builtins-mem" -TARGET_JSON = "-Zjson-target-spec --target=${CARGO_MAKE_WORKSPACE_WORKING_DIRECTORY}/targets/${TARGET}.json" -DEVICE_FEATURES = "noserial" -QEMU_FEATURES = "qemu" -TARGET_FEATURES = "" -# Working objcopy from `brew install aarch64-elf-binutils` -# OBJCOPY = "/opt/homebrew/Cellar/aarch64-elf-binutils/2.40/bin/aarch64-elf-objcopy" -# LLVM's objcopy, usually full of bugs like https://github.com/llvm/llvm-project/issues/58407 -OBJCOPY = "rust-objcopy" # Part of `cargo objcopy` in cargo-binutils -OBJCOPY_PARAMS = "--strip-all -O binary" -NM = "rust-nm" # Part of `cargo nm` in cargo-binutils -UTILS_CONTAINER = "andrerichter/raspi3-utils" -DOCKER_CMD = "docker run -it --rm -v ${PWD}:/work -w /work -p 5900:5900" -QEMU_CONTAINER_CMD = "qemu-system-aarch64" -# -# Could additionally use -nographic to disable GUI -- this shall be useful for automated tests. -# -# QEMU has renamed the RasPi machines since version 6.2.0, use just `raspi3` for previous versions. -QEMU_OPTS = "-M ${QEMU_MACHINE} -semihosting" -QEMU_ARM_TRACE_OPTS = "arm_gt_cntvoff_write,arm_gt_ctl_write,arm_gt_cval_write,arm_gt_imask_toggle,arm_gt_recalc,arm_gt_recalc_disabled,arm_gt_tval_write,armsse_cpu_pwrctrl_read,armsse_cpu_pwrctrl_write,armsse_cpuid_read,armsse_cpuid_write,armsse_mhu_read,armsse_mhu_write" -QEMU_BCM_TRACE_OPTS = "bcm2835_cprman_read,bcm2835_cprman_write,bcm2835_cprman_write_invalid_magic,bcm2835_ic_set_cpu_irq,bcm2835_ic_set_gpu_irq,bcm2835_mbox_irq,bcm2835_mbox_property,bcm2835_mbox_read,bcm2835_mbox_write,bcm2835_sdhost_edm_change,bcm2835_sdhost_read,bcm2835_sdhost_update_irq,bcm2835_sdhost_write,bcm2835_systmr_irq_ack,bcm2835_systmr_read,bcm2835_systmr_run,bcm2835_systmr_timer_expired,bcm2835_systmr_write" -QEMU_TRACE_OPTS = "trace:${QEMU_ARM_TRACE_OPTS},${QEMU_BCM_TRACE_OPTS}" # @todo trace: prefix for each opt -QEMU_DISASM_OPTS = "-d in_asm,unimp,int,mmu,cpu_reset,guest_errors,nochain,plugin" -QEMU_SERIAL_OPTS = "-serial stdio -serial pty" -QEMU_TESTS_OPTS = "-nographic" -# For gdb connection: -# - if this is set, MUST have gdb attached for SYS_WRITE0 to work, otherwise QEMU will crash. -# - port 5555 used to match JLink configuration, so we can reuse the same GDB command for both QEMU and JTAG. -QEMU_GDB_OPTS = "-gdb tcp::5555 -S" -GDB_CONNECT_FILE = "${CARGO_MAKE_WORKSPACE_WORKING_DIRECTORY}/target/${TARGET}/gdb-connect" -KERNEL_ELF = "${CARGO_MAKE_WORKSPACE_WORKING_DIRECTORY}/target/${TARGET}/release/nucleus" -KERNEL_BIN = "${CARGO_MAKE_WORKSPACE_WORKING_DIRECTORY}/target/nucleus.bin" -CHAINBOOT_SERIAL = "/dev/tty.SLAB_USBtoUART" -CHAINBOOT_BAUD = 115200 - -# -# === Base reusable commands === -# - -# #Target dependencies: -# [tasks.kernel-binary] -- in nucleus -# alias = "empty" -# -# [tasks.chainboot-binary] -- in bin/chainboot -# alias = "empty" -# -# [tasks.chainofcommand-binary] -- in bin/chainofcommand -# alias = "empty" -# -# [tasks.ttt-binary] -- in tools/ttt -# alias = "empty" - -# By default, build everything. -[tasks.default] -alias = "build" - -# All primary dependencies to build. -[tasks.build] -dependencies = ["x-build-nucleus", "x-build-bin-chainboot", "x-build-bin-chainofcommand"] -# ttt-binary too - -[tasks.x-build-nucleus] -command = "cargo" -# Have to use args to support env vars across platforms, regular scripts don't work on Windows -args = [ - "make", - "--makefile", - "${CARGO_MAKE_WORKING_DIRECTORY}/nucleus/Makefile.toml", - "--cwd", - "nucleus", - "build", -] - -[tasks.x-build-bin-chainboot] -command = "cargo" -# Have to use args to support env vars across platforms, regular scripts don't work on Windows -args = [ - "make", - "--makefile", - "${CARGO_MAKE_WORKING_DIRECTORY}/bin/chainboot/Makefile.toml", - "--cwd", - "bin/chainboot", - "build", -] - -[tasks.x-build-bin-chainofcommand] -command = "cargo" -# Have to use args to support env vars across platforms, regular scripts don't work on Windows -args = [ - "make", - "--makefile", - "${CARGO_MAKE_WORKING_DIRECTORY}/bin/chainofcommand/Makefile.toml", - "--cwd", - "bin/chainofcommand", - "build", -] - -# Run a target build with current platform configuration. -[tasks.build-target] -env = { RUSTFLAGS = "${FIXED_RUSTFLAGS} ${BOARD_RUSTFLAGS} ${TARGET_LINK_RUSTFLAGS}" } -command = "cargo" -args = [ - "build", - "@@split(TARGET_JSON, )", - "--features=${TARGET_FEATURES}", - "@@split(RUST_STD, )", - "--release", -] - -[tasks.build-qemu] -env = { "TARGET_FEATURES" = "${QEMU_FEATURES}", "TARGET_BOARD" = "rpi3", "BOARD_RUSTFLAGS" = "${BOARD_rpi3_RUSTFLAGS}" } -run_task = "build-target" - -[tasks.qemu-runner] -dependencies = ["build-qemu", "convert-kernel-binary"] -env = { "TARGET_FEATURES" = "${QEMU_FEATURES}" } -script = [ - "echo 🚜 Run QEMU ${QEMU_OPTS} ${QEMU_RUNNER_OPTS}", - "echo 🚜 .. with ${KERNEL_BIN}", - "echo 🚜 .. on ${TARGET_DTB}", - "echo", - "echo", - "rm -f qemu.log", - "${QEMU} ${QEMU_OPTS} ${QEMU_RUNNER_OPTS} -dtb \"${TARGET_DTB}\" -kernel \"${KERNEL_BIN}\" 2>&1 | tee qemu.log", - "echo", - "echo", -] - -[tasks.test] -dependencies = ["x-test-nucleus", "x-test-bin-chainboot", "x-test-bin-chainofcommand"] - -[tasks.x-test-nucleus] -command = "cargo" -# Have to use args to support env vars across platforms, regular scripts don't work on Windows -args = [ - "make", - "--makefile", - "${CARGO_MAKE_WORKING_DIRECTORY}/nucleus/Makefile.toml", - "--cwd", - "nucleus", - "test-nucleus", -] - -[tasks.x-test-bin-chainboot] -command = "cargo" -# Have to use args to support env vars across platforms, regular scripts don't work on Windows -args = [ - "make", - "--makefile", - "${CARGO_MAKE_WORKING_DIRECTORY}/bin/chainboot/Makefile.toml", - "--cwd", - "bin/chainboot", - "test-chainboot", -] - -[tasks.x-test-bin-chainofcommand] -command = "cargo" -# Have to use args to support env vars across platforms, regular scripts don't work on Windows -args = [ - "make", - "--makefile", - "${CARGO_MAKE_WORKING_DIRECTORY}/bin/chainofcommand/Makefile.toml", - "--cwd", - "bin/chainofcommand", - "test", -] - -[tasks.test-on-rpi3-qemu] -env = { TARGET_FEATURES = "${QEMU_FEATURES}", TARGET_BOARD = "rpi3", RUSTFLAGS = "${FIXED_RUSTFLAGS} ${BOARD_rpi3_RUSTFLAGS} ${TARGET_LINK_RUSTFLAGS}" } -command = "cargo" -args = [ - "test", - "@@split(TARGET_JSON, )", - "--features=${TARGET_FEATURES}", - "@@split(RUST_STD, )", -] - -# @todo add [tasks.test-on-rpi4-qemu] - -[tasks.docs] -# env = { "TARGET_FEATURES" = "" } -command = "cargo" -args = [ - "doc", - "--open", - "--no-deps", - "@@split(TARGET_JSON, )", - "--features=${TARGET_FEATURES}", -] - -# Run xtool-clippy in all subprojects -[tasks.clippy] -dependencies = ["x-clippy-nucleus", "x-clippy-bin-chainboot", "x-clippy-bin-chainofcommand"] - -[tasks.x-clippy-nucleus] -command = "cargo" -# Have to use args to support env vars across platforms, regular scripts don't work on Windows -args = [ - "make", - "--makefile", - "${CARGO_MAKE_WORKING_DIRECTORY}/nucleus/Makefile.toml", - "--cwd", - "nucleus", - "clippy", -] - -[tasks.x-clippy-bin-chainboot] -command = "cargo" -# Have to use args to support env vars across platforms, regular scripts don't work on Windows -args = [ - "make", - "--makefile", - "${CARGO_MAKE_WORKING_DIRECTORY}/bin/chainboot/Makefile.toml", - "--cwd", - "bin/chainboot", - "clippy", -] - -[tasks.x-clippy-bin-chainofcommand] -command = "cargo" -# Have to use args to support env vars across platforms, regular scripts don't work on Windows -args = [ - "make", - "--makefile", - "${CARGO_MAKE_WORKING_DIRECTORY}/bin/chainofcommand/Makefile.toml", - "--cwd", - "bin/chainofcommand", - "clippy", -] - -# These tasks are written in cargo-make's own script to make them portable across platforms (no `basename` on Windows) - -# #Copy and prepare a given ELF file. Convert to binary output format. -[tasks.convert-custom-binary] -env = { "BINARY_FILE" = "${BINARY_FILE}" } -script_runner = "@duckscript" -script = ''' - binaryFile = basename ${BINARY_FILE} - outElf = set ${CARGO_MAKE_WORKSPACE_WORKING_DIRECTORY}/target/${binaryFile}.elf - outBin = set ${CARGO_MAKE_WORKSPACE_WORKING_DIRECTORY}/target/${binaryFile}.bin - cp ${BINARY_FILE} ${outElf} - exec --fail-on-error ${OBJCOPY} %{OBJCOPY_PARAMS} ${BINARY_FILE} ${outBin} - elfSize = get_file_size ${outElf} - binSize = get_file_size ${outBin} - shortBinary = replace ${BINARY_FILE} "${CARGO_MAKE_WORKSPACE_WORKING_DIRECTORY}/" "" - shortOutElf = replace ${outElf} "${CARGO_MAKE_WORKSPACE_WORKING_DIRECTORY}/" "" - shortOutBin = replace ${outBin} "${CARGO_MAKE_WORKSPACE_WORKING_DIRECTORY}/" "" - echo - echo 🔄 Processing ${shortBinary}: - echo 🔄 Copied ${binaryFile} to ${shortOutElf} (${elfSize} bytes) - echo 💫 Converted ${binaryFile} to ${shortOutBin} (${binSize} bytes) - echo -''' -install_crate = { crate_name = "cargo-binutils", binary = "rust-objcopy", test_arg = [ - "--help", -] } - -# #Copy and prepare binary with tests. -[tasks.test-binary] -env = { "BINARY_FILE" = "${CARGO_MAKE_TASK_ARGS}" } -extend = "convert-custom-binary" - -# #Run binary with tests in QEMU. -# This target is invoked by runner in .cargo/config.toml to run test binary. -[tasks.qemu-test-runner] -dependencies = ["test-binary"] -script_runner = "@duckscript" -script = [ - ''' - binaryFile = basename ${CARGO_MAKE_TASK_ARGS} - echo 🚜 Run QEMU %{QEMU_OPTS} %{QEMU_TESTS_OPTS} - echo 🚜 .. with target/${binaryFile}.bin - # echo 🚜 .. and targets/bcm2710-rpi-3-b-plus.dtb - # echo ${QEMU} %{QEMU_OPTS} %{QEMU_TESTS_OPTS} -dtb ${CARGO_MAKE_WORKSPACE_WORKING_DIRECTORY}/targets/bcm2710-rpi-3-b-plus.dtb -kernel ${CARGO_MAKE_WORKSPACE_WORKING_DIRECTORY}/target/${binaryFile}.bin - echo ${QEMU} %{QEMU_OPTS} %{QEMU_TESTS_OPTS} -kernel ${CARGO_MAKE_WORKSPACE_WORKING_DIRECTORY}/target/${binaryFile}.bin - exec --fail-on-error ${QEMU} %{QEMU_OPTS} %{QEMU_TESTS_OPTS} -dtb ${CARGO_MAKE_WORKSPACE_WORKING_DIRECTORY}/targets/bcm2710-rpi-3-b-plus.dtb -kernel ${CARGO_MAKE_WORKSPACE_WORKING_DIRECTORY}/target/${binaryFile}.bin -''', -] - -# #Generate GDB startup configuration file. -[tasks.generate-gdb-config] -script_runner = "@duckscript" -script = [ - ''' - writefile ${GDB_CONNECT_FILE} "target extended-remote :5555\n" - appendfile ${GDB_CONNECT_FILE} "break *0x80000\n" - appendfile ${GDB_CONNECT_FILE} "break kernel_init\n" - appendfile ${GDB_CONNECT_FILE} "break kernel_main\n" - echo 🖌️ Generated GDB config file -''', -] -# appendfile ${GDB_CONNECT_FILE} "continue\n" - -# #Generate zellij configuration file. -[tasks.generate-zellij-config] -dependencies = ["build-qemu", "kernel-binary"] -script_runner = "@duckscript" -env = { "ZELLIJ_CONFIG_FILE" = "${CARGO_MAKE_WORKSPACE_WORKING_DIRECTORY}/emulation/zellij-config.sh" } -script = [ - ''' - writefile ${ZELLIJ_CONFIG_FILE} "QEMU=\"${QEMU}\"\n" - appendfile ${ZELLIJ_CONFIG_FILE} "QEMU_OPTS=\"${QEMU_OPTS}\"\n" - appendfile ${ZELLIJ_CONFIG_FILE} "QEMU_RUNNER_OPTS=\"${QEMU_RUNNER_OPTS}\"\n" - appendfile ${ZELLIJ_CONFIG_FILE} "CARGO_MAKE_WORKSPACE_WORKING_DIRECTORY=\"${CARGO_MAKE_WORKSPACE_WORKING_DIRECTORY}\"\n" - appendfile ${ZELLIJ_CONFIG_FILE} "TARGET_DTB=\"${TARGET_DTB}\"\n" - appendfile ${ZELLIJ_CONFIG_FILE} "KERNEL_BIN=\"${KERNEL_BIN}\"\n" -''', -] -install_crate = { crate_name = "zellij", binary = "zellij", test_arg = [ - "--help", -] } - -[tasks.openocd] -script = [ - "${OPENOCD} -f interface/jlink.cfg -f ../ocd/${TARGET_BOARD}_target.cfg", -] - -[tasks.sdeject] -dependencies = ["sdcard"] -script = ["diskutil ejectAll ${VOLUME}"] - -[tasks.clean] -command = "cargo" -args = ["clean"] - -# -# === Other commands === -# - -[tasks.xtool-modules] -command = "cargo" -args = ["modules", "tree"] - -[tasks.xtool-expand-target] -# env = { "TARGET_FEATURES" = "" } -command = "cargo" -args = [ - "expand", - "@@split(TARGET_JSON, )", - "--features=${TARGET_FEATURES}", - "--release" -] - -[tasks.xtool-clippy] -env = { RUSTFLAGS = "${FIXED_RUSTFLAGS} ${BOARD_RUSTFLAGS} ${TARGET_LINK_RUSTFLAGS}" } -script = ''' - export RUSTFLAGS="${RUSTFLAGS}" - cargo clippy ${TARGET_JSON} --features=${CLIPPY_FEATURES} ${RUST_STD} -- --deny warnings --allow deprecated -''' diff --git a/bin/chainboot/Cargo.toml b/bin/chainboot/Cargo.toml index f8565f80a..697ad3931 100644 --- a/bin/chainboot/Cargo.toml +++ b/bin/chainboot/Cargo.toml @@ -22,18 +22,11 @@ noserial = [] [dependencies] aarch64-cpu = { workspace = true } -bit_field = { workspace = true } -bitflags = { workspace = true } -cfg-if = { workspace = true } libconsole = { workspace = true } liblog = { workspace = true } libmachine = { workspace = true } libplatform = { workspace = true } seahash = { workspace = true } -snafu = { workspace = true } -tock-registers = { workspace = true } -usize_conversions = { workspace = true } -ux = { workspace = true } [dev-dependencies] libqemu = { workspace = true } diff --git a/bin/chainboot/Makefile.toml b/bin/chainboot/Makefile.toml deleted file mode 100644 index 5b26b7205..000000000 --- a/bin/chainboot/Makefile.toml +++ /dev/null @@ -1,100 +0,0 @@ -# -# SPDX-License-Identifier: BlueOak-1.0.0 -# -# Copyright (c) Berkus Decker -# -# Build chainboot binary -# -extend = "../../Makefile.toml" - -[env] -CHAINBOOT_LINK = "bin/chainboot/src/link.ld" -CHAINBOOT_ELF = "${CARGO_MAKE_WORKSPACE_WORKING_DIRECTORY}/target/${TARGET}/release/chainboot" -CHAINBOOT_BIN = "${CARGO_MAKE_WORKSPACE_WORKING_DIRECTORY}/target/chainboot.bin" - -[tasks.build] -clear = true -dependencies = ["build-target-chainboot", "convert-kernel-binary"] -script_runner = "@duckscript" -script = ''' - shortBinary = replace ${KERNEL_BIN} "${CARGO_MAKE_WORKSPACE_WORKING_DIRECTORY}/" "" - - echo - echo ✅ chainboot built for hw ${TARGET_BOARD} ${BOARD_RUSTFLAGS}. - echo - echo 🎠 Run the following command in your terminal: - echo 🎠 target/debug/chainofcommand ${CHAINBOOT_SERIAL} ${CHAINBOOT_BAUD} --kernel ${shortBinary} - echo -''' - -[tasks.build-target-chainboot] -env = { TARGET_BOARD = "rpi4", TARGET_LINK_RUSTFLAGS = "-C link-arg=--script=${CHAINBOOT_LINK}", BOARD_RUSTFLAGS = "${BOARD_rpi4_RUSTFLAGS}" } -run_task = "build-target" - -# Generate a bootable binary file from CHAINBOOT_ELF -[tasks.convert-kernel-binary] -workspace = false -env = { "BINARY_FILE" = "${CHAINBOOT_ELF}" } -extend = "convert-custom-binary" - -[tasks.test-chainboot] -env = { TARGET_LINK_RUSTFLAGS = "-C link-arg=--script=${CHAINBOOT_LINK}" } -run_task = "test-on-rpi3-qemu" - -[tasks.zellij] -workspace = false -env = { "KERNEL_BIN" = "${CHAINBOOT_BIN}", "QEMU_OPTS" = "${QEMU_OPTS} ${QEMU_DISASM_OPTS}" } -extend = "zellij-config" - -[tasks.zellij-gdb] -workspace = false -env = { "KERNEL_BIN" = "${CHAINBOOT_BIN}", "QEMU_OPTS" = "${QEMU_OPTS} ${QEMU_DISASM_OPTS} ${QEMU_GDB_OPTS}", "TARGET_BOARD" = "rpi3", "TARGET_DTB" = "${CARGO_MAKE_WORKSPACE_WORKING_DIRECTORY}/targets/bcm2710-rpi-3-b-plus.dtb" } -extend = "zellij-config" - -[tasks.qemu] -workspace = false -env = { "QEMU_RUNNER_OPTS" = "${QEMU_DISASM_OPTS} -serial pty", "KERNEL_BIN" = "${CHAINBOOT_BIN}", "TARGET_DTB" = "${CARGO_MAKE_WORKSPACE_WORKING_DIRECTORY}/targets/bcm2710-rpi-3-b-plus.dtb" } -extend = "qemu-runner" - -[tasks.qemu-gdb] -workspace = false -env = { "QEMU_RUNNER_OPTS" = "${QEMU_DISASM_OPTS} ${QEMU_GDB_OPTS} -serial pty", "KERNEL_BIN" = "${CHAINBOOT_BIN}", "TARGET_DTB" = "${CARGO_MAKE_WORKSPACE_WORKING_DIRECTORY}/targets/bcm2710-rpi-3-b-plus.dtb" } -extend = "qemu-runner" - -[tasks.gdb] -workspace = false -dependencies = ["build", "build-kernel-binary", "gdb-config"] -env = { "RUST_GDB" = "${GDB}" } -script = ["exec < /dev/tty && rust-gdb -x ${GDB_CONNECT_FILE} ${CHAINBOOT_ELF}"] - -[tasks.sdcard] -workspace = false -dependencies = ["build", "build-kernel-binary"] -script_runner = "@duckscript" -script = [ - ''' - kernelImage = set "chain_boot_rpi4.img" - cp ${CHAINBOOT_BIN} ${VOLUME}/${kernelImage} - echo 🔄 Copied chainboot to ${VOLUME}/${kernelImage} -''', -] - -# Just use sdeject -# [tasks.cb-eject] -# clean = true -# alias = "cb-eject-chainboot" -# -# [tasks.cb-eject-chainboot] -# dependencies = ["sdeject"] - -[tasks.clippy] -clear = true -dependencies = ["clippy-rpi3", "clippy-rpi4"] - -[tasks.clippy-rpi3] -env = { TARGET_BOARD = "rpi3", TARGET_LINK_RUSTFLAGS = "-C link-arg=--script=${CHAINBOOT_LINK}", BOARD_RUSTFLAGS = "${BOARD_rpi3_RUSTFLAGS}" } -run_task = "xtool-clippy" - -[tasks.clippy-rpi4] -env = { TARGET_BOARD = "rpi4", TARGET_LINK_RUSTFLAGS = "-C link-arg=--script=${CHAINBOOT_LINK}", BOARD_RUSTFLAGS = "${BOARD_rpi4_RUSTFLAGS}" } -run_task = "xtool-clippy" diff --git a/bin/chainboot/src/main.rs b/bin/chainboot/src/main.rs index 5191391cd..8c490abfb 100644 --- a/bin/chainboot/src/main.rs +++ b/bin/chainboot/src/main.rs @@ -9,7 +9,7 @@ use { core::hash::Hasher, libconsole::console::console, liblog::{print, println}, - libplatform::platform::raspberrypi::BcmHost, + libplatform::raspberrypi::BcmHost, seahash::SeaHasher, }; @@ -20,25 +20,25 @@ use { /// - Only a single core must be active and running this function. /// - The init calls in this function must appear in the correct order. #[unsafe(no_mangle)] -unsafe extern "C" fn kernel_init(_dtb: u32, max_kernel_size: u64) -> ! { +unsafe extern "C" fn kernel_init(dtb: u32, max_kernel_size: u64) -> ! { #[cfg(feature = "jtag")] libmachine::debug::jtag::wait_debugger(); // SAFETY: VERY SAFE - if let Err(x) = unsafe { libplatform::platform::drivers::init() } { + if let Err(x) = unsafe { libplatform::drivers::init() } { panic!("Error initializing platform drivers: {}", x); } // Initialize all device drivers. // SAFETY: Relatively safe. unsafe { - libplatform::platform::drivers::driver_manager().init_drivers_and_irqs(); + libplatform::drivers::driver_manager().init_drivers_and_irqs(); } // println! is usable from here on. // Transition from unsafe to safe. - kernel_main(max_kernel_size) + kernel_main(dtb, max_kernel_size) } // https://onlineasciitools.com/convert-text-to-ascii-art (FIGlet) with `cricket` font @@ -63,9 +63,13 @@ fn read_u64() -> u64 { /// The main function running after the early init. #[inline(always)] -fn kernel_main(max_kernel_size: u64) -> ! { +fn kernel_main(dtb: u32, max_kernel_size: u64) -> ! { + #[cfg(test)] + test_main(); + print!("{}", LOGO); println!("{:>51}\n", BcmHost::board_name()); + println!("Preserving DTB at {:8x}", dtb); println!("⏪ Requesting kernel image..."); let kernel_addr: *mut u8 = BcmHost::kernel_load_address() as *mut u8; @@ -136,13 +140,13 @@ fn kernel_main(max_kernel_size: u64) -> ! { // Use black magic to create a function pointer. // SAFETY: We're getting to safety soon! - let kernel: fn() -> ! = unsafe { core::mem::transmute(kernel_addr) }; + let kernel: fn(u32) -> ! = unsafe { core::mem::transmute(kernel_addr) }; // Force everything to complete before we jump. barrier::isb(barrier::SY); // Jump to loaded kernel! - kernel() + kernel(dtb) } #[panic_handler] diff --git a/bin/chainboot/tests/chainboot.rs b/bin/chainboot/tests/chainboot.rs index b4c8fc8be..7993d7ae9 100644 --- a/bin/chainboot/tests/chainboot.rs +++ b/bin/chainboot/tests/chainboot.rs @@ -7,7 +7,7 @@ mod common; // This test essentially validates that a relocated chainboot executable runs correctly -#[test_case] -fn relocated_binary_works() { - assert_eq!(2 + 2, 4); -} +// #[test_case] +// fn relocated_binary_works() { +// assert_eq!(2 + 2, 4); +// } diff --git a/bin/chainofcommand/Cargo.toml b/bin/chainofcommand/Cargo.toml index b0d965562..241a8078a 100644 --- a/bin/chainofcommand/Cargo.toml +++ b/bin/chainofcommand/Cargo.toml @@ -19,12 +19,12 @@ argh = { workspace = true } bytes = { workspace = true } crossterm = { workspace = true } futures = { workspace = true } -futures-util = { workspace = true } +# futures-util = { workspace = true } seahash = { workspace = true } tokio = { workspace = true } tokio-serial = { workspace = true } tokio-stream = { workspace = true } -tokio-util = { workspace = true } +# tokio-util = { workspace = true } # [lints.clippy] # needless_return = "allow" # clippy hallucinates like chatgpt diff --git a/bin/chainofcommand/Makefile.toml b/bin/chainofcommand/Makefile.toml deleted file mode 100644 index 8ea3b7149..000000000 --- a/bin/chainofcommand/Makefile.toml +++ /dev/null @@ -1,34 +0,0 @@ -# -# SPDX-License-Identifier: BlueOak-1.0.0 -# -# Copyright (c) Berkus Decker -# -# Build chainofcommand tool -# -extend = "../../Makefile.toml" - -[tasks.build] -alias = "build-coc" - -[tasks.build-coc] -workspace = false -command = "cargo" -args = ["build"] - -[tasks.test] -clear = true -command = "cargo" -args = ["test"] - -[tasks.clippy] -clear = true -command = "cargo" -args = [ - "clippy", - "--", - "--no-deps", - "--deny", - "warnings", - "--allow", - "deprecated", -] diff --git a/doc/Power_ePAPR_APPROVED_v1.1.pdf b/doc/Power_ePAPR_APPROVED_v1.1.pdf new file mode 100644 index 000000000..8f30274e5 Binary files /dev/null and b/doc/Power_ePAPR_APPROVED_v1.1.pdf differ diff --git a/doc/devicetree-specification-v0.3.pdf b/doc/devicetree-specification-v0.3.pdf new file mode 100644 index 000000000..16ed91e78 Binary files /dev/null and b/doc/devicetree-specification-v0.3.pdf differ diff --git a/emulation/layout.zellij b/emulation/layout.zellij index 8698da68c..f8e6b8b60 100644 --- a/emulation/layout.zellij +++ b/emulation/layout.zellij @@ -7,14 +7,14 @@ layout { } tab split_direction="Vertical" { pane split_direction="Vertical" { - pane command="bash" borderless=true close_on_exit=true { + pane command="bash" borderless=true close_on_exit=false { args "-c" "bash emulation/qemu_multi_uart.sh" } pane split_direction="Horizontal" { - pane command="bash" size="30%" close_on_exit=true { + pane command="bash" size="30%" close_on_exit=false { args "-c" "clear; echo -e \"\\033]0;MiniUart\\007\"; bash /dev/ptmx FIRST=1" } - pane command="bash" size="70%" close_on_exit=true { + pane command="bash" size="70%" close_on_exit=false { args "-c" "clear; echo -e \"\\033]0;PL011 Uart\\007\"; bash /dev/ptmx SECOND=1" } } diff --git a/kernel/Plan.md b/kernel/Plan.md new file mode 100644 index 000000000..e7dd87581 --- /dev/null +++ b/kernel/Plan.md @@ -0,0 +1,48 @@ +What's needed: quick-n-dirty higher-half mappings setup and kernel physical memory view setup (both in TTBR1), identity mapping for kickstart (in TTBR0). + +- `__kernel_start` to `__kernel_end` map at KERNEL_HIGH_BASE +- 0 to phys_ram_size (from DTB) map at KERNEL_PHYS_WINDOW +- `__kickstart_start` till `__kickstart_end` identity-map (no code/data split yet?) + +Steps: +- [ ] Move tests from kernel/tests to individual libs? + +- [ ] Make some caps work - Untypeds, Domains, Buffers, what else? +- [ ] Test out syscalls from EL0 + +- START FILLING IN CAPS + - [ ] untypeds + - [ ] kickstart context and domain + +--- + +# Completed + +- [x] Buildable +- [x] build all shit together into a binary +- [x] make separate kickstart section and start booting from kickstart +- [x] make early_print work +- [x] parse dtb +- [x] Print that we can invoke kernel function using a syscall from the kickstart (even it if runs at the same EL for now) +- [x] Enter kernel init in EL2 - this will be needed to set up kernel mappings +- [x] Print DTB +- [x] Print max RAM from DTB + +- [x] Allocate a single key slot in a global domain struct +- [x] Fill it with capability to DebugConsole +- [x] Invoke DebugConsole.Write via syscall + +- [x] Generate complete memory map from DTB, print it out + +- [x] Print kernel covered area +- [x] Print KERNEL_HIGH_BASE +- [x] Print kernel mappings size and attribs +- [x] Print kickstart covered area +- [x] Print kickstart mappings size + + + +Whatever kernel links must also be located in high-mem mapping, so we cannot share this code with kickstart at all! +This means it's probably sensible to build kernel as a separate ELF file linked entirely high, then merge it with the kickstart binary through specially-named sections; there should be no symbol resolution across two binaries, so the nucleus image is solely pulled via it's PHDRS (but we need to place the BSS which will be erased by the kickstart before turning the MMU on) + +- [x] See gh:metta-systems/kernel-embed-prototype for an outline of this approach - copy it here and lets go. diff --git a/kernel/kernel.map b/kernel/kernel.map new file mode 100644 index 000000000..6147d679a --- /dev/null +++ b/kernel/kernel.map @@ -0,0 +1,53 @@ + [pa00_0000_0000 - pa00_0000_1000) | 4 KiB | (Used) C RW PXN | Reserved + [pa00_0000_1000 - pa00_0008_0000) | 508 KiB | (Free) C RW PXN | RAM + [pa00_0008_0000 - pa00_0020_0000) | 2 MiB | (Drop) C RW PXN | Init_Thread + [pa00_0020_0000 - pa00_0020_7260) | 29 KiB | (Free) C RW PXN | RAM + [pa00_0020_0000 - pa00_0020_7260) | 29 KiB | (Drop) C RW PXN | DTB index + [pa00_0020_7260 - pa00_0040_0000) | 2 MiB | (Drop) C RW PXN | Init_Thread + [pa00_0040_0000 - pa00_0040_3800) | 14 KiB | (Used) C RO PX | Nucleus code + [pa00_0040_3800 - pa00_0041_0000) | 50 KiB | (Drop) C RW PXN | Init_Thread + [pa00_0041_0000 - pa00_0042_0000) | 64 KiB | (Used) C RO PXN | Nucleus read-only data + [pa00_0042_0000 - pa00_0043_0000) | 64 KiB | (Used) C RW PXN | Nucleus data + [pa00_0043_0000 - pa00_0044_0000) | 64 KiB | (Used) C RW PXN | Nucleus BSS + [pa00_0044_0000 - pa00_004c_0000) | 512 KiB | (Used) C RW PXN | Nucleus stack + [pa00_004c_0000 - pa00_004c_1000) | 4 KiB | (Drop) C RW PXN | user L0 + [pa00_004c_1000 - pa00_004c_2000) | 4 KiB | (Used) C RW PXN | kernel L0 + [pa00_004c_2000 - pa00_004c_3000) | 4 KiB | (Drop) C RW PXN | Init_Thread identity mapping + [pa00_004c_3000 - pa00_004c_4000) | 4 KiB | (Drop) C RW PXN | Init_Thread identity mapping + [pa00_004c_4000 - pa00_004c_5000) | 4 KiB | (Used) C RW PXN | Nucleus section mapping + [pa00_004c_5000 - pa00_004c_6000) | 4 KiB | (Used) C RW PXN | Nucleus section mapping + [pa00_004c_6000 - pa00_004c_7000) | 4 KiB | (Used) C RW PXN | Nucleus section mapping + [pa00_004c_7000 - pa00_004c_8000) | 4 KiB | (Used) C RW PXN | Nucleus phys memory mapping + [pa00_004c_8000 - pa00_004c_9000) | 4 KiB | (Used) C RW PXN | Nucleus phys memory mapping + [pa00_004c_9000 - pa00_0800_0000) | 124 MiB | (Free) C RW PXN | RAM + [pa00_0800_0000 - pa00_0801_2d8e) | 76 KiB | (Drop) C RW PXN | DTB + [pa00_0801_2d8e - pa00_3c00_0000) | 832 MiB | (Free) C RW PXN | RAM + [pa00_4000_0000 - pa00_4000_0100) | 256 B | (Used) Dev RW PXN | local_intc + [pa00_7e00_4000 - pa00_7e00_4020) | 32 B | (Used) Dev RW PXN | txp + [pa00_7e00_7000 - pa00_7e00_7f00) | 4 KiB | (Used) Dev RW PXN | dma + [pa00_7e00_9800 - pa00_7e00_9900) | 256 B | (Used) Dev RW PXN | axiperf + [pa00_7e00_a000 - pa00_7e00_a024) | 36 B | (Used) Dev RW PXN | watchdog + [pa00_7e00_b200 - pa00_7e00_b400) | 512 B | (Used) Dev RW PXN | interrupt-controller + [pa00_7e00_b840 - pa00_7e00_b87c) | 60 B | (Used) Dev RW PXN | mailbox + [pa00_7e00_b880 - pa00_7e00_b8c0) | 64 B | (Used) Dev RW PXN | mailbox + [pa00_7e10_0000 - pa00_7e10_0114) | 276 B | (Used) Dev RW PXN | watchdog + [pa00_7e10_1000 - pa00_7e10_3000) | 8 KiB | (Used) Dev RW PXN | cprman + [pa00_7e10_4000 - pa00_7e10_4010) | 16 B | (Used) Dev RW PXN | rng + [pa00_7e20_0000 - pa00_7e20_1000) | 4 KiB | (Used) Dev RW PXN | gpiomem + [pa00_7e20_1000 - pa00_7e20_1200) | 512 B | (Used) Dev RW PXN | serial + [pa00_7e20_3000 - pa00_7e20_3024) | 36 B | (Used) Dev RW PXN | i2s + [pa00_7e20_6000 - pa00_7e20_6100) | 256 B | (Used) Dev RW PXN | pixelvalve + [pa00_7e20_7000 - pa00_7e20_7100) | 256 B | (Used) Dev RW PXN | pixelvalve + [pa00_7e20_c000 - pa00_7e20_c028) | 40 B | (Used) Dev RW PXN | pwm + [pa00_7e21_2000 - pa00_7e21_2008) | 8 B | (Used) Dev RW PXN | thermal + [pa00_7e21_5000 - pa00_7e21_5008) | 8 B | (Used) Dev RW PXN | aux + [pa00_7e21_5040 - pa00_7e21_5080) | 64 B | (Used) Dev RW PXN | serial + [pa00_7e30_0000 - pa00_7e30_0100) | 256 B | (Used) Dev RW PXN | mmcnr + [pa00_7e40_0000 - pa00_7e40_6000) | 24 KiB | (Used) Dev RW PXN | hvs + [pa00_7e60_0000 - pa00_7e60_0100) | 256 B | (Used) Dev RW PXN | firmwarekms + [pa00_7e80_6000 - pa00_7e80_7000) | 4 KiB | (Used) Dev RW PXN | vec + [pa00_7e80_7000 - pa00_7e80_7100) | 256 B | (Used) Dev RW PXN | pixelvalve + [pa00_7e80_8000 - pa00_7e80_8100) | 256 B | (Used) Dev RW PXN | hdmi + [pa00_7e90_2000 - pa00_7e90_2600) | 2 KiB | (Used) Dev RW PXN | hdmi + [pa00_7ec0_0000 - pa00_7ec0_1000) | 4 KiB | (Used) Dev RW PXN | v3d + [pa00_7ee0_8000 - pa00_7ee0_8100) | 256 B | (Used) Dev RW PXN | axiperf diff --git a/kernel/kickstart/Cargo.toml b/kernel/kickstart/Cargo.toml new file mode 100644 index 000000000..6f90a9cee --- /dev/null +++ b/kernel/kickstart/Cargo.toml @@ -0,0 +1,68 @@ +[package] +name = "kickstart" + +description = "Vesper nanokernel startup initialization." + +authors = { workspace = true } +categories = { workspace = true } +documentation = { workspace = true } +edition = { workspace = true } +homepage = { workspace = true } +license = { workspace = true } +publish = { workspace = true } +readme = { workspace = true } +repository = { workspace = true } +version = { workspace = true } + +[features] +default = [] +# Enable JTAG debugging of kernel - enable jtag helpers and +# block waiting for JTAG probe attach at the start of kernel main. +jtag = ["libmachine/jtag"] +noserial = [] #"libplatform/noserial" +# Build for running under QEMU with semihosting, so various halt/reboot options would for example quit QEMU instead. +qemu = [ + "dep:libqemu", + # "libconsole/qemu", + "libexception/qemu", + "libmachine/qemu", + # "libplatform/qemu", +] + +[dependencies] +aarch64-cpu = { workspace = true } +cfg-if = { workspace = true } +libaddress = { workspace = true } +libboot = { workspace = true } +libcpu = { workspace = true } +libexception = { workspace = true } +liblog = { workspace = true } +liblocking = { workspace = true } +libmachine = { workspace = true } +libmapping = { workspace = true } +libprint = { workspace = true } +libqemu = { workspace = true, optional = true } +libobject = { workspace = true } +libsyscall = { workspace = true } +snafu = { workspace = true } +shrinkwraprs = { version = "0.3", default-features = false } +fdt-rs = { version = "=0.4.4", default-features = false } + +[dev-dependencies] +libexception = { workspace = true, features = ["test_build"] } +liblocking = { workspace = true } +libqemu = { workspace = true } +libtest = { workspace = true } + +[build-dependencies] +goblin = { workspace = true } +build-rs = { workspace = true } +build-print = { workspace = true } +minijinja = { workspace = true } + +[[bin]] +name = "kickstart" +test = false # test are all concentrated in integration tests/ + +[lints] +workspace = true diff --git a/kernel/kickstart/DTB_devices_rpi3.txt b/kernel/kickstart/DTB_devices_rpi3.txt new file mode 100644 index 000000000..546c7e30d --- /dev/null +++ b/kernel/kickstart/DTB_devices_rpi3.txt @@ -0,0 +1,57 @@ + [1a] local_intc @ 0x40000000 +256 (brcm,bcm2836-l1-intc) + [01] interrupt-controller @ 0x7e00b200 +512 (brcm,bcm2836-armctrl-ic) + [25] serial @ 0x7e201000 +512 (arm,pl011) + [15] aux @ 0x7e215000 +8 (brcm,bcm2835-aux) + [26] serial @ 0x7e215040 +64 (brcm,bcm2835-aux-uart) + [7e] mailbox @ 0x7e00b840 +60 (brcm,bcm2836-vchiq) + [1f] mailbox @ 0x7e00b880 +64 (brcm,bcm2835-mbox) + + [10] gpio @ 0x7e200000 +180 (brcm,bcm2835-gpio) + [00] gpiomem @ 0x7e200000 +4096 (brcm,bcm2835-gpiomem) + [72] usb @ 0x7e006000 +0 (brcm,bcm2708-usb) + [72] usb @ 0x7e980000 +0 (brcm,bcm2708-usb) + [0b] dma @ 0x7e007000 +3840 (brcm,bcm2835-dma) + [2d] rng @ 0x7e104000 +16 (brcm,bcm2835-rng) + [2e] mmc @ 0x7e202000 +256 (brcm,bcm2835-sdhost) + [30] mmcnr @ 0x7e300000 +256 (brcm,bcm2835-mmc) +-[2f] mmc @ 0x7e300000 +256 (brcm,bcm2835-mmc) +-[28] spi @ 0x7e204000 +0 (brcm,bcm2835-spi) +-[6d] spi @ 0x7e215080 +0 (brcm,bcm2835-aux-spi) +-[6e] spi @ 0x7e2150c0 +0 (brcm,bcm2835-aux-spi) +-[27] i2s @ 0x7e203000 +36 (brcm,bcm2835-i2s) +-[11] i2c @ 0x7e205000 +0 (brcm,bcm2835-i2c) +-[2a] i2c @ 0x7e804000 +0 (brcm,bcm2835-i2c) +-[1b] i2c @ 0x7e805000 +0 (brcm,bcm2835-i2c) + [02] thermal @ 0x7e212000 +8 (brcm,bcm2837-thermal) + +-[3a] txp @ 0x7e004000 +32 (brcm,bcm2835-txp) + [2c] watchdog @ 0x7e00a000 +36 (brcm,bcm2835-pm) + [2c] watchdog @ 0x7e100000 +276 (brcm,bcm2835-pm) + [07] cprman @ 0x7e101000 +8192 (brcm,bcm2835-cprman) + The CPRMAN clock controller generates clocks in the audio power domain of the BCM2835. + https://github.com/torvalds/linux/blob/master/Documentation/devicetree/bindings/clock/brcm%2Cbcm2835-cprman.yaml + +-[70] hvs @ 0x7e400000 +24576 (brcm,bcm2835-hvs) +-[71] vec @ 0x7e806000 +4096 (brcm,bcm2835-vec) +-[74] pixelvalve @ 0x7e206000 +256 (brcm,bcm2835-pixelvalve0) +-[75] pixelvalve @ 0x7e207000 +256 (brcm,bcm2835-pixelvalve1) +-[76] pixelvalve @ 0x7e807000 +256 (brcm,bcm2835-pixelvalve2) +-[77] hdmi @ 0x7e808000 +256 (brcm,bcm2835-hdmi) +-[77] hdmi @ 0x7e902000 +1536 (brcm,bcm2835-hdmi) +-[78] v3d @ 0x7ec00000 +4096 (brcm,vc4-v3d) + https://github.com/torvalds/linux/blob/master/Documentation/devicetree/bindings/gpu/brcm%2Cbcm-v3d.yaml + +-[7c] csi @ 0x7e800000 +0 (brcm,bcm2835-unicam) +-[7d] csi @ 0x7e801000 +0 (brcm,bcm2835-unicam) +-[7c] csi @ 0x7e802000 +0 (brcm,bcm2835-unicam) +-[7d] csi @ 0x7e802004 +0 (brcm,bcm2835-unicam) +-[6b] dpi @ 0x7e208000 +0 (brcm,bcm2835-dpi) + +-[04] dsi @ 0x7e209000 +0 (brcm,bcm2835-dsi0) +-[05] dsi @ 0x7e700000 +0 (brcm,bcm2835-dsi1) +-[7b] smi @ 0x7e600000 +256 (brcm,bcm2835-smi) +-[6f] pwm @ 0x7e20c000 +40 (brcm,bcm2835-pwm) + +-[31] axiperf @ 0x7e009800 +256 (brcm,bcm2835-axiperf) +-[31] axiperf @ 0x7ee08000 +256 (brcm,bcm2835-axiperf) +-[7a] firmwarekms @ 0x7e600000 +256 (raspberrypi,rpi-firmware-kms) diff --git a/kernel/kickstart/build.rs b/kernel/kickstart/build.rs new file mode 100644 index 000000000..9748324ea --- /dev/null +++ b/kernel/kickstart/build.rs @@ -0,0 +1,363 @@ +// Parse nucleus ELF and extract sections + +use { + build_print::info, + build_rs::output, + goblin::elf::{Elf, Sym, program_header::PT_LOAD, section_header::SHT_NOBITS}, + std::{ + env, + fs::{self, File}, + io::Read, + path::Path, + }, +}; + +fn main() { + let kernel_elf_path = "../../target/aarch64-metta-none-eabi/release/nucleus"; // must be passed-in as input? + + output::rerun_if_changed(kernel_elf_path); + output::rerun_if_changed("build.rs"); + output::rerun_if_changed("kernel_sections.template.rs"); + + let out_dir = env::var("OUT_DIR").unwrap(); + let out_path = Path::new(&out_dir); + + // Read kernel ELF + let mut elf_bytes = Vec::new(); + File::open(kernel_elf_path) + .expect("Failed to open kernel ELF - build kernel first with: just build") + .read_to_end(&mut elf_bytes) + .expect("Failed to read kernel ELF"); + + // Parse with goblin + let elf = Elf::parse(&elf_bytes).expect("Failed to parse kernel ELF"); + + // Verify it's what we expect + assert!(elf.is_64, "Kernel must be ELF64"); + assert_eq!( + elf.header.e_machine, + goblin::elf::header::EM_AARCH64, + "Kernel must be AArch64" + ); + + // Extract section information + let sections = extract_sections(&elf, &elf_bytes, out_path); + + // Extract stack mapping information + let stack_virt_bottom = stack_virt_bottom(&elf); + + // Generate Rust code + generate_rust_code(§ions, stack_virt_bottom, out_path); +} + +/// Extracted section with all metadata needed for loading +#[derive(Debug)] +struct ExtractedSection { + name: String, + /// Virtual address in kernel's address space (higher-half) + virt_addr: u64, + /// Size in memory + mem_size: u64, + /// Size in file (0 for BSS) + // file_size: usize, + /// Required alignment + alignment: u64, + /// Permissions + readable: bool, + writable: bool, + executable: bool, + /// Is this a NOBITS section (BSS)? + // is_nobits: bool, + /// Output binary file name (None for BSS) + bin_file: Option, +} + +impl ExtractedSection { + fn meta_name(&self) -> String { + format!("{}_META", self.name.trim_start_matches('.').to_uppercase()) + } + + fn bin_name(&self) -> String { + format!( + "KERNEL_{}_BIN", + self.name.trim_start_matches('.').to_uppercase() + ) + } +} + +/// Exception vector table information +#[derive(Debug)] +struct VectorTableInfo { + /// Virtual address of the vector table + virt_addr: u64, + /// Size of the vector table (should be 0x800 = 2048 bytes) + mem_size: u64, + /// Alignment requirement (must be 2KB aligned for VBAR) + alignment: u64, +} + +/// All extracted kernel metadata +#[derive(Debug)] +struct KernelSections { + /// Virtual base address (start of kernel in virtual memory) + virt_base: u64, + /// Loadable sections (text, rodata, data) + load_sections: Vec, + /// BSS section (separate because it has no file content) + bss_section: Option, + /// Exception vector table info + vector_table: Option, +} + +fn extract_sections(elf: &Elf, elf_bytes: &[u8], out_path: &Path) -> KernelSections { + let mut load_sections = Vec::new(); + let mut bss_section = None; + let mut virt_base = u64::MAX; + + // First pass: find virtual base from program headers + for ph in &elf.program_headers { + if ph.p_type == PT_LOAD { + virt_base = virt_base.min(ph.p_vaddr); + } + } + + // Build a map of section names to their info + let section_info: Vec<_> = elf + .section_headers + .iter() + .filter_map(|sh| { + let name = elf.shdr_strtab.get_at(sh.sh_name)?; + Some((name, sh)) + }) + .collect(); + + // Extract relevant sections + for (name, sh) in §ion_info { + // Skip sections we don't care about + if !matches!(*name, ".text" | ".rodata" | ".data" | ".bss") { + continue; + } + + let is_nobits = sh.sh_type == SHT_NOBITS; + + // Determine permissions from section flags + let readable = true; + let writable = sh.sh_flags & u64::from(goblin::elf::section_header::SHF_WRITE) != 0; + let executable = sh.sh_flags & u64::from(goblin::elf::section_header::SHF_EXECINSTR) != 0; + + let section = ExtractedSection { + name: name.to_string(), + virt_addr: sh.sh_addr, + mem_size: sh.sh_size, + // file_size: if is_nobits { 0 } else { sh.sh_size as usize }, + alignment: sh.sh_addralign, + readable, + writable, + executable, + // is_nobits, + bin_file: None, + }; + + if is_nobits { + bss_section = Some(section); + } else { + load_sections.push(section); + } + } + + // Try to find vectors via symbol + let vector_table = find_vector_table_from_symbols(elf); + + // Sort load sections by virtual address + load_sections.sort_by_key(|s| s.virt_addr); + + // Extract binary content for each loadable section + for section in &mut load_sections { + let sh = section_info + .iter() + .find(|(name, _)| *name == section.name) + .map(|(_, sh)| *sh) + .unwrap(); + + let start = usize::try_from(sh.sh_offset).unwrap(); + let end = usize::try_from(sh.sh_offset + sh.sh_size).unwrap(); + let content = &elf_bytes[start..end]; + + let bin_filename = format!("kernel_{}.bin", section.name.trim_start_matches('.')); + let bin_path = out_path.join(&bin_filename); + fs::write(&bin_path, content) + .unwrap_or_else(|e| panic!("Failed to write {bin_filename}: {e}")); + + section.bin_file = Some(bin_filename); + + info!( + "Extracted {}: vaddr=0x{:016X}, size=0x{:X}, align={}, perms={}{}{}", + section.name, + section.virt_addr, + section.mem_size, + section.alignment, + if section.readable { "R" } else { "-" }, + if section.writable { "W" } else { "-" }, + if section.executable { "X" } else { "-" }, + ); + } + + if let Some(ref bss) = bss_section { + info!( + "BSS section: vaddr=0x{:016X}, size=0x{:X}, align={}", + bss.virt_addr, bss.mem_size, bss.alignment, + ); + } + + KernelSections { + virt_base, + load_sections, + bss_section, + vector_table, + } +} + +fn find_symbol(elf: &Elf, symbol_name: &str) -> Option { + for sym in &elf.syms { + if let Some(name) = elf.strtab.get_at(sym.st_name) + && symbol_name == name + { + return Some(sym); + } + } + + None +} + +fn map_section_name(name: &String) -> String { + match name.as_ref() { + ".text" => "Nucleus code".to_string(), + ".rodata" => "Nucleus read-only data".to_string(), + ".data" => "Nucleus data".to_string(), + x => x.to_string(), + } +} + +/// Try to find vector table location from symbols +fn find_vector_table_from_symbols(elf: &Elf) -> Option { + // Common symbol names for exception vectors + const VECTOR_SYMBOL: &str = "__exception_vectors_start"; // From libexception::arch + + if let Some(sym) = find_symbol(elf, VECTOR_SYMBOL) { + info!( + "Found vector table symbol '{}': vaddr=0x{:016X}, size=0x{:X}", + VECTOR_SYMBOL, sym.st_value, sym.st_size + ); + + // Verify 2KB alignment + if sym.st_value & 0x7FF != 0 { + info!( + "Vector table symbol at 0x{:016X} is not 2KB aligned!", + sym.st_value + ); + } + + return Some(VectorTableInfo { + virt_addr: sym.st_value, + // If size is 0, assume standard size of 0x800 + mem_size: if sym.st_size > 0 { sym.st_size } else { 0x800 }, + alignment: 2048, // VBAR requirement + }); + } + + output::error("No vector table symbol found! Kernel must define __vectors symbol"); + + None +} + +fn stack_virt_bottom(elf: &Elf) -> u64 { + const STACK_VIRT_BOTTOM: &str = "__STACK_VIRT_BOTTOM"; + if let Some(sym) = find_symbol(elf, STACK_VIRT_BOTTOM) { + return sym.st_value; + } + output::error("No stack bottom symbol found! Kernel must define __STACK_VIRT_BOTTOM symbol"); + 0 +} + +fn generate_rust_code(sections: &KernelSections, stack_virt_bottom: u64, out_path: &Path) { + use minijinja::{Environment, context}; + + let mut code = Environment::new(); + code.set_trim_blocks(true); + code.add_filter("address", |v: usize| format!("0x{v:016X}")); + code.add_filter("hex", |v: usize| format!("0x{v:X}")); + code.add_template_owned( + "sections", + std::fs::read_to_string("kernel_sections.template.rs").unwrap(), + ) + .unwrap(); + let tmpl = code.get_template("sections").unwrap(); + + let sections_tmpl: Vec<_> = sections + .load_sections + .iter() + .map(|section| { + context! { + name => map_section_name(§ion.name), + meta_name => section.meta_name(), + bin_name => section.bin_name(), + bin_file => section.bin_file, + virt_addr => section.virt_addr, + size => section.mem_size, + align => section.alignment, + r => section.readable, + w => section.writable, + x => section.executable, + } + }) + .collect(); + + if sections.bss_section.is_none() { + output::error("No BSS section for kernel, that's a bummer!"); + } + + let bss_section = sections.bss_section.as_ref().unwrap(); + + let bss_tmpl = context! { + virt_addr => bss_section.virt_addr, + size => bss_section.mem_size, + align => bss_section.alignment, + }; + + if sections.vector_table.is_none() { + output::error("No vector table section for kernel, that's a bummer!"); + } + + let vector_table = sections.vector_table.as_ref().unwrap(); + + let vector_tmpl = context! { + virt_addr => vector_table.virt_addr, + size => vector_table.mem_size, + align => vector_table.alignment, + }; + + let context = context! { + virt_addr => sections.virt_base, + sections => sections_tmpl, + bss => bss_tmpl, + vectors => vector_tmpl, + stack_virt_bottom, + }; + + // Write generated code + let dest_path = out_path.join("kernel_sections.rs"); + fs::write( + &dest_path, + tmpl.render(context).expect("failed to render template"), + ) + .expect("Failed to write generated code"); + + info!("Generated: {}", dest_path.display()); + info!("Kernel virtual base: 0x{:016X}", sections.virt_base); + if let Some(ref v) = sections.vector_table { + info!( + "Vector table: 0x{:016X} (size: 0x{:X})", + v.virt_addr, v.mem_size + ); + } +} diff --git a/kernel/kickstart/kernel_sections.template.rs b/kernel/kickstart/kernel_sections.template.rs new file mode 100644 index 000000000..9e5f2cce3 --- /dev/null +++ b/kernel/kickstart/kernel_sections.template.rs @@ -0,0 +1,69 @@ +// Auto-generated kernel section metadata and binary includes +// DO NOT EDIT - Generated by build.rs + +#[allow(unused)] +use crate::{ + loader::{ImageInfo, SectionMeta, LoadableSection}, + memory::MemoryPermissions +}; + +// ═══════════════════════════════════════════════════════════════ +// Binary section data (included at compile time) +// ═══════════════════════════════════════════════════════════════ + +{% for section in sections %} +static {{section.bin_name}}: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/{{section.bin_file}}")); +{% endfor %} + +// ═══════════════════════════════════════════════════════════════ +// Section metadata +// ═══════════════════════════════════════════════════════════════ + +{% for section in sections %} +pub const {{section.meta_name}}: SectionMeta = SectionMeta { + name: "{{section.name}}", + virt_addr: {{section.virt_addr | address}}, + size: {{section.size | hex}}, + alignment: {{section.align | hex}}, + permissions: MemoryPermissions { readable: {{section.r}}, writable: {{section.w}}, executable: {{section.x}} }, +}; +{% endfor %} + +pub const BSS_META: SectionMeta = SectionMeta { + name: ".bss", + virt_addr: {{bss.virt_addr | address}}, + size: {{bss.size | hex}}, + alignment: {{bss.align | hex}}, + permissions: MemoryPermissions { readable: true, writable: true, executable: false }, +}; + +// ═══════════════════════════════════════════════════════════════ +// Exception vector table +// ═══════════════════════════════════════════════════════════════ + +pub const VECTORS_META: SectionMeta = SectionMeta { + name: ".vectors", + virt_addr: {{vectors.virt_addr | address}}, + size: {{vectors.size | hex}}, + alignment: {{vectors.align | hex}}, + permissions: MemoryPermissions { readable: true, writable: false, executable: true }, +}; + +// ═══════════════════════════════════════════════════════════════ +// Combined kernel sections +// ═══════════════════════════════════════════════════════════════ + +static LOADABLE_SECTIONS: &[LoadableSection] = &[ +{% for section in sections %} + LoadableSection { meta: {{section.meta_name}}, data: {{section.bin_name}} }, +{% endfor %} +]; + +/// Complete kernel image information +pub static KERNEL: ImageInfo = ImageInfo { + virt_base: {{virt_addr | address}}, + sections: LOADABLE_SECTIONS, + bss: BSS_META, + vectors: VECTORS_META, + stack_virt_bottom: {{stack_virt_bottom | address}}, +}; diff --git a/kernel/kickstart/src/boot_info.rs b/kernel/kickstart/src/boot_info.rs new file mode 100644 index 000000000..449210c1b --- /dev/null +++ b/kernel/kickstart/src/boot_info.rs @@ -0,0 +1,1042 @@ +//! Boot regions +//! +//! Define a map of memory regions used during boot allocations. +//! +//! Insert sections that are either "free" or "used", disparate used sections can not overlap, +//! overlapping free sections are merged (unless they have different `MemAttributes`). +//! +#[cfg(feature = "qemu")] +use libqemu::semi_println; +use { + core::{cell::LazyCell, fmt}, + libaddress::PhysAddr, + liblocking::IRQSafeNullLock, + libmapping::{AccessPermissions, AttributeFields, MemAttributes}, + snafu::Snafu, +}; + +//================================================================================================= +// BootInfoMemRegion +//================================================================================================= + +/// Memory region. +#[derive(Default, Copy, Clone, Debug)] +pub struct BootInfoMemRegion { + /// Region start is inclusive. + pub start_inclusive: PhysAddr, + /// Region end is exclusive. + pub end_exclusive: PhysAddr, + pub attributes: AttributeFields, + pub name: &'static str, +} + +impl BootInfoMemRegion { + /// Create an empty region. + pub const fn new() -> Self { + Self { + start_inclusive: PhysAddr::zero(), + end_exclusive: PhysAddr::zero(), + attributes: AttributeFields::defaulted(), + name: "", + } + } + + /// Create an occupied or free region with start and end. + /// Region is in range [start, end), that is, for start 0x0 and end 0x2000 the region will + /// occupy memory between addresses 0x0 and 0x1fff. + pub fn at( + start_inclusive: PhysAddr, + end_exclusive: PhysAddr, + free: bool, + name: &'static str, + ) -> Self { + Self { + start_inclusive: start_inclusive.min(end_exclusive), + end_exclusive: end_exclusive.max(start_inclusive), + attributes: AttributeFields { + occupied: !free, + ..AttributeFields::default() + }, + name, + } + } + + /// Create a region with explicit attributes. + pub fn with_attributes( + start_inclusive: PhysAddr, + end_exclusive: PhysAddr, + attributes: AttributeFields, + name: &'static str, + ) -> Self { + Self { + start_inclusive: start_inclusive.min(end_exclusive), + end_exclusive: end_exclusive.max(start_inclusive), + attributes, + name, + } + } + + /// Calculate region size. + pub fn size(&self) -> usize { + self.end_exclusive - self.start_inclusive + } + + /// Is this region empty? + pub fn is_empty(&self) -> bool { + self.start_inclusive == self.end_exclusive + } + + /// Is this a free (unoccupied) region? + pub fn is_free(&self) -> bool { + !self.attributes.occupied && !self.is_empty() + } + + /// Is this a used (occupied) region? + pub fn is_used(&self) -> bool { + self.attributes.occupied && !self.is_empty() + } + + /// Clear the region to empty. + pub fn clear(&mut self) { + self.start_inclusive = PhysAddr::zero(); + self.end_exclusive = PhysAddr::zero(); + self.attributes = AttributeFields::defaulted(); + self.name = ""; + } + + /// Does this region intersect the given one? + /// Based on [Intersection of 1D segments](https://eli.thegreenplace.net/2008/08/15/intersection-of-1d-segments/). + /// + /// Since end is exclusive, the actual value is one less than what it contains, for this reason, + /// end equal to other's start means they touch but do not intersect. + /// + /// Assumes `start_inclusive` <= `end_exclusive`, which holds for memory regions by construction. + pub fn intersects(&self, other: &BootInfoMemRegion) -> bool { + self.end_exclusive > other.start_inclusive && other.end_exclusive > self.start_inclusive + } + + /// Does this region touch or overlap the given one? + /// Two regions touch when one's end equals the other's start. + /// This is used for merging adjacent free regions. + pub fn touches_or_overlaps(&self, other: &BootInfoMemRegion) -> bool { + self.end_exclusive >= other.start_inclusive && other.end_exclusive >= self.start_inclusive + } + + /// Check if two regions have compatible attributes for merging. + /// Compares `mem_attributes`, `acc_perms`, and executable — ignores the occupied flag. + pub fn compatible_attributes(&self, other: &BootInfoMemRegion) -> bool { + self.attributes.mem_attributes == other.attributes.mem_attributes + && self.attributes.acc_perms == other.attributes.acc_perms + && self.attributes.executable == other.attributes.executable + } +} + +impl fmt::Display for BootInfoMemRegion { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let (size, unit) = liblog::size_human_readable_ceil(self.size()); + + write!( + f, + "[{} - {}) | {: >3} {: <3} | {} | {}", + self.start_inclusive, self.end_exclusive, size, unit, self.attributes, self.name + ) + } +} + +// #[cfg(test)] +// mod boot_info_region_tests { +// use super::*; + +// #[test_case] +// fn test_construct_regular_region() { +// let region = BootInfoMemRegion::at(0x0.into(), 0x2000.into(), true, "RAM"); +// assert_eq!(region.start_inclusive, 0x0); +// assert_eq!(region.end_exclusive, 0x2000); +// assert_eq!(region.size(), 0x2000); +// assert_eq!(region.attributes.occupied, false); +// assert_eq!(region.name, "RAM"); +// } + +// #[test_case] +// fn test_construct_reverse_region() { +// let region = BootInfoMemRegion::at(0x2000.into(), 0x0.into(), true, "RAM"); +// assert_eq!(region.start_inclusive, 0x0); +// assert_eq!(region.end_exclusive, 0x2000); +// assert_eq!(region.size(), 0x2000); +// assert_eq!(region.attributes.occupied, false); +// assert_eq!(region.name, "RAM"); +// } + +// #[test_case] +// fn test_regions_touch() { +// let region1 = BootInfoMemRegion::at(0x0.into(), 0x2000.into(), false, "R1"); +// let region2 = BootInfoMemRegion::at(0x2000.into(), 0x4000.into(), false, "R2"); +// assert_eq!(region1.intersects(®ion2), false); +// assert_eq!(region2.intersects(®ion1), false); +// // But they do touch +// assert_eq!(region1.touches_or_overlaps(®ion2), true); +// assert_eq!(region2.touches_or_overlaps(®ion1), true); +// } + +// #[test_case] +// fn test_regions_intersect() { +// let region1 = BootInfoMemRegion::at(0x0.into(), 0x2000.into(), false, "R1"); +// let region2 = BootInfoMemRegion::at(0x1000.into(), 0x3000.into(), false, "R2"); +// assert_eq!(region1.intersects(®ion2), true); +// assert_eq!(region2.intersects(®ion1), true); +// } + +// #[test_case] +// fn test_self_intersect() { +// let region1 = BootInfoMemRegion::at(0x0.into(), 0x2000.into(), false, "R1"); +// let region2 = BootInfoMemRegion::at(0x0.into(), 0x2000.into(), false, "R2"); +// assert_eq!(region1.intersects(®ion2), true); +// assert_eq!(region2.intersects(®ion1), true); +// } + +// #[test_case] +// fn test_regions_fully_overlap() { +// let outer = BootInfoMemRegion::at(0x0.into(), 0x4000.into(), false, "outer"); +// let inner = BootInfoMemRegion::at(0x1000.into(), 0x3000.into(), false, "inner"); +// assert_eq!(outer.intersects(&inner), true); +// assert_eq!(inner.intersects(&outer), true); +// } + +// #[test_case] +// fn test_regions_disjoint() { +// let region1 = BootInfoMemRegion::at(0x0.into(), 0x1000.into(), false, "R1"); +// let region2 = BootInfoMemRegion::at(0x2000.into(), 0x3000.into(), false, "R2"); +// assert_eq!(region1.intersects(®ion2), false); +// assert_eq!(region2.intersects(®ion1), false); +// assert_eq!(region1.touches_or_overlaps(®ion2), false); +// assert_eq!(region2.touches_or_overlaps(®ion1), false); +// } + +// #[test_case] +// fn test_compatible_attributes() { +// let r1 = BootInfoMemRegion::at(0x0.into(), 0x1000.into(), true, "R1"); +// let r2 = BootInfoMemRegion::at(0x1000.into(), 0x2000.into(), true, "R2"); +// assert_eq!(r1.compatible_attributes(&r2), true); + +// let r3 = BootInfoMemRegion::with_attributes( +// 0x2000.into(), +// 0x3000.into(), +// AttributeFields { +// mem_attributes: MemAttributes::Device, +// ..AttributeFields::default() +// }, +// "MMIO", +// ); +// assert_eq!(r1.compatible_attributes(&r3), false); +// } + +// #[test_case] +// fn test_is_free_and_is_used() { +// let free = BootInfoMemRegion::at(0x0.into(), 0x1000.into(), true, "free"); +// assert_eq!(free.is_free(), true); +// assert_eq!(free.is_used(), false); + +// let used = BootInfoMemRegion::at(0x0.into(), 0x1000.into(), false, "used"); +// assert_eq!(used.is_free(), false); +// assert_eq!(used.is_used(), true); + +// let empty = BootInfoMemRegion::new(); +// assert_eq!(empty.is_free(), false); +// assert_eq!(empty.is_used(), false); +// } +// } + +//================================================================================================= +// BootInfo +//================================================================================================= + +const NUM_MEM_REGIONS: usize = 256; + +#[derive(Snafu, Debug, PartialEq)] +pub enum BootInfoError { + NoFreeSlots, + OverlappingUsedRegions, +} + +pub struct BootInfo { + pub regions: [BootInfoMemRegion; NUM_MEM_REGIONS], + pub num_regions: usize, +} + +/// Implement Default manually to work around Rust not defining Default for arrays over 32 items. +impl Default for BootInfo { + fn default() -> Self { + Self::new() + } +} + +impl BootInfo { + /// Create empty boot region map. + pub const fn new() -> BootInfo { + BootInfo { + regions: [BootInfoMemRegion::new(); NUM_MEM_REGIONS], + num_regions: 0, + } + } + + /// Low-level: find an empty slot and place a region there. + fn insert_raw(&mut self, region: BootInfoMemRegion) -> Result { + if region.is_empty() { + return Ok(0); + } + for (i, slot) in self.regions.iter_mut().enumerate() { + if slot.is_empty() { + *slot = region; + if i >= self.num_regions { + self.num_regions = i + 1; + } + return Ok(i); + } + } + Err(BootInfoError::NoFreeSlots) + } + + /// Insert a free memory region. + /// + /// The region is added and then merged with any adjacent or overlapping free regions + /// that have compatible attributes (same `mem_attributes`, `acc_perms`, executable). + pub fn insert_free_region( + &mut self, + start: PhysAddr, + end: PhysAddr, + attributes: AttributeFields, + name: &'static str, + ) -> Result<(), BootInfoError> { + let region = BootInfoMemRegion::with_attributes( + start, + end, + AttributeFields { + occupied: false, + ..attributes + }, + name, + ); + if region.is_empty() { + return Ok(()); + } + #[cfg(feature = "qemu")] + semi_println!("BOOT_INFO.insert_free_region: {}", region); + self.insert_raw(region)?; + self.merge_free_regions(); + Ok(()) + } + + /// Insert a used (occupied) memory region. + /// + /// The region is recorded and then cut out of any overlapping free regions. + /// Returns an error if the new used region overlaps an existing used region. + pub fn insert_used_region( + &mut self, + start: PhysAddr, + end: PhysAddr, + attributes: AttributeFields, + name: &'static str, + ) -> Result<(), BootInfoError> { + let region = BootInfoMemRegion::with_attributes( + start, + end, + AttributeFields { + occupied: true, + ..attributes + }, + name, + ); + if region.is_empty() { + return Ok(()); + } + #[cfg(feature = "qemu")] + semi_println!("BOOT_INFO.insert_used_region: {}", region); + + // Check for overlap with existing used regions. + for slot in &self.regions { + if slot.is_used() && slot.intersects(®ion) { + #[cfg(feature = "qemu")] + semi_println!( + "BOOT_INFO.insert_used_region: ERROR overlaps existing used region: {}", + slot + ); + return Err(BootInfoError::OverlappingUsedRegions); + } + } + + self.insert_raw(region)?; + self.cut_out_used_from_free(®ion)?; + Ok(()) + } + + /// Insert an overlay region that fills gaps between existing used regions. + /// + /// The overlay covers `[start, end)` but instead of failing on overlap with + /// existing used regions, it inserts new used regions only for the gaps. + /// This is useful for marking a large area (like the `Kickstart` bump allocator + /// arena) as used, where sub-regions have already been individually recorded. + pub fn insert_overlay_region( + &mut self, + start: PhysAddr, + end: PhysAddr, + attributes: AttributeFields, + name: &'static str, + ) -> Result<(), BootInfoError> { + let overlay_start = start.min(end); + let overlay_end = end.max(start); + + if overlay_start == overlay_end { + return Ok(()); + } + + #[cfg(feature = "qemu")] + semi_println!( + "BOOT_INFO.insert_overlay_region: [{} - {}) {}", + overlay_start, + overlay_end, + name + ); + + // Collect starts and ends of existing used regions that intersect the overlay. + // We only need the boundaries to compute gaps, so collect (start, end) pairs. + let mut boundaries: [(PhysAddr, PhysAddr); NUM_MEM_REGIONS] = + [(PhysAddr::zero(), PhysAddr::zero()); NUM_MEM_REGIONS]; + let mut count = 0; + + for slot in &self.regions { + if !slot.is_used() { + continue; + } + // Clip to overlay range. + let s = slot.start_inclusive.max(overlay_start); + let e = slot.end_exclusive.min(overlay_end); + if s < e { + boundaries[count] = (s, e); + count += 1; + } + } + + // Sort by start address. + boundaries[..count].sort_unstable_by_key(|&(s, _)| s); + + // Walk left-to-right, inserting gap regions. + let mut cursor = overlay_start; + + for &(used_start, used_end) in boundaries.iter().take(count) { + if cursor < used_start { + // Gap before this used region. + let gap = BootInfoMemRegion::with_attributes( + cursor, + used_start, + AttributeFields { + occupied: true, + ..attributes + }, + name, + ); + self.insert_raw(gap)?; + self.cut_out_used_from_free(&gap)?; + } + // Advance cursor past this used region. + if used_end > cursor { + cursor = used_end; + } + } + + // Trailing gap after the last used region. + if cursor < overlay_end { + let gap = BootInfoMemRegion::with_attributes( + cursor, + overlay_end, + AttributeFields { + occupied: true, + ..attributes + }, + name, + ); + self.insert_raw(gap)?; + self.cut_out_used_from_free(&gap)?; + } + + Ok(()) + } + + /// Merge all overlapping or adjacent free regions with compatible attributes. + /// Repeats until no more merges are possible. + fn merge_free_regions(&mut self) { + loop { + let mut merged = false; + // Find a pair of free regions to merge. + // We need indices so we can mutate the array. + 'outer: for i in 0..self.num_regions { + if !self.regions[i].is_free() { + continue; + } + for j in (i + 1)..self.num_regions { + if !self.regions[j].is_free() { + continue; + } + if self.regions[i].touches_or_overlaps(&self.regions[j]) + && self.regions[i].compatible_attributes(&self.regions[j]) + { + // Merge j into i. + let new_start = self.regions[i] + .start_inclusive + .min(self.regions[j].start_inclusive); + let new_end = self.regions[i] + .end_exclusive + .max(self.regions[j].end_exclusive); + self.regions[i].start_inclusive = new_start; + self.regions[i].end_exclusive = new_end; + // Keep name from the larger region (or the first one). + self.regions[j].clear(); + merged = true; + break 'outer; + } + } + } + if !merged { + break; + } + } + } + + /// Cut a used region out of all overlapping free regions. + fn cut_out_used_from_free(&mut self, used: &BootInfoMemRegion) -> Result<(), BootInfoError> { + // Collect regions to split (we can't mutate while iterating for splits that + // need to insert new regions). + // Process in a loop: each iteration handles one free region. + let mut idx = 0; + while idx < self.num_regions { + let free = &self.regions[idx]; + if !free.is_free() || !free.intersects(used) { + idx += 1; + continue; + } + + let free_start = self.regions[idx].start_inclusive; + let free_end = self.regions[idx].end_exclusive; + let free_attrs = self.regions[idx].attributes; + let free_name = self.regions[idx].name; + + // Case 1: used fully contains free → remove free entirely + if used.start_inclusive <= free_start && used.end_exclusive >= free_end { + self.regions[idx].clear(); + idx += 1; + continue; + } + + // Case 2: used overlaps start of free → shrink free + if used.start_inclusive <= free_start && used.end_exclusive < free_end { + self.regions[idx].start_inclusive = used.end_exclusive; + idx += 1; + continue; + } + + // Case 3: used overlaps end of free → shrink free + if used.start_inclusive > free_start && used.end_exclusive >= free_end { + self.regions[idx].end_exclusive = used.start_inclusive; + idx += 1; + continue; + } + + // Case 4: used is entirely inside free → split free into two + if used.start_inclusive > free_start && used.end_exclusive < free_end { + // Shrink current region to the left part. + self.regions[idx].end_exclusive = used.start_inclusive; + // Insert right part as a new free region. + let right = BootInfoMemRegion::with_attributes( + used.end_exclusive, + free_end, + free_attrs, + free_name, + ); + self.insert_raw(right)?; + idx += 1; + continue; + } + + idx += 1; + } + Ok(()) + } + + /// Allocate a region of given `size_bits` size. + /// + /// Search for a free mem region that will be the best fit for an allocation. We favour + /// allocations that are aligned to either end of the region. If an allocation must split + /// a region we favour an unbalanced split. In both cases we attempt to use the smallest + /// region possible. In general this means we aim to make the size of the smallest remaining + /// region smaller (ideally zero) followed by making the size of the largest remaining + /// region smaller. + pub fn alloc_region( + &mut self, + size_bits: usize, + name: &'static str, + ) -> Result { + let mut reg_index: usize = 0; + let mut reg = BootInfoMemRegion::new(); + let mut rem_small = BootInfoMemRegion::new(); + let mut rem_large = BootInfoMemRegion::new(); + + // Iterate only free regions. + for (i, reg_iter) in self + .regions + .iter() + .enumerate() + .filter(|(_, reg)| reg.is_free()) + { + // Determine whether placing the region at the start or the end + // will create a bigger left over region. + let aligned_start = reg_iter.start_inclusive.aligned_up(1_u64 << size_bits); + let aligned_end = reg_iter.end_exclusive.aligned_down(1_u64 << size_bits); + let new_reg = if aligned_start - reg_iter.start_inclusive + < reg_iter.end_exclusive - aligned_end + { + BootInfoMemRegion::at( + aligned_start, + aligned_start + (1_u64 << size_bits), + false, + name, + ) + } else { + BootInfoMemRegion::at(aligned_end - (1_u64 << size_bits), aligned_end, false, name) + }; + + if new_reg.start_inclusive >= reg_iter.start_inclusive + && new_reg.end_exclusive <= reg_iter.end_exclusive + { + let mut new_rem_small = BootInfoMemRegion::new(); + let mut new_rem_large = BootInfoMemRegion::new(); + + if new_reg.start_inclusive - reg_iter.start_inclusive + < reg_iter.end_exclusive - new_reg.end_exclusive + { + new_rem_small.start_inclusive = reg_iter.start_inclusive; + new_rem_small.end_exclusive = new_reg.start_inclusive; + new_rem_large.start_inclusive = new_reg.end_exclusive; + new_rem_large.end_exclusive = reg_iter.end_exclusive; + } else { + new_rem_large.start_inclusive = reg_iter.start_inclusive; + new_rem_large.end_exclusive = new_reg.start_inclusive; + new_rem_small.start_inclusive = new_reg.end_exclusive; + new_rem_small.end_exclusive = reg_iter.end_exclusive; + } + // Find better fit. + if reg.is_empty() + || (new_rem_small.size() < rem_small.size()) + || (new_rem_small.size() == rem_small.size() + && new_rem_large.size() < rem_large.size()) + { + reg = new_reg; + rem_small = new_rem_small; + rem_large = new_rem_large; + reg_index = i; + } + } + } + + if reg.is_empty() { + return Err(BootInfoError::NoFreeSlots); + } + + // Remove the region in question. + self.regions[reg_index].clear(); + + // Add the remaining regions in largest to smallest order. + self.insert_raw(rem_large)?; + if self.insert_raw(rem_small).is_err() { + #[cfg(feature = "qemu")] + semi_println!( + "BootInfo::alloc_region(): wasted {} bytes due to alignment, try to increase NUM_MEM_REGIONS", + rem_small.size() + ); + } + Ok(reg.start_inclusive) + } + + /// Sort regions by start address. Empty regions sort to the end. + pub fn sort(&mut self) { + self.regions + .sort_unstable_by(|a, b| match (a.is_empty(), b.is_empty()) { + (true, true) => core::cmp::Ordering::Equal, + (true, false) => core::cmp::Ordering::Greater, + (false, true) => core::cmp::Ordering::Less, + (false, false) => a.start_inclusive.cmp(&b.start_inclusive), + }); + // Update num_regions after sort. + self.num_regions = self + .regions + .iter() + .rposition(|r| !r.is_empty()) + .map_or(0, |p| p + 1); + } + + /// Remove empty gaps in the region array by shifting entries down. + pub fn compact(&mut self) { + let mut write = 0; + for read in 0..self.num_regions { + if !self.regions[read].is_empty() { + if write != read { + self.regions[write] = self.regions[read]; + self.regions[read].clear(); + } + write += 1; + } + } + self.num_regions = write; + } + + /// Print all non-empty regions for debug purposes. + pub fn dump(&self) { + #[cfg(feature = "qemu")] + semi_println!("BOOT_INFO: {} region(s):", self.count()); + for region in &self.regions { + if !region.is_empty() { + #[cfg(feature = "qemu")] + semi_println!(" {}", region); + } + } + } + + /// Count non-empty regions. + pub fn count(&self) -> usize { + self.regions.iter().filter(|r| !r.is_empty()).count() + } + + /// Iterate over all non-empty regions. + pub fn iter(&self) -> impl Iterator { + self.regions.iter().filter(|r| !r.is_empty()) + } + + /// Iterate over free regions. + pub fn free_regions(&self) -> impl Iterator { + self.regions.iter().filter(|r| r.is_free()) + } + + /// Iterate over used regions. + pub fn used_regions(&self) -> impl Iterator { + self.regions.iter().filter(|r| r.is_used()) + } + + /// Total free memory in bytes. + pub fn total_free(&self) -> usize { + self.free_regions().map(BootInfoMemRegion::size).sum() + } + + /// Total used memory in bytes. + pub fn total_used(&self) -> usize { + self.used_regions().map(BootInfoMemRegion::size).sum() + } +} + +// Should go to BSS +pub static BOOT_INFO: IRQSafeNullLock> = + IRQSafeNullLock::new(LazyCell::new(BootInfo::new)); + +// #[cfg(test)] +// mod boot_info_tests { +// use super::*; + +// fn default_attrs() -> AttributeFields { +// AttributeFields::default() +// } + +// fn device_attrs() -> AttributeFields { +// AttributeFields { +// mem_attributes: MemAttributes::Device, +// acc_perms: AccessPermissions::ReadWrite, +// executable: false, +// occupied: false, +// } +// } + +// // -- insert_free_region tests -- + +// #[test_case] +// fn test_insert_free_region() { +// let mut bi = BootInfo::new(); +// let res = bi.insert_free_region(0x0.into(), 0x4000.into(), default_attrs(), "RAM"); +// assert!(res.is_ok()); +// assert_eq!(bi.count(), 1); +// assert_eq!(bi.total_free(), 0x4000); +// } + +// #[test_case] +// fn test_insert_free_region_empty_is_noop() { +// let mut bi = BootInfo::new(); +// let res = bi.insert_free_region(0x1000.into(), 0x1000.into(), default_attrs(), "empty"); +// assert!(res.is_ok()); +// assert_eq!(bi.count(), 0); +// } + +// #[test_case] +// fn test_insert_free_merge_adjacent() { +// let mut bi = BootInfo::new(); +// bi.insert_free_region(0x0.into(), 0x2000.into(), default_attrs(), "RAM") +// .unwrap(); +// bi.insert_free_region(0x2000.into(), 0x4000.into(), default_attrs(), "RAM") +// .unwrap(); +// // Should merge into one region [0x0, 0x4000). +// assert_eq!(bi.count(), 1); +// assert_eq!(bi.total_free(), 0x4000); +// let r = bi.free_regions().next().unwrap(); +// assert_eq!(r.start_inclusive, 0x0); +// assert_eq!(r.end_exclusive, 0x4000); +// } + +// #[test_case] +// fn test_insert_free_merge_overlapping() { +// let mut bi = BootInfo::new(); +// bi.insert_free_region(0x0.into(), 0x3000.into(), default_attrs(), "RAM") +// .unwrap(); +// bi.insert_free_region(0x2000.into(), 0x5000.into(), default_attrs(), "RAM") +// .unwrap(); +// assert_eq!(bi.count(), 1); +// assert_eq!(bi.total_free(), 0x5000); +// let r = bi.free_regions().next().unwrap(); +// assert_eq!(r.start_inclusive, 0x0); +// assert_eq!(r.end_exclusive, 0x5000); +// } + +// #[test_case] +// fn test_insert_free_no_merge_different_attrs() { +// let mut bi = BootInfo::new(); +// bi.insert_free_region(0x0.into(), 0x2000.into(), default_attrs(), "RAM") +// .unwrap(); +// bi.insert_free_region(0x2000.into(), 0x4000.into(), device_attrs(), "MMIO") +// .unwrap(); +// // Different attributes: should NOT merge. +// assert_eq!(bi.count(), 2); +// } + +// #[test_case] +// fn test_insert_free_merge_three_regions() { +// let mut bi = BootInfo::new(); +// bi.insert_free_region(0x0.into(), 0x1000.into(), default_attrs(), "R1") +// .unwrap(); +// bi.insert_free_region(0x2000.into(), 0x3000.into(), default_attrs(), "R3") +// .unwrap(); +// // Insert a bridging region that touches both. +// bi.insert_free_region(0x1000.into(), 0x2000.into(), default_attrs(), "R2") +// .unwrap(); +// assert_eq!(bi.count(), 1); +// assert_eq!(bi.total_free(), 0x3000); +// } + +// // -- insert_used_region tests -- + +// #[test_case] +// fn test_insert_used_cuts_free_start() { +// let mut bi = BootInfo::new(); +// bi.insert_free_region(0x0.into(), 0x4000.into(), default_attrs(), "RAM") +// .unwrap(); +// bi.insert_used_region(0x0.into(), 0x1000.into(), default_attrs(), "kernel") +// .unwrap(); +// // Free region should be shrunk to [0x1000, 0x4000). +// assert_eq!(bi.total_free(), 0x3000); +// assert_eq!(bi.total_used(), 0x1000); +// let free = bi.free_regions().next().unwrap(); +// assert_eq!(free.start_inclusive, 0x1000); +// assert_eq!(free.end_exclusive, 0x4000); +// } + +// #[test_case] +// fn test_insert_used_cuts_free_end() { +// let mut bi = BootInfo::new(); +// bi.insert_free_region(0x0.into(), 0x4000.into(), default_attrs(), "RAM") +// .unwrap(); +// bi.insert_used_region(0x3000.into(), 0x4000.into(), default_attrs(), "kernel") +// .unwrap(); +// assert_eq!(bi.total_free(), 0x3000); +// let free = bi.free_regions().next().unwrap(); +// assert_eq!(free.start_inclusive, 0x0); +// assert_eq!(free.end_exclusive, 0x3000); +// } + +// #[test_case] +// fn test_insert_used_splits_free() { +// let mut bi = BootInfo::new(); +// bi.insert_free_region(0x0.into(), 0x8000.into(), default_attrs(), "RAM") +// .unwrap(); +// bi.insert_used_region(0x2000.into(), 0x4000.into(), default_attrs(), "kernel") +// .unwrap(); +// // Should split into [0x0, 0x2000) and [0x4000, 0x8000). +// let free: usize = bi.free_regions().map(|r| r.size()).sum(); +// assert_eq!(free, 0x6000); +// assert_eq!(bi.free_regions().count(), 2); +// assert_eq!(bi.used_regions().count(), 1); +// } + +// #[test_case] +// fn test_insert_used_subsumes_free() { +// let mut bi = BootInfo::new(); +// bi.insert_free_region(0x1000.into(), 0x3000.into(), default_attrs(), "RAM") +// .unwrap(); +// bi.insert_used_region(0x0.into(), 0x4000.into(), default_attrs(), "kernel") +// .unwrap(); +// assert_eq!(bi.total_free(), 0); +// assert_eq!(bi.total_used(), 0x4000); +// } + +// #[test_case] +// fn test_insert_used_overlapping_error() { +// let mut bi = BootInfo::new(); +// bi.insert_used_region(0x0.into(), 0x2000.into(), default_attrs(), "kernel") +// .unwrap(); +// let res = bi.insert_used_region(0x1000.into(), 0x3000.into(), default_attrs(), "dtb"); +// assert_eq!(res, Err(BootInfoError::OverlappingUsedRegions)); +// } + +// #[test_case] +// fn test_insert_used_no_overlap_ok() { +// let mut bi = BootInfo::new(); +// bi.insert_used_region(0x0.into(), 0x2000.into(), default_attrs(), "kernel") +// .unwrap(); +// bi.insert_used_region(0x2000.into(), 0x4000.into(), default_attrs(), "dtb") +// .unwrap(); +// assert_eq!(bi.used_regions().count(), 2); +// } + +// #[test_case] +// fn test_insert_used_cuts_multiple_free_regions() { +// let mut bi = BootInfo::new(); +// // Two separate free regions. +// bi.insert_free_region(0x0.into(), 0x2000.into(), default_attrs(), "R1") +// .unwrap(); +// bi.insert_free_region(0x3000.into(), 0x5000.into(), default_attrs(), "R2") +// .unwrap(); +// // Used region spans across both. +// bi.insert_used_region(0x1000.into(), 0x4000.into(), default_attrs(), "kernel") +// .unwrap(); +// // R1 shrunk to [0x0, 0x1000), R2 shrunk to [0x4000, 0x5000). +// assert_eq!(bi.total_free(), 0x2000); +// assert_eq!(bi.total_used(), 0x3000); +// } + +// // -- sort and compact tests -- + +// #[test_case] +// fn test_sort() { +// let mut bi = BootInfo::new(); +// bi.insert_free_region(0x4000.into(), 0x8000.into(), default_attrs(), "R2") +// .unwrap(); +// bi.insert_free_region(0x0.into(), 0x2000.into(), default_attrs(), "R1") +// .unwrap(); +// bi.sort(); +// let regions: Vec<_> = bi.iter().collect(); +// assert!(regions[0].start_inclusive < regions[1].start_inclusive); +// } + +// #[test_case] +// fn test_compact() { +// let mut bi = BootInfo::new(); +// bi.insert_free_region(0x0.into(), 0x2000.into(), default_attrs(), "R1") +// .unwrap(); +// bi.insert_free_region(0x4000.into(), 0x6000.into(), default_attrs(), "R2") +// .unwrap(); +// // Manually clear first to create a gap. +// bi.regions[0].clear(); +// assert_eq!(bi.count(), 1); +// bi.compact(); +// // After compact, the remaining region should be in slot 0. +// assert!(!bi.regions[0].is_empty()); +// assert_eq!(bi.regions[0].start_inclusive, 0x4000); +// } + +// // -- alloc_region tests -- + +// #[test_case] +// fn test_alloc_region_no_memory() { +// let mut bi = BootInfo::new(); +// let res = bi.alloc_region(12, "test"); +// assert_eq!(res, Err(BootInfoError::NoFreeSlots)); +// } + +// #[test_case] +// fn test_alloc_region_basic() { +// let mut bi = BootInfo::new(); +// bi.insert_free_region(0x0.into(), 0x10_0000.into(), default_attrs(), "RAM") +// .unwrap(); +// let addr = bi.alloc_region(12, "page").unwrap(); // 4 KiB +// // The allocated address should be within the original free region. +// assert!(addr >= PhysAddr::from(0x0u64)); +// assert!(addr < PhysAddr::from(0x10_0000u64)); +// // Free space should be reduced by 4 KiB. +// assert_eq!(bi.total_free(), 0x10_0000 - 0x1000); +// } + +// // -- insert_overlay_region tests -- + +// #[test_case] +// fn test_overlay_fills_gaps_between_used() { +// let mut bi = BootInfo::new(); +// bi.insert_free_region(0x0.into(), 0xA000.into(), default_attrs(), "RAM") +// .unwrap(); +// bi.insert_used_region(0x1000.into(), 0x3000.into(), default_attrs(), "kernel") +// .unwrap(); +// bi.insert_used_region(0x5000.into(), 0x7000.into(), default_attrs(), "stack") +// .unwrap(); +// // Overlay [0x0, 0xA000) should fill three gaps. +// bi.insert_overlay_region(0x0.into(), 0xA000.into(), default_attrs(), "init") +// .unwrap(); + +// bi.compact(); +// bi.sort(); + +// // Should have: [0,1000) init, [1000,3000) kernel, [3000,5000) init, +// // [5000,7000) stack, [7000,A000) init = 5 used regions +// assert_eq!(bi.used_regions().count(), 5); +// assert_eq!(bi.total_used(), 0xA000); +// assert_eq!(bi.total_free(), 0); +// } + +// #[test_case] +// fn test_overlay_no_existing_used() { +// let mut bi = BootInfo::new(); +// bi.insert_free_region(0x0.into(), 0x4000.into(), default_attrs(), "RAM") +// .unwrap(); +// // Overlay with no existing used regions — becomes one big used region. +// bi.insert_overlay_region(0x0.into(), 0x4000.into(), default_attrs(), "init") +// .unwrap(); +// assert_eq!(bi.used_regions().count(), 1); +// assert_eq!(bi.total_used(), 0x4000); +// assert_eq!(bi.total_free(), 0); +// } + +// #[test_case] +// fn test_overlay_fully_covered() { +// let mut bi = BootInfo::new(); +// // Used region covers entire overlay — no gaps to fill. +// bi.insert_used_region(0x0.into(), 0x4000.into(), default_attrs(), "kernel") +// .unwrap(); +// bi.insert_overlay_region(0x0.into(), 0x4000.into(), default_attrs(), "init") +// .unwrap(); +// assert_eq!(bi.used_regions().count(), 1); +// assert_eq!(bi.total_used(), 0x4000); +// } + +// #[test_case] +// fn test_overlay_empty_is_noop() { +// let mut bi = BootInfo::new(); +// bi.insert_overlay_region(0x1000.into(), 0x1000.into(), default_attrs(), "init") +// .unwrap(); +// assert_eq!(bi.count(), 0); +// } + +// #[test_case] +// fn test_overlay_cuts_from_free() { +// let mut bi = BootInfo::new(); +// bi.insert_free_region(0x0.into(), 0x8000.into(), default_attrs(), "RAM") +// .unwrap(); +// bi.insert_used_region(0x2000.into(), 0x4000.into(), default_attrs(), "kernel") +// .unwrap(); +// // Overlay [0x1000, 0x5000) — gaps: [0x1000,0x2000) and [0x4000,0x5000) +// bi.insert_overlay_region(0x1000.into(), 0x5000.into(), default_attrs(), "init") +// .unwrap(); +// // Used: kernel [2000,4000) + init [1000,2000) + init [4000,5000) = 0x4000 +// assert_eq!(bi.total_used(), 0x4000); +// // Free: [0x0,0x1000) + [0x5000,0x8000) = 0x4000 +// assert_eq!(bi.total_free(), 0x4000); +// } +// } diff --git a/kernel/kickstart/src/device_tree.rs b/kernel/kickstart/src/device_tree.rs new file mode 100644 index 000000000..77b25a1aa --- /dev/null +++ b/kernel/kickstart/src/device_tree.rs @@ -0,0 +1,557 @@ +#![allow(dead_code)] + +use { + core::{alloc::Layout, ptr::read_unaligned}, + fdt_rs::{ + base::{DevTree, iters::StringPropIter}, + error::{DevTreeError, Result as DevTreeResult}, + index::{DevTreeIndex, DevTreeIndexNode, DevTreeIndexProp}, + prelude::{FallibleIterator, PropReader}, + }, + shrinkwraprs::Shrinkwrap, +}; + +fn get_size_cell_tree_value<'a, 'i: 'a, 'dt: 'i>( + node: &DevTreeIndexNode<'a, 'i, 'dt>, + name: &str, +) -> u32 { + const DEFAULT: u32 = 1; + + let res: Result<_, DevTreeError> = node.props().try_find(|prop| Ok(prop.name()? == name)); + + if res.is_ok() + && let Some(res) = res.unwrap() + { + return res.u32(0).unwrap_or(DEFAULT); + } + + while let Some(node) = node.parent() { + let res: Result<_, DevTreeError> = node.props().try_find(|prop| Ok(prop.name()? == name)); + + if res.is_err() { + // @todo abort on error? because it's not a None, but an actual read error.. + continue; + } + + if let Some(res) = res.unwrap() { + return res.u32(0).unwrap_or(DEFAULT); + } + } + + DEFAULT +} + +pub fn get_address_cells<'a, 'i: 'a, 'dt: 'i>(node: &DevTreeIndexNode<'a, 'i, 'dt>) -> u32 { + get_size_cell_tree_value(node, "#address-cells") +} + +pub fn get_size_cells<'a, 'i: 'a, 'dt: 'i>(node: &DevTreeIndexNode<'a, 'i, 'dt>) -> u32 { + get_size_cell_tree_value(node, "#size-cells") +} + +/// Uses `DevTreeIndex` implementation for simpler navigation. +/// This requires allocation of a single buffer, which is done at boot time via bump allocator. +/// This means we can only parse the tree after bump allocator is initialized. +#[derive(Shrinkwrap)] +pub struct DeviceTree<'a>(DevTreeIndex<'a, 'a>); + +impl<'a> DeviceTree<'a> { + pub fn dumper(&'a self, indent: usize) -> FdtDumper<'a> { + FdtDumper { + index: &self.0, + indent, + } + } + + pub fn layout(tree: DevTree<'a>) -> Result { + DevTreeIndex::get_layout(&tree) + } + + pub fn new(tree: DevTree<'a>, raw_slice: &'a mut [u8]) -> Result { + Ok(Self(DevTreeIndex::new(tree, raw_slice)?)) + } + + // @todo drop all the wrapper shenanigans and just export this one fn + /// Iterate path separated by / starting from the root "/" and find props one by one. + pub fn get_prop_by_path( + &self, + path: &str, + ) -> Result, DevTreeError> { + let mut path = PathSplit::new(path); + let mut node_iter = self.0.root().children(); + let mut node: Option = Some(self.0.root()); + if path.component().is_empty() { + // Root "/" + path.move_next(); + } + while !path.is_finished() { + let res: Result<_, DevTreeError> = + node_iter.try_find(|node| Ok(node.name()? == path.component())); + node = res?; + if node.is_none() { + return Err(DevTreeError::InvalidParameter("Invalid path")); // @todo + } + node_iter = node.as_ref().unwrap().children(); + path.move_next(); + } + assert!(path.is_finished()); // tbd + assert!(node.is_some()); + let mut prop_iter = node.unwrap().props(); + let res: Result<_, DevTreeError> = + prop_iter.try_find(|prop| Ok(prop.name()? == path.component())); + let prop = res?; + if prop.is_none() { + return Err(DevTreeError::InvalidParameter("Invalid path")); // @todo + } + Ok(prop.unwrap()) + } +} + +/// Augment `DevTreeIndexProp` with a set of pairs accessor. +#[derive(Shrinkwrap)] +pub struct DeviceTreeProp<'a, 'i: 'a, 'dt: 'i>(DevTreeIndexProp<'a, 'i, 'dt>); + +impl<'a, 'i: 'a, 'dt: 'i> DeviceTreeProp<'a, 'i, 'dt> { + pub fn new(source: DevTreeIndexProp<'a, 'i, 'dt>) -> Self { + Self(source) + } + + pub fn payload_pairs_iter(&'a self) -> PayloadPairsIter<'a, 'i, 'dt> { + let address_cells = get_address_cells(&self.node()); + let size_cells = get_size_cells(&self.node()); + + // @todo boot this on 8Gb RasPi, because I'm not sure how it allocates memory regions there. + // libqemu::semi_println!( + // "Address cells: {}, size cells {}", + // address_cells, + // size_cells + // ); + + PayloadPairsIter::new(&self.0, address_cells, size_cells) + } +} + +pub struct PayloadPairsIter<'a, 'i: 'a, 'dt: 'i> { + prop: &'a DevTreeIndexProp<'a, 'i, 'dt>, + total: usize, + offset: usize, + address_cells: u32, + size_cells: u32, +} + +impl<'a, 'i: 'a, 'dt: 'i> PayloadPairsIter<'a, 'i, 'dt> { + pub fn new( + prop: &'a DevTreeIndexProp<'a, 'i, 'dt>, + address_cells: u32, + size_cells: u32, + ) -> Self { + Self { + prop, + total: prop.length(), + offset: 0_usize, + address_cells, + size_cells, + } + } + + // @todo get rid of unwrap()s here + fn prop_u32(&mut self, index: usize) -> u64 { + self.prop.u32(index).unwrap().into() + } + + // @todo get rid of unwrap()s here + fn prop_u64(&mut self, index: usize) -> u64 { + u64::from(self.prop.u32(index).unwrap()) << 32 + | u64::from(self.prop.u32(index + 1).unwrap()) + } + + fn read_pair( + &mut self, + size: usize, + read_first: impl Fn(&mut Self, usize) -> u64, + first_index: usize, + read_second: impl Fn(&mut Self, usize) -> u64, + second_index: usize, + ) -> Option<(u64, u64)> { + if self.offset + size > self.total { + return None; + } + let result: (u64, u64) = ( + read_first(self, first_index), + read_second(self, second_index), + ); + self.offset += size; + Some(result) + } +} + +impl<'a, 'i: 'a, 'dt: 'i> Iterator for PayloadPairsIter<'a, 'i, 'dt> { + /// Return a pair of (address, size) values on each iteration. + type Item = (u64, u64); + + fn next(&mut self) -> Option { + const STEP: usize = size_of::(); + // libqemu::semi_println!("Offset {}, total {}", self.offset, self.total); + if self.offset >= self.total { + // @todo check for sufficient space for the following read or the reads below may fail! + return None; + } + match (self.address_cells, self.size_cells) { + (1, 1) => { + const SIZE: usize = 8; + self.read_pair( + SIZE, + Self::prop_u32, + self.offset / STEP, + Self::prop_u32, + self.offset / STEP + 1, + ) + } + (1, 2) => { + const SIZE: usize = 12; + self.read_pair( + SIZE, + Self::prop_u32, + self.offset / STEP, + Self::prop_u64, + self.offset / STEP + 1, + ) + } + (2, 1) => { + const SIZE: usize = 12; + self.read_pair( + SIZE, + Self::prop_u64, + self.offset / STEP, + Self::prop_u32, + self.offset / STEP + 2, + ) + } + (2, 2) => { + const SIZE: usize = 16; + self.read_pair( + SIZE, + Self::prop_u64, + self.offset / STEP, + Self::prop_u64, + self.offset / STEP + 2, + ) + } + (1, 0) => { + const SIZE: usize = 4; + self.prop + .u32(self.offset / STEP) + .map(|first| { + self.offset += SIZE; // emulate read_pair() + (u64::from(first), 0) + }) + .ok() + } + (2, 0) => { + const SIZE: usize = 8; + self.prop + .u64(self.offset / STEP) + .map(|first| { + self.offset += SIZE; // emulate read_pair() + (first, 0) + }) + .ok() + } + _ => panic!("oooops, (0,x) cells are not supported"), + } + } +} + +// #[cfg(test)] +// mod tests { +// use super::PayloadPairsIter; + +// const BUF: [u32; 4] = [0x0000_0000, 0x2000_0000, 0x4000_0000, 0x8000_0000]; + +// #[test_case] +// fn parse_1_1_prop_correctly() { +// PayloadPairsIter +// } + +// #[test_case] +// fn parse_1_2_prop_correctly() { +// PayloadPairsIter +// } + +// #[test_case] +// fn parse_2_1_prop_correctly() { +// PayloadPairsIter +// } + +// #[test_case] +// fn parse_2_2_prop_correctly() { +// PayloadPairsIter +// } +// } + +// See "2.2.3 Path Names" in DTSpec v0.3 +// This is based on https://lib.rs/dtb implementation (c) Simon Prykhodko, MIT license. +struct PathSplit<'a> { + path: &'a str, + path_component: &'a str, + index: usize, + total: usize, +} + +impl<'a> PathSplit<'a> { + pub fn new(path: &'a str) -> PathSplit<'a> { + let path = if let Some(p) = path.strip_suffix('/') { + p + } else { + path + }; + let mut split = PathSplit { + path, + path_component: "", + index: 0, + total: path.split('/').count(), + }; + split.update(); + split + } + + fn update(&mut self) { + for (i, comp) in self.path.split('/').enumerate() { + if i == self.index { + self.path_component = comp; + return; + } + } + } + + pub fn component(&self) -> &'a str { + self.path_component + } + + pub fn level(&self) -> usize { + self.index + } + + pub fn is_finished(&self) -> bool { + self.index >= self.total - 1 + } + + pub fn move_prev(&mut self) -> bool { + if self.index > 0 { + self.index -= 1; + self.update(); + return true; + } + false + } + + pub fn move_next(&mut self) -> bool { + if self.index < self.total - 1 { + self.index += 1; + self.update(); + return true; + } + false + } +} + +// #[cfg(test)] +// mod tests { +// use super::PathSplit; + +// #[test_case] +// fn test_single_level_path_split() { +// let mut path = PathSplit::new("/#address-cells"); +// assert!(!path.is_finished()); +// assert_eq!(path.level(), 0); +// assert_eq!(path.component(), ""); + +// assert_eq!(path.move_next(), true); + +// assert!(path.is_finished()); +// assert_eq!(path.level(), 1); +// assert_eq!(path.component(), "#address-cells"); + +// assert_eq!(path.move_next(), false); +// } + +// #[test_case] +// fn test_multiple_level_path_split() { +// let mut path = PathSplit::new("/some/_other/#address-cells"); +// assert!(!path.is_finished()); +// assert_eq!(path.level(), 0); +// assert_eq!(path.component(), ""); + +// assert_eq!(path.move_next(), true); + +// assert!(!path.is_finished()); +// assert_eq!(path.level(), 1); +// assert_eq!(path.component(), "some"); + +// assert_eq!(path.move_next(), true); + +// assert!(!path.is_finished()); +// assert_eq!(path.level(), 2); +// assert_eq!(path.component(), "_other"); + +// assert_eq!(path.move_next(), true); + +// assert!(path.is_finished()); +// assert_eq!(path.level(), 3); +// assert_eq!(path.component(), "#address-cells"); + +// assert_eq!(path.move_next(), false); +// } +// } + +//================================================================================================= +// Dump the entire FDT +// From https://github.com/rs-embedded/fdtdump/blob/master/src/main.rs +//================================================================================================= + +fn are_printable_strings(mut prop_iter: StringPropIter) -> bool { + loop { + match prop_iter.next() { + Ok(Some(s_ref)) => { + if s_ref.is_empty() { + return false; + } + } + Ok(None) => return true, + Err(_) => return false, + } + } +} + +pub struct FdtDumper<'a> { + index: &'a DevTreeIndex<'a, 'a>, + indent: usize, +} + +impl FdtDumper<'_> { + fn push_indent(&mut self) { + for _ in 0..self.indent { + #[cfg(feature = "qemu")] + libqemu::semi_print!(" "); + } + } + + fn dump_node_name(&mut self, name: &str) { + self.push_indent(); + #[cfg(feature = "qemu")] + libqemu::semi_println!("{name} {{"); + } + + fn dump_node(&mut self, node: &DevTreeIndexNode) -> DevTreeResult<()> { + let mut name = node.name()?; + if name.is_empty() { + name = "/"; + } else { + name = node.name()?; + } + self.dump_node_name(name); + Ok(()) + } + + fn dump_property(&mut self, prop: &DevTreeIndexProp) -> DevTreeResult<()> { + self.push_indent(); + + #[cfg(feature = "qemu")] + libqemu::semi_print!("{}", prop.name()?); + + if prop.length() == 0 { + #[cfg(feature = "qemu")] + libqemu::semi_println!(";"); + return Ok(()); + } + #[cfg(feature = "qemu")] + libqemu::semi_print!(" = "); + + // SAFETY: Unsafe Ok - we're reinterpreting the data as expected. + unsafe { + // First try to parse as an array of strings + if are_printable_strings(prop.iter_str()) { + let mut iter = prop.iter_str(); + while let Some(s) = iter.next()? { + #[cfg(feature = "qemu")] + libqemu::semi_print!("\"{}\", ", s); + } + // let _ = self.dump.pop(); + // let _ = self.dump.pop(); + } else if prop.propbuf().len() % size_of::() == 0 { + #[cfg(feature = "qemu")] + libqemu::semi_print!("<"); + for val in prop.propbuf().chunks_exact(size_of::()) { + // We use read_unaligned + let v = u32::from_be(read_unaligned::(val.as_ptr().cast::())); + #[cfg(feature = "qemu")] + libqemu::semi_print!("{:#010x} ", v); + } + // let _ = self.dump.pop(); // Pop off extra space + #[cfg(feature = "qemu")] + libqemu::semi_print!(">"); + } else { + #[cfg(feature = "qemu")] + libqemu::semi_print!("["); + for val in prop.propbuf() { + #[cfg(feature = "qemu")] + libqemu::semi_print!("{:02x} ", val); + } + // let _ = self.dump.pop(); // Pop off extra space + #[cfg(feature = "qemu")] + libqemu::semi_print!("]"); + } + } + + #[cfg(feature = "qemu")] + libqemu::semi_println!(";"); + Ok(()) + } + + pub fn dump_level(&mut self, node: &DevTreeIndexNode) -> DevTreeResult<()> { + self.dump_node(node)?; + self.indent += 1; + for prop in node.props() { + self.dump_property(&prop)?; + } + for child in node.children() { + self.dump_level(&child)?; + } + self.indent -= 1; + self.push_indent(); + #[cfg(feature = "qemu")] + libqemu::semi_println!("}};"); + Ok(()) + } + + pub fn dump_root(&mut self) -> DevTreeResult<()> { + self.dump_level(&self.index.root()) + } + + pub fn dump_metadata(&mut self) { + let fdt = self.index.fdt(); + #[cfg(feature = "qemu")] + libqemu::semi_println!("// magic:\t\t{:#x}", fdt.magic()); + let s = fdt.totalsize(); + #[cfg(feature = "qemu")] + libqemu::semi_println!("// totalsize:\t\t{:#x} ({})", s, s); + #[cfg(feature = "qemu")] + libqemu::semi_println!("// off_dt_struct:\t{:#x}", fdt.off_dt_struct()); + #[cfg(feature = "qemu")] + libqemu::semi_println!("// off_dt_strings:\t{:#x}", fdt.off_dt_strings()); + #[cfg(feature = "qemu")] + libqemu::semi_println!("// off_mem_rsvmap:\t{:#x}", fdt.off_mem_rsvmap()); + #[cfg(feature = "qemu")] + libqemu::semi_println!("// version:\t\t{:}", fdt.version()); + #[cfg(feature = "qemu")] + libqemu::semi_println!("// last_comp_version:\t{:}", fdt.last_comp_version()); + #[cfg(feature = "qemu")] + libqemu::semi_println!("// boot_cpuid_phys:\t{:#x}", fdt.boot_cpuid_phys()); + #[cfg(feature = "qemu")] + libqemu::semi_println!("// size_dt_strings:\t{:#x}", fdt.size_dt_strings()); + #[cfg(feature = "qemu")] + libqemu::semi_println!("// size_dt_struct:\t{:#x}", fdt.size_dt_struct()); + #[cfg(feature = "qemu")] + libqemu::semi_println!(); + } +} diff --git a/kernel/kickstart/src/el_switch.rs b/kernel/kickstart/src/el_switch.rs new file mode 100644 index 000000000..401ba02bd --- /dev/null +++ b/kernel/kickstart/src/el_switch.rs @@ -0,0 +1,170 @@ +// kickstart/src/el_switch.rs - EL2 to EL1 transition with VBAR setup + +use { + crate::loader::memory_barrier, + aarch64_cpu::{ + asm::{self, barrier}, + registers::{ + ELR_EL2, HCR_EL2, MAIR_EL1, ReadWriteable, SCTLR_EL1, SP_EL1, SPSR_EL1, SPSR_EL2, + TCR_EL1, TTBR0_EL1, TTBR1_EL1, VBAR_EL1, Writeable, + }, + }, +}; + +/// Configure and enable the MMU, set `VBAR_EL1`, then drop to EL1 +/// +/// # Arguments +/// +/// * `ttbr0` - Translation table base for low addresses (identity map) +/// * `ttbr1` - Translation table base for high addresses (kernel) +/// * `vbar` - Exception vector base address (physical, must be 2KB aligned) +/// * `entry_point` - Kernel entry point (virtual address) +/// * `stack_pointer` - Initial stack pointer for EL1 +/// +/// # Safety +/// +/// This function never returns to the caller. +#[inline(never)] +pub unsafe fn enable_mmu_and_drop_to_el1( + ttbr0: u64, + ttbr1: u64, + vbar: u64, + entry_point: u64, + stack_pointer: u64, +) -> ! { + // MAIR: Memory Attribute Indirection Register + // Index 0: Normal memory, Write-Back, Read/Write Allocate + // Index 1: Device-nGnRnE memory + #[expect(clippy::identity_op)] + let mair: u64 = 0xFF | (0x00 << 8); + + // ═══════════════════════════════════════════════════════════ + // STEP 1: Configure EL2 to allow EL1 operation + // ═══════════════════════════════════════════════════════════ + + // Set Hypervisor Configuration Register (EL2) + // Set EL1 execution state to AArch64 + // @todo Explain the SWIO bit (SWIO hardwired on Pi3) + HCR_EL2.write(HCR_EL2::RW::EL1IsAarch64 + HCR_EL2::SWIO::SET); + // @todo disable VM bit to prevent stage 2 MMU translations + + // ═══════════════════════════════════════════════════════════ + // STEP 2: Set up VBAR_EL1 (Exception Vector Base Address) + // ═══════════════════════════════════════════════════════════ + + assert!( + vbar.trailing_zeros() >= 11, + "Vector table address {vbar} is NOT properly aligned!" + ); + + // The address must be 2KB aligned (bits [10:0] must be 0). + // We set the virtual address here since VBAR_EL1 is only + // used after MMU is enabled (exceptions before ERET would + // be taken at EL2, not EL1). + VBAR_EL1.set(vbar); + + // Force VBAR update to complete before next instruction. + barrier::isb(barrier::SY); + + liblog::info!("[!] Exception traps set up"); + + // ═══════════════════════════════════════════════════════════ + // STEP 3: Configure EL1 MMU settings + // ═══════════════════════════════════════════════════════════ + + MAIR_EL1.set(mair); + + TCR_EL1.write( + TCR_EL1::TBI0::Ignored // Top byte ignored, can be used for tagging. + // + TCR_EL1::IPS.val(ips) // Intermediate Physical Address Size + // ttbr0 user memory addresses + + TCR_EL1::TG0::KiB_4 // 4 KiB granule + + TCR_EL1::SH0::Inner + + TCR_EL1::ORGN0::WriteBack_ReadAlloc_WriteAlloc_Cacheable + + TCR_EL1::IRGN0::WriteBack_ReadAlloc_WriteAlloc_Cacheable + // + TCR_EL1::EPD0::EnableTTBR0Walks + + TCR_EL1::T0SZ.val(16) // T0SZ = 16 (48-bit VA for TTBR0) + // ttbr1 kernel memory addresses + // + TCR_EL1::TBI1::Ignored // Top byte ignored, can be used for tagging. @todo remove! + + TCR_EL1::TG1::KiB_4 // 4 KiB granule + + TCR_EL1::SH1::Inner + + TCR_EL1::ORGN1::WriteBack_ReadAlloc_WriteAlloc_Cacheable + + TCR_EL1::IRGN1::WriteBack_ReadAlloc_WriteAlloc_Cacheable + // + TCR_EL1::EPD1::DisableTTBR1Walks // @fixme disabled for now + + TCR_EL1::T1SZ.val(16), // T1SZ = 16 (48-bit VA for TTBR1) + ); + + TTBR0_EL1.set(ttbr0); + TTBR1_EL1.set(ttbr1); + + // ═══════════════════════════════════════════════════════════ + // STEP 4: Prepare to drop to EL1 with MMU enabled + // ═══════════════════════════════════════════════════════════ + + // Enable MMU (takes effect after ERET) + SCTLR_EL1.modify( + SCTLR_EL1::EE::LittleEndian // Endianness select in EL1 + + SCTLR_EL1::E0E::LittleEndian // Endianness select in EL0 + + SCTLR_EL1::WXN::Disable // Writable means Execute Never + + SCTLR_EL1::SA::Disable // SP Alignment check in EL1, 16 byte align + + SCTLR_EL1::SA0::Disable // SP Alignment check in EL0, 16 byte align + + SCTLR_EL1::A::Disable // No alignment checks + + SCTLR_EL1::UCI::Trap // Unified Cache instructions trap + + SCTLR_EL1::UCT::Trap // CTR_EL0 instructions trap + + SCTLR_EL1::UMA::Trap // User Mask Access, trap on DAIF access + + SCTLR_EL1::NTWE::Trap // WFE/WFET instruction trap + + SCTLR_EL1::NTWI::Trap // WFI/WFIT instruction trap + + SCTLR_EL1::DZE::Trap // DC ZVA/GVA/GZVA instructions trap + + SCTLR_EL1::C::Cacheable + + SCTLR_EL1::I::Cacheable + + SCTLR_EL1::M::Enable, + ); + + // Set Saved Program Status Register (EL2) + // Set up a simulated exception return. + // + // Fake a saved program status, where all interrupts were + // masked and SP_EL1 was used as a stack pointer. + SPSR_EL2.write( + SPSR_EL2::D::Masked + + SPSR_EL2::A::Masked + + SPSR_EL2::I::Masked + + SPSR_EL2::F::Masked + + SPSR_EL2::M::EL1h, // Use SP_EL1, Return to EL1 + ); + + // TODO: Mask interrupts in EL1 + SPSR_EL1.write( + SPSR_EL1::D::Masked + + SPSR_EL1::A::Masked + + SPSR_EL1::I::Masked + + SPSR_EL1::F::Masked + + SPSR_EL1::M::EL1h, // Use SP_EL1 + ); + + // Set return address and stack + ELR_EL2.set(entry_point); + SP_EL1.set(stack_pointer); + + memory_barrier(); + + // ═══════════════════════════════════════════════════════════ + // STEP 5: Drop to EL1 + // ═══════════════════════════════════════════════════════════ + asm::eret() // FIXME this doesn't pass DTB (and we hopefully don't need it anymore) +} + +/// Invalidate all TLB entries +#[inline(always)] +pub fn tlb_invalidate_all() { + // SAFETY: Unsafe + unsafe { + core::arch::asm!( + "dsb ishst", + "tlbi vmalle1", + "dsb ish", + "isb", + options(nostack, preserves_flags) + ); + } +} diff --git a/kernel/kickstart/src/embed.rs b/kernel/kickstart/src/embed.rs new file mode 100644 index 000000000..9c49eeb14 --- /dev/null +++ b/kernel/kickstart/src/embed.rs @@ -0,0 +1,2 @@ +// Include generated types +include!(concat!(env!("OUT_DIR"), "/kernel_sections.rs")); diff --git a/kernel/kickstart/src/loader.rs b/kernel/kickstart/src/loader.rs new file mode 100644 index 000000000..01330671d --- /dev/null +++ b/kernel/kickstart/src/loader.rs @@ -0,0 +1,216 @@ +// kickstart/src/loader.rs + +#[cfg(feature = "qemu")] +use libqemu::semi_println; +use { + crate::{ + embed::KERNEL, + memory::{Alloc, BootAllocator, KernelLayout, MemoryPermissions}, + }, + core::ptr, + libaddress::{PhysAddr, VirtAddr}, +}; + +/// Metadata for a kernel section +#[derive(Debug, Clone, Copy)] +pub struct SectionMeta { + /// Section name (for debugging) + pub name: &'static str, + /// Virtual address in kernel's higher-half address space + pub virt_addr: u64, + /// Size of section in bytes + pub size: usize, + /// Required alignment in bytes + pub alignment: u64, + /// Memory protection permissions + pub permissions: MemoryPermissions, +} + +impl SectionMeta { + /// Calculate offset from kernel virtual base + pub const fn offset_from_base(&self, virt_base: u64) -> u64 { + self.virt_addr - virt_base + } + + /// Calculate physical address given kernel physical base + pub const fn phys_addr(&self, kernel_phys_base: u64, kernel_virt_base: u64) -> u64 { + kernel_phys_base + self.offset_from_base(kernel_virt_base) + } + + /// Number of 4KB pages needed + pub const fn page_count(&self) -> usize { + self.size.div_ceil(0x1000) + } +} + +/// Complete kernel image information +#[derive(Debug)] +pub struct ImageInfo { + /// Virtual base address (higher-half) -- FIXME: don't need this necessarily + pub virt_base: u64, + /// Loadable sections with their binary data + pub sections: &'static [LoadableSection], + /// BSS section metadata (no binary data - must be zeroed) + pub bss: SectionMeta, + /// BSS section metadata (no binary data - must be zeroed) + pub stack_virt_bottom: u64, + /// Exception vector table metadata (to set up VBAR) + pub vectors: SectionMeta, +} + +impl ImageInfo { + /// Total size needed for kernel in physical memory (all sections + BSS) + pub fn total_size(&self) -> usize { + let mut max_end: u64 = 0; + + for section in self.sections { + let end = section.meta.virt_addr + section.meta.size as u64; + max_end = max_end.max(end); + } + + let bss_end = self.bss.virt_addr + self.bss.size as u64; + max_end = max_end.max(bss_end); + + let size = usize::try_from(max_end - self.virt_base).unwrap(); + (size + 0xFFF) & !0xFFF // FIXME: aligned to a page size + } +} + +/// A loadable section with its binary content +#[derive(Debug)] +pub struct LoadableSection { + pub meta: SectionMeta, + pub data: &'static [u8], // or Option<&'static [u8]>? +} + +pub fn load_kernel(allocator: &mut BootAllocator) -> Result { + let total_size = KERNEL.total_size(); + let total_pages = total_size.div_ceil(0x1000); + + // Allocate 2MB-aligned for potential huge page mapping -- FIXME: with this we can abandon the whole loaded image and do ASLR easy + let phys_base = allocator + .alloc_aligned( + total_pages * 0x1000, + 2 * 1024 * 1024, + ("", Alloc::Persistent), + ) + .ok_or("Failed to allocate memory for kernel")?; + + #[cfg(feature = "qemu")] + semi_println!( + "Nucleus is {total_pages} * 4K pages @ {:#016X}", + phys_base.as_u64() + ); + + // Load each section + for section in KERNEL.sections { + load_section(section, phys_base)?; + } + + // Zero BSS section + zero_bss(&KERNEL.bss, phys_base)?; + + memory_barrier(); + + // Build layout information + let bss_info = { + let phys = + PhysAddr::new(phys_base.as_u64() + KERNEL.bss.offset_from_base(KERNEL.virt_base)); + (phys, VirtAddr::new(KERNEL.bss.virt_addr), KERNEL.bss.size) + }; + + // Calculate vector table addresses + let vectors_virt = { + let virt = VirtAddr::new(KERNEL.vectors.virt_addr); + + // Verify alignment + assert!( + virt.is_aligned(2048_u64), + "Vector table virtual address 0x{:016X} is not 2KB aligned!", + virt.as_u64() + ); + + virt + }; + + Ok(KernelLayout { + phys_base, + virt_base: VirtAddr::new(KERNEL.virt_base), + total_size, + sections: KERNEL.sections, + bss_phys: bss_info.0, + bss_virt: bss_info.1, + bss_size: bss_info.2, + stack_virt_bottom: VirtAddr::new(KERNEL.stack_virt_bottom), + vectors_virt, + }) +} + +fn load_section(section: &LoadableSection, kernel_phys_base: PhysAddr) -> Result<(), &'static str> { + let offset = section.meta.offset_from_base(KERNEL.virt_base); + let dest_phys = PhysAddr::new(kernel_phys_base.as_u64() + offset); + + #[cfg(feature = "qemu")] + semi_println!( + "> section {}, copy {} bytes of {} bytes total to {:#016X}", + section.meta.name, + section.data.len(), + section.meta.size, + dest_phys.as_u64() + ); + + if !dest_phys.as_u64().is_multiple_of(section.meta.alignment) { + return Err("Section alignment violated"); + } + + // SAFETY: Unsafe + unsafe { + ptr::copy_nonoverlapping( + section.data.as_ptr(), + dest_phys.as_mut_ptr::(), + section.data.len(), + ); + } + + if section.meta.size > section.data.len() { + let zero_start = PhysAddr::new(dest_phys.as_u64() + section.data.len() as u64); + let zero_size = section.meta.size - section.data.len(); + // SAFETY: Unsafe + unsafe { + ptr::write_bytes(zero_start.as_mut_ptr::(), 0, zero_size); + } + } + + Ok(()) +} + +fn zero_bss(bss: &SectionMeta, kernel_phys_base: PhysAddr) -> Result<(), &'static str> { + let offset = bss.offset_from_base(KERNEL.virt_base); + let dest_phys = PhysAddr::new(kernel_phys_base.as_u64() + offset); + + #[cfg(feature = "qemu")] + semi_println!( + "> section {}, zero {} bytes at {:#016X}", + bss.name, + bss.size, + dest_phys.as_u64() + ); + + if !dest_phys.as_u64().is_multiple_of(bss.alignment) { + return Err("BSS alignment violated"); + } + + // SAFETY: Unsafe + unsafe { + ptr::write_bytes(dest_phys.as_mut_ptr::(), 0, bss.size); + } + Ok(()) +} + +#[inline(always)] +pub fn memory_barrier() { + // SAFETY: Unsafe + unsafe { + core::arch::asm!("dsb sy", "isb", options(nostack, preserves_flags)); + } +} diff --git a/kernel/kickstart/src/main.rs b/kernel/kickstart/src/main.rs new file mode 100644 index 000000000..a52dfee70 --- /dev/null +++ b/kernel/kickstart/src/main.rs @@ -0,0 +1,1186 @@ +#![no_std] +#![no_main] +#![allow(unused)] +#![feature(format_args_nl)] +#![feature(try_find)] // For DeviceTree iterators + +// Init-thread process. +// - Start initializing the kernel +// - Enter itself into process list as high-priority privileged process +// - The bootup is driven by the kickstart which loads and parses devtree, maps the kernel, gives itself all necessary +// capabilities, (probably loads more things into their places) and transitions to user mode. +// - From user mode it can continue distributing capabilities and launching servers until everything is handed out. +// - After init is completed, it should create more low-priority user processes including idle, fs, some other handlers and +// a user-space boot process like /sbin/init or sth, with scripts to control the boot up. +// - At this point, exit the kickstart process normally. +// - The init-thread can be terminated and its memory freed up (should it inject a process descriptor for itself somehow to allow normal shutdown mechanisms to clean up? most probably). + +// create "tracing" and "debug" components for kernel call keys (intercepting syscall caps) + +// Kernel's main.rs just brings together all libs and syscall entry points. +// kickstart.rs provides a boot up entry point which sets up everything. + +// kickstart should do some shared init +// and some system-specific loading like parsing the DTB and loading system drivers +// distribute keys - this should be listed somewhere in the definitions (CapDL?) + +// kernel entities: +// - keys +// - key invocation +// - time access and timers activation - ?? + +// userspace entities: +// - processes +// - threads (first version - threads are in-process, kernel has no idea) +// - scheduler (invokes process upcall key) + +mod boot_info; +mod device_tree; +mod el_switch; +mod embed; +mod loader; +mod memory; +mod paging; +mod qsort; + +#[cfg(feature = "qemu")] +use libqemu::semi_println; +use { + crate::{boot_info::BOOT_INFO, memory::Alloc}, + aarch64_cpu::registers::{SPSR_EL2, Writeable}, + core::{cell::UnsafeCell, panic::PanicInfo, ptr::write_bytes, slice}, + device_tree::{DeviceTree, DeviceTreeProp}, + fdt_rs::{ + base::DevTree, + error::DevTreeError, + prelude::{FallibleIterator, PropReader}, + }, + libaddress::{PhysAddr, VirtAddr}, + libboot::entry, + libcpu::endless_sleep, + liblocking::interface::Mutex, + libmapping::{AccessPermissions, AttributeFields, MemAttributes}, + libobject::{DebugConsoleKey, KeySlot}, + libsyscall::protected_call6, + memory::BootAllocator, +}; + +unsafe extern "C" { + static __INIT_START: UnsafeCell<()>; + static __INIT_END: UnsafeCell<()>; + static __FREE_MEMORY_START: UnsafeCell<()>; +} + +#[panic_handler] +fn panic(info: &PanicInfo) -> ! { + #[cfg(feature = "qemu")] + semi_println!("PANICKED: {info}"); + cfg_if::cfg_if! { + if #[cfg(feature = "qemu")] { + libqemu::semihosting::exit_failure() + } else { + endless_sleep() + } + } +} + +fn dump_memory_map() { + // Output the memory map as we could derive from FDT and information about our loaded image + // Use it to imagine how the memmap would look like in the end. + BOOT_INFO.lock(|bi| { + bi.compact(); + bi.sort(); + bi.dump(); + }); +} + +entry!(init_main_el2); + +/// Kernel early init code. +/// `arch` crate is responsible for calling it. +/// +/// Safety +/// +/// - Only a single core must be active and running this function. +/// - The init calls in this function must appear in the correct order: +/// - MMU + Data caching must be activated at the earliest. Without it, any atomic operations, +/// e.g. the yet-to-be-introduced spinlocks in the device drivers (which currently employ +/// `IRQSafeNullLocks` instead of spinlocks), will fail to work (properly) on the `RPi` `SoCs`. +/// +pub fn init_main_el2(dtb: u32) -> ! { + let dtb_ptr = dtb as *const u8; + + SPSR_EL2.write( + SPSR_EL2::D::Masked + + SPSR_EL2::A::Masked + + SPSR_EL2::I::Masked + + SPSR_EL2::F::Masked + + SPSR_EL2::M::EL1h, // Use SP_EL1/2 + ); + + #[cfg(feature = "jtag")] + libmachine::debug::jtag::wait_debugger(); + + #[cfg(feature = "qemu")] + semi_println!("init_main started"); + + // unsafe { + // BOOT_INFO.dtb_phys = PhysAddr::new(dtb_phys); + // } + + // ───────────────────────────────────────────────────────────────────── + // Initialize early UART for debug output + // ───────────────────────────────────────────────────────────────────── + + // Hardcoded UART address for early boot (RPi4: 0xFE201000) + // Will be properly mapped later + // early_uart_init(0xFE20_1000); + #[cfg(feature = "qemu")] + semi_println!("DTB at physical: {:#016x}", dtb_ptr as u64); + + // ───────────────────────────────────────────────────────────────────── + // Start bump allocator + // ───────────────────────────────────────────────────────────────────── + + // SAFETY: Unsafe + let init_start = unsafe { __INIT_START.get() as u64 }; + // SAFETY: Unsafe + let init_end = unsafe { __INIT_END.get() as u64 }; + // SAFETY: Unsafe + let free_start = unsafe { __FREE_MEMORY_START.get() as u64 }; + + let memory_size = 256 * 1024 * 1024; + let mut allocator = BootAllocator::new(PhysAddr::new(free_start), memory_size); + let memory_end = allocator.end(); + #[cfg(feature = "qemu")] + semi_println!( + "init_main: Created BootAllocator {memory_size} @ {:#016x}", + free_start + ); + + // ───────────────────────────────────────────────────────────────────── + // Parse Device Tree + // ───────────────────────────────────────────────────────────────────── + + #[cfg(feature = "qemu")] + semi_println!("Parsing device tree..."); + + // Safety: we got the address from the bootloader, if it lied - well, we're screwed! + let device_tree = + unsafe { DevTree::from_raw_pointer(dtb_ptr).expect("DeviceTree failed to read") }; + + let layout = DeviceTree::layout(device_tree).expect("Couldn't calculate DeviceTree index"); + + let block = allocator + .alloc_aligned( + layout.size(), + layout.align(), + ("DTB index", Alloc::Droppable), + ) + .expect("Couldn't allocate DeviceTree index"); + // SAFETY: Unsafe call. + let raw_slice = unsafe { core::slice::from_raw_parts_mut(block.as_mut_ptr(), layout.size()) }; + + let device_tree = + DeviceTree::new(device_tree, raw_slice).expect("Couldn't initialize indexed DeviceTree"); + + let board = device_tree.get_prop_by_path("/model").unwrap().str(); + if let Ok(board_name) = board { + #[cfg(feature = "qemu")] + semi_println!("Running on {board_name}"); + } + + // let mut dumper = device_tree.dumper(0); + // dumper.dump_metadata(); + // dumper.dump_root().expect("oof"); + + // To init memory allocation we need to parse memory regions from dtb and add the regions to + // available memory regions list. Then initial BootRegionAllocator will get memory from these + // regions and record their usage into some OTHER structures, removing these allocations from + // the free regions list. + // memory allocation is described by reg attribute of /memory block. + // /#address-cells and /#size-cells specify the sizes of address and size attributes in reg. + // To get memory size from DTB: + // 1. Find nodes with unit-names `/memory` + // 2. From those read reg entries, using `/#address-cells` and `/#size-cells` as units + // 3. Union of all these reg entries will be the available memory. Enter it as mem-regions. + + let res: Result<_, DevTreeError> = device_tree + .props() + .try_find(|p| Ok(p.name()? == "device_type" && p.str()? == "memory")); + let mem_prop = res.unwrap().expect("Unable to find memory node."); + let _mem_node = mem_prop.node(); + // let parent_node = mem_node.parent_node(); + + // reg == region, usually defines LOC+SIZE unless #size-cells is set to 0 + // can also be reg = <0x7e100000 0x00000114 0x7e00a000 0x00000024 >; to define two locations + let reg_prop = device_tree + .get_prop_by_path("/memory@0/reg") + .expect("Unable to figure out memory-reg"); + + #[cfg(feature = "qemu")] + semi_println!( + "Found memnode with reg prop: name {:?}, size {}", + reg_prop.name(), + reg_prop.length() + ); + + let reg_prop = DeviceTreeProp::new(reg_prop); + + let mut total_memory = 0; + + for (mem_addr, mem_size) in reg_prop.payload_pairs_iter() { + #[cfg(feature = "qemu")] + semi_println!("Memory: {} KiB at offset {}", mem_size / 1024, mem_addr); + total_memory += mem_size; + BOOT_INFO.lock(|bi| { + bi.insert_free_region( + PhysAddr::new(mem_addr), + PhysAddr::new(mem_addr + mem_size), + AttributeFields::default(), + "RAM", + ) + .expect("tough luck"); + }); + } + + // 4. List unusable memory, and remove it from the memory regions for the allocator. + for entry in device_tree.fdt().reserved_entries() { + let size: u64 = entry.size.into(); + let address: u64 = entry.address.into(); + #[cfg(feature = "qemu")] + semi_println!("Reserved memory: {size:?} bytes at {address:?}"); + BOOT_INFO.lock(|bi| { + bi.insert_used_region( + PhysAddr::new(entry.address.into()), + PhysAddr::new(u64::from(entry.address) + u64::from(entry.size)), + AttributeFields::default(), + "Reserved", + ) + .expect("tough luck"); + }); + } + + // 5. Also list memreserve entries, and remove then from allocator regions? + // From FDT dump: + // memreserve = <0x3b400000 0x04c00000 >; + + // Iterate compatible nodes (example): + for entry in device_tree.compatible_nodes("arm,pl011") { + #[cfg(feature = "qemu")] + semi_println!("PL011 device: {:?}", entry.name() /*, entry.address*/); + } + + // 6. Also, remove the DTB memory region + index + #[cfg(feature = "qemu")] + semi_println!( + "DTB region: {} bytes at {:#016x}", + device_tree.fdt().totalsize(), + dtb_ptr as usize + ); // also include the raw_slice allocated bit + BOOT_INFO.lock(|bi| { + bi.insert_used_region( + PhysAddr::new(dtb_ptr as u64), + PhysAddr::new(dtb_ptr as u64 + device_tree.fdt().totalsize() as u64), + AttributeFields { + droppable: true, + ..Default::default() + }, + "DTB", + ) + .expect("tough luck"); + }); + + // Next step: parse DTB! + // iterate nodes, look for reg, status, compat props + + #[expect(clippy::items_after_statements)] + #[derive(Default, Copy, Clone)] + struct Node { + name: &'static str, + compat: &'static str, + phandle: u32, + disabled: bool, + start: u64, + size: u64, + } + + let mut nodes: [Node; 100] = [Node::default(); 100]; + let mut num_nodes = 0; + + // See https://mjmwired.net/kernel/Documentation/devicetree/bindings/display/brcm,bcm-vc4.txt + // All these DT thingies are Broadcom= and Linux-specific, so need to read both to decode anything useful. + // https://mjmwired.net/kernel/Documentation/devicetree/usage-model.rst <- entry point + + // The trick is that the kernel starts at the root of the tree and looks + // for nodes that have a 'compatible' property. First, it is generally + // assumed that any node with a 'compatible' property represents a device + // of some kind, and second, it can be assumed that any node at the root + // of the tree is either directly attached to the processor bus, or is a + // miscellaneous system device that cannot be described any other way. + // For each of these nodes, Linux allocates and registers a + // platform_device, which in turn may get bound to a platform_driver. + + // Gather and print the following info: reg (start + size) x times, phandle if any, name, compat + // Sort them by start address to get ordered device map. + // + // e.g.: + // mmcnr@7e300000 @ 0x7e300000 +0x100 ("brcm,bcm2835-mmc", "brcm,bcm2835-sdhci") + // [0x2f] mmc@7e300000 @ 0x7e300000 +0x100 ("brcm,bcm2835-mmc", "brcm,bcm2835-sdhci") + // + // To add later: clocks and interrupts, if any + // Print "-" prefix is status = disabled; + + for entry in device_tree.nodes() { + if let Some(item) = entry.props().find(|p| p.name() == Ok("reg")) { + let compat_names = entry + .props() + .find(|p| p.name() == Ok("compatible")) + .and_then(|prop| prop.str().ok()) + .unwrap_or(""); + let phandle = entry + .props() + .find(|p| p.name() == Ok("phandle")) + .and_then(|prop| prop.phandle(0).ok()); + let disabled = entry + .props() + .find(|p| p.name() == Ok("status")) + .and_then(|prop| prop.str().ok()) + .is_some_and(|value| value == "disabled"); + let name = entry.name().unwrap(); + let name = name.split_once('@').unwrap_or((name, "")).0; + + let reg_prop = DeviceTreeProp::new(item); + for (mem_base, mem_size) in reg_prop.payload_pairs_iter() { + nodes[num_nodes] = Node { + start: mem_base, + size: mem_size, + name, + compat: compat_names, + disabled, + phandle: phandle.unwrap_or_default(), + }; + num_nodes += 1; + } + } + } + + let mut nodes = &mut nodes[..num_nodes]; + + // Other in-place sorting available: + if !nodes.is_sorted_by_key(|item| item.start) { + nodes.sort_unstable_by_key(|item| item.start); + } + + for node in nodes { + #[cfg(feature = "qemu")] + semi_println!( + "{}[{:02x}] {:<22} @ {} +{} ({})", + if node.disabled { "-" } else { " " }, + node.phandle, + node.name, + PhysAddr::new(node.start), + node.size, + node.compat + ); + + if node.name != "memory" && node.name != "gpio" && node.name != "mmc" && node.name != "smi" + { + BOOT_INFO.lock(|bi| { + bi.insert_used_region( + PhysAddr::new(node.start), + PhysAddr::new(node.start + node.size), + AttributeFields { + mem_attributes: MemAttributes::Device, + ..AttributeFields::default() + }, + node.name, + ) + .expect("tough luck"); + }); + } + } + + #[cfg(feature = "qemu")] + semi_println!(""); + #[cfg(feature = "qemu")] + semi_println!(""); + + for entry in device_tree.nodes() { + if entry.name() == Ok("chosen") { + #[cfg(feature = "qemu")] + semi_println!("Found /chosen node"); + } + } + + // unsafe { + // BOOT_INFO.dtb_size = dtb.total_size(); + + // // Extract memory regions + // for region in dtb.memory_regions() { + // if BOOT_INFO.memory_region_count < 16 { + // BOOT_INFO.memory_regions[BOOT_INFO.memory_region_count] = region; + // BOOT_INFO.memory_region_count += 1; + // semi_println!( + // " RAM: {:#x} - {:#x}", + // region.base.as_u64(), + // region.base.as_u64() + region.size as u64 + // ); + // } + // } + + // // Extract reserved regions + // for reserved in dtb.reserved_regions() { + // if BOOT_INFO.reserved_region_count < 32 { + // BOOT_INFO.reserved_regions[BOOT_INFO.reserved_region_count] = reserved; + // BOOT_INFO.reserved_region_count += 1; + // } + // } + + // // Extract boot modules (loaded by bootloader) + // for module in dtb.modules() { + // if BOOT_INFO.module_count < 8 { + // BOOT_INFO.modules[BOOT_INFO.module_count] = module; + // BOOT_INFO.module_count += 1; + // semi_println!( + // " Module '{}': {:#x}, {} bytes", + // core::str::from_utf8(&module.name).unwrap_or("???"), + // module.phys_start.as_u64(), + // module.size + // ); + // } + // } + // } + + // ═══════════════════════════════════════════════════════════════ + // PHASE 1: Load kernel + // ═══════════════════════════════════════════════════════════════ + + #[cfg(feature = "qemu")] + semi_println!("init_main: Load kernel"); + + let kernel_layout = loader::load_kernel(&mut allocator).expect("Failed to load nucleus"); + #[cfg(feature = "qemu")] + semi_println!("init_main: Loaded nucleus image"); + + // ═══════════════════════════════════════════════════════════════ + // PHASE 2: Set up page tables + // ═══════════════════════════════════════════════════════════════ + + let (el1_stack, el1_stack_size) = { + // Allocate EL1 stack + let el1_stack_size = 128; // pages + let el1_stack = allocator + .alloc_pages(el1_stack_size, ("Nucleus stack", Alloc::Persistent)) + .expect("Failed to allocate EL1 stack"); + let el1_stack_size = el1_stack_size * 4096; // 64KiB stack + (el1_stack, el1_stack_size) + }; + + // Mark kernel memory used: + // TODO: add alignment requirements to boot_info regions (align up to) + BOOT_INFO.lock(|bi| { + for sec in kernel_layout.iter_sections() { + bi.insert_used_region( + sec.phys_start, + sec.phys_start + sec.size, + AttributeFields { + acc_perms: if sec.permissions.writable { + AccessPermissions::ReadWrite + } else { + AccessPermissions::ReadOnly + }, + executable: sec.permissions.executable, + ..Default::default() + }, + sec.name, + ); + } + bi.insert_used_region( + kernel_layout.bss_phys, + kernel_layout.bss_phys + kernel_layout.bss_size, + AttributeFields::defaulted(), + "Nucleus BSS", + ); + bi.insert_used_region( + el1_stack, + el1_stack + el1_stack_size, + AttributeFields::defaulted(), + "Nucleus stack", + ); + }); + + let mut mmu_setup = paging::MmuSetup::new(&mut allocator).expect("Failed to create MMU setup"); + #[cfg(feature = "qemu")] + semi_println!("init_main: Created MmuSetup"); + + // Identity map kickstart + paging::create_identity_mapping(&mut mmu_setup, PhysAddr::new(init_start), memory_end) + .expect("Failed to create identity mapping"); + #[cfg(feature = "qemu")] + semi_println!("init_main: Identity mapped the Kickstart"); + + // Create kernel mapping with per-section permissions + let (el1_stack_top,) = paging::create_kernel_mapping( + &mut mmu_setup, + &kernel_layout, + total_memory, + el1_stack.as_u64(), + el1_stack_size, + ) + .expect("Failed to create kernel mapping"); + #[cfg(feature = "qemu")] + semi_println!("init_main: Higher-half mapped the nucleus"); + + // ═══════════════════════════════════════════════════════════════ + // Interlude: Print the BOOT_INFO region map + // ═══════════════════════════════════════════════════════════════ + + BOOT_INFO.lock(|bi| { + bi.insert_overlay_region( + PhysAddr::new(init_start), + mmu_setup.memory_top(), // Up to allocated watermark + AttributeFields { + droppable: true, + ..Default::default() + }, + "Kickstart", + ); + }); + + #[cfg(feature = "qemu")] + semi_println!("init_main: BOOT_INFO map after kernel load and mapping"); + dump_memory_map(); + + // ═══════════════════════════════════════════════════════════════ + // PHASE 3: Prepare for EL1 + // ═══════════════════════════════════════════════════════════════ + + let ttbr0 = mmu_setup.ttbr0(); + let ttbr1 = mmu_setup.ttbr1(); + #[cfg(feature = "qemu")] + semi_println!("init_main: TTBR0_EL1 at {ttbr0:#016x}, TTBR1_EL1 at {ttbr1:#016x}"); + + // Get vector table virtual address for VBAR_EL1 + // VBAR is only used after MMU is enabled, so we set the virtual address directly + let vbar = kernel_layout.vbar_el1_virt(); + + #[cfg(feature = "qemu")] + semi_println!("init_main: EL1 stack at {el1_stack_top:#016x}, vbar {vbar:#016x}"); + + // ═══════════════════════════════════════════════════════════════ + // PHASE 4: Enable MMU and drop to EL1 + // ═══════════════════════════════════════════════════════════════ + + #[cfg(feature = "qemu")] + semi_println!("Init thread image covers phys -:- identity mapped"); + #[cfg(feature = "qemu")] + semi_println!("Init thread mapping tables filled in as - entries"); + #[cfg(feature = "qemu")] + semi_println!("Kernel image covers phys -:- mapped to KERNEL_HIGH_BASE:-"); + #[cfg(feature = "qemu")] + semi_println!("Kernel mapping tables filled in as - for kernel, as - for phys memory"); + + print_my_sp(); + + unsafe extern "Rust" { + // Stack top + static __STACK_TOP: UnsafeCell<()>; + } + + // SAFETY: Not safe. + unsafe { + #[expect(clippy::fn_to_numeric_cast_any)] + el_switch::enable_mmu_and_drop_to_el1( + ttbr0, + ttbr1, + vbar, + kickstart_run as *const u8 as u64, + // el1_stack_top, // This is solely for the kernel + __STACK_TOP.get() as u64, + ); + } +} + +// DTB should be available to this code through BOOT_INFO records. +pub fn kickstart_run() -> ! { + // ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + // PHASE 5: Initialize kernel objects and structures + // ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + + // Run initial thread further in EL1, seting up the capDL etc. + #[cfg(feature = "qemu")] + semi_println!("init_main_run: enabled MMU and dropped to EL1"); + print_my_sp(); + // SAFETY: Not safe. + unsafe { + protected_call6(0, 0, 0, 0, 0, 0, 0, 0); + } + #[cfg(feature = "qemu")] + semi_println!("init_main_run: Returned from fake syscall"); + print_my_sp(); + + // ───────────────────────────────────────────────────────────────────── + // Initialize kernel subsystems + // ───────────────────────────────────────────────────────────────────── + + #[cfg(feature = "qemu")] + semi_println!("Initializing kernel subsystems..."); + + // Initialize per-CPU data structures + // percpu::init(); + + // Initialize interrupt controller (GIC on RPi4) + // let boot_info = unsafe { &BOOT_INFO }; + // interrupts::init_gic(boot_info); + + // ───────────────────────────────────────────────────────────────────── + // Build physical memory map and create Untyped caps + // ───────────────────────────────────────────────────────────────────── + + // semi_println!("Building physical memory allocator..."); + + // Create the root untyped capability list + // This will be delegated to init process + // let mut untyped_list = UntypedList::new(); + // // let untyped_list = create_untyped_caps(); + + // for i in 0..boot_info.memory_region_count { + // let region = &boot_info.memory_regions[i]; + + // // Skip reserved regions (kernel image, DTB, modules, page tables) + // let usable_ranges = subtract_reserved_regions(region, boot_info); + + // for range in usable_ranges { + // // Create untyped caps for each usable chunk + // // Align to largest power-of-2 for efficient retyping + // let untypeds = create_untyped_caps_for_range(range); + // untyped_list.extend(untypeds); + + // semi_println!( + // " Untyped: {:#x} - {:#x} ({} caps)", + // range.base.as_u64(), + // range.base.as_u64() + range.size as u64, + // untypeds.len() + // ); + // } + // } + + // semi_println!("Total untyped caps: {}", untyped_list.len()); + + // ───────────────────────────────────────────────────────────────────── + // Initialize DCB shared pages + // ───────────────────────────────────────────────────────────────────── + + // semi_println!("Initializing DCB pages..."); + + // // Allocate DCB pages from a reserved untyped + // // These are special: mapped RW in kernel, RO in all user domains + // let dcb_pages = allocate_dcb_pages(&mut untyped_list, MAX_DOMAINS); + // dcb::init(dcb_pages); + + // ───────────────────────────────────────────────────────────────────── + // Create kernel idle domain (domain 0) + // ───────────────────────────────────────────────────────────────────── + + // semi_println!("Creating idle domain..."); + + // let idle_domain = Domain::create_idle(); + // SCHEDULER.set_idle(idle_domain); + + // ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + // PHASE 6: Create the init domain and its capability space + // ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + + // semi_println!("Creating init domain..."); + + // let init_module = boot_info + // .modules + // .iter() + // .find(|m| { + // let name = core::str::from_utf8(&m.name).unwrap_or(""); + // name.matches("init") + // }) + // .expect("No init module found in boot modules"); + + // let init_domain = create_init_domain(init_module, &mut untyped_list); + + // ───────────────────────────────────────────────────────────────────── + // Mark init thread memory as reclaimable + // ───────────────────────────────────────────────────────────────────── + + // semi_println!("Marking init thread memory for reclamation..."); + + // // The init stack and any init-only code/data can now be reclaimed, the are in the Untypeds table now. + // mark_init_memory_reclaimable(boot_info, &mut untyped_list); + + // ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + // PHASE 7: Delegate all resources to init domain + // ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + + // semi_println!( + // "Delegating {} untyped caps to init...", + // untyped_list.len() + // ); + + // delegate_untypeds_to_init(&init_domain, untyped_list); + + // // Create module caps for other boot modules and delegate + // delegate_module_caps_to_init(&init_domain, boot_info); + + // ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + // PHASE 8: Context switch to init domain + // ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + + // We have domain caps here, can use: + let dbg = DebugConsoleKey::new(); + dbg.write( + "DEBCON| Debug output via capability invocation on domain's debug console capability\n", + ); + + let err = DebugConsoleKey::new_slot(KeySlot::CAPTBL_SELF); + err.write("DEBCON| Invalid capability invocation - no output"); + + let (_, privilege_level) = libexception::current_privilege_level(); + liblog::info!("Current privilege level: {privilege_level}"); + + liblog::info!("Exception handling state:"); + libexception::asynchronous::print_state(); + + // semi_println!("Switching to init domain..."); + // semi_println!("═══════════════════════════════════════════════════════════"); + + // // Create initial time budget for init + // let init_time = TimeCap::create_root(INIT_TIME_BUDGET_US); + + // // Finally: switch to userspace init + // // This replaces TTBR0 with init's page tables + // // Kernel high map (TTBR1) is ready for when init makes syscalls + // // This never returns + // switch_to_domain(init_domain, init_time); + print_my_sp(); + + cfg_if::cfg_if! { + if #[cfg(feature = "qemu")] { + libqemu::semihosting::exit_success() + } else { + endless_sleep() + } + } + + // kernel_init_mmio_va_allocator() + + // SAFETY: Not safe! + // if let Err(x) = unsafe { libplatform::platform::drivers::init() } { + // panic!("Error initializing platform drivers: {}", x); + // } + + // Initialize all device drivers. + // SAFETY: Not safe! + // unsafe { + // libplatform::platform::drivers::driver_manager().init_drivers_and_irqs(); + // } + + // Unmask interrupts on the boot CPU core. + // libexception::exception::asynchronous::local_irq_unmask(); + + // Announce conclusion of the kernel_init() phase. + // libkernel_state::state_manager().transition_to_single_core_main(); + + // libconsole::init_logger(); + + // info!("{}", libkernel::version()); + + // info!( + // "{} version {}", + // env!("CARGO_PKG_NAME"), + // env!("CARGO_PKG_VERSION") + // ); + // info!( + // "Booting on: {}", + // libplatform::platform::BcmHost::board_name() + // ); + + // info!("MMU online. Special regions:"); + // machine::platform::memory::mmu::virt_mem_layout().print_layout(); + + // dump_memory_map(); + + // info!( + // "Architectural timer resolution: {} ns", + // libtime::time::time_manager().resolution().as_nanos() + // ); + + // info!("Drivers loaded:"); + // libplatform::platform::drivers::driver_manager().enumerate(); + + // info!("Registered IRQ handlers:"); + // libplatform::platform::exception::asynchronous::irq_manager().print_handler(); + + // // Test a failing timer case. + // libtime::time::time_manager().spin_for(Duration::from_nanos(1)); + + // for _ in 0..3 { + // info!("Spinning for 1 second"); + // libtime::time::time_manager().spin_for(Duration::from_secs(1)); + // } +} + +fn print_my_sp() { + use aarch64_cpu::registers::Readable; + let sp = aarch64_cpu::registers::SP.get(); + #[cfg(feature = "qemu")] + semi_println!("Current SP: {sp:016x}"); +} +/* +// ───────────────────────────────────────────────────────────────────── +// Create init domain from boot module +// ───────────────────────────────────────────────────────────────────── + +/// Create the init domain from a loaded ELF module +fn create_init_domain(module: &LoadedModule, untyped_list: &mut UntypedList) -> DomainRef { + // ───────────────────────────────────────────────────────────────────── + // Allocate domain kernel structures + // ───────────────────────────────────────────────────────────────────── + + // Take an untyped for domain structures + let domain_untyped = untyped_list + .take_of_size(DOMAIN_STRUCT_SIZE) + .expect("No memory for init domain"); + + let domain = Domain::create_from_untyped( + domain_untyped, + DomainId(1), // Init is domain 1 (0 is idle) + "init", + ); + + // ───────────────────────────────────────────────────────────────────── + // Create init's TTBR0 page tables + // ───────────────────────────────────────────────────────────────────── + + // Allocate page table memory from untyped + let pt_untyped = untyped_list + .take_of_size(PAGE_TABLE_SIZE) + .expect("No memory for init page tables"); + + let page_tables = UserPageTables::create_from_untyped(pt_untyped); + domain.set_page_tables(page_tables); + + // ───────────────────────────────────────────────────────────────────── + // Parse ELF and create address space + // ───────────────────────────────────────────────────────────────────── + + // Module is loaded in physical memory, accessible via kernel linear map + let elf_data = unsafe { + let virt = phys_to_kernel_virt(module.phys_start); + core::slice::from_raw_parts(virt as *const u8, module.size) + }; + + let elf = Elf64::parse(elf_data).expect("Invalid ELF"); + + semi_println!(" ELF entry point: {:#x}", elf.entry_point()); + + // Map each loadable segment + for phdr in elf.program_headers() { + if phdr.p_type != PT_LOAD { + continue; + } + + let virt_start = VirtAddr::new(phdr.p_vaddr); + let virt_end = virt_start + phdr.p_memsz as u64; + let file_size = phdr.p_filesz as usize; + let mem_size = phdr.p_memsz as usize; + + // Determine page flags from ELF flags + let flags = elf_flags_to_page_flags(phdr.p_flags); + + semi_println!( + " Segment: {:#x} - {:#x} ({:?})", + virt_start.as_u64(), + virt_end.as_u64(), + flags + ); + + // Allocate physical pages for this segment + let pages_needed = mem_size.div_ceil(PAGE_SIZE); + let segment_untyped = untyped_list + .take_of_size(pages_needed * PAGE_SIZE) + .expect("No memory for init segment"); + + // Map pages into init's address space + let phys_base = segment_untyped.phys_addr(); + domain + .page_tables() + .map_range(virt_start, phys_base, pages_needed, flags); + + // Copy segment data + let src = &elf_data[phdr.p_offset as usize..][..file_size]; + let dst = unsafe { + let virt = phys_to_kernel_virt(phys_base); + core::slice::from_raw_parts_mut(virt as *mut u8, mem_size) + }; + dst[..file_size].copy_from_slice(src); + + // Zero BSS portion + write_bytes(&mut dst[file_size..], 0, dst.len() - file_size); + + // Create BufferCap for this region and add to init's cspace -- FIXME: This is how we pass the process images to init domain + let buffer_cap = BufferCap::create_from_untyped(segment_untyped, flags.into()); + domain.cspace().insert_at_next_free(buffer_cap); + } + + // ───────────────────────────────────────────────────────────────────── + // Create init's stack + // ───────────────────────────────────────────────────────────────────── + + const INIT_STACK_SIZE: usize = 64 * 1024; // 64KB + const INIT_STACK_TOP: u64 = 0x7FFF_FFFF_0000; // should be dynamic.. + + let stack_untyped = untyped_list + .take_of_size(INIT_STACK_SIZE) + .expect("No memory for init stack"); + + domain.page_tables().map_range( + VirtAddr::new(INIT_STACK_TOP - INIT_STACK_SIZE as u64), + stack_untyped.phys_addr(), + INIT_STACK_SIZE / PAGE_SIZE, + PageFlags::USER_RW, + ); + + semi_println!( + " Stack: {:#x} - {:#x}", + INIT_STACK_TOP - INIT_STACK_SIZE as u64, + INIT_STACK_TOP + ); + + // ───────────────────────────────────────────────────────────────────── + // Setup initial register state + // ───────────────────────────────────────────────────────────────────── + + domain.set_entry_point(VirtAddr::new(elf.entry_point())); + domain.set_stack_pointer(VirtAddr::new(INIT_STACK_TOP)); + + // ───────────────────────────────────────────────────────────────────── + // Create init's initial capability space + // ───────────────────────────────────────────────────────────────────── + + setup_init_cspace(&domain); + + domain +} + +// ───────────────────────────────────────────────────────────────────── +// Create domain's initial capability space +// ───────────────────────────────────────────────────────────────────── + +/// Setup init's capability space with well-known slots +fn setup_init_cspace(domain: &DomainRef) { + let cspace = domain.cspace(); + + // Slot 0: NULL (always invalid) + // Slot 1: Self domain cap + cspace.insert(CSPACE_SLOT_SELF, domain.self_cap()); + + // Slot 2: Parent domain cap (for init, this is invalid/null) + // Slot 3: Current TimeCap (kernel sets this on activation) + + // Slots 0x10-0x1F: Will be filled with notification caps + // Slots 0x20-0x2F: Will be filled with event count caps + // etc. per the CSpace layout + + // Create a notification for init to receive kernel events + let kernel_notify = NotifyCap::create(); + cspace.insert(CSPACE_SLOT_KERNEL_NOTIFY, kernel_notify); +} + +// ───────────────────────────────────────────────────────────────────── +// Delegate all remaining untypeds to init +// ───────────────────────────────────────────────────────────────────── + +/// Delegate all untyped caps to init's cspace +fn delegate_untypeds_to_init(init_domain: &DomainRef, untyped_list: UntypedList) { + let cspace = init_domain.cspace(); + + // Start placing untypeds at well-known slot range + const UNTYPED_SLOT_START: u32 = 0x1000; + let mut slot = UNTYPED_SLOT_START; + + for untyped in untyped_list.into_iter() { + // Create capability referencing this untyped + let cap = UntypedCap::new(untyped); + cspace.insert(slot, cap); + slot += 1; + } + + // Store count so init knows how many it received + init_domain.dcb_mut().untyped_cap_count = slot - UNTYPED_SLOT_START; + + semi_prinln!( + " Delegated {} untyped caps to slots {:#x}-{:#x}", + slot - UNTYPED_SLOT_START, + UNTYPED_SLOT_START, + slot - 1 + ); +} + +/// Create caps for boot modules and delegate to init +fn delegate_module_caps_to_init(init_domain: &DomainRef, boot_info: &BootInfo) { + let cspace = init_domain.cspace(); + + const MODULE_SLOT_START: u32 = 0x2000; + let mut slot = MODULE_SLOT_START; + + for i in 0..boot_info.module_count { + let module = &boot_info.modules[i]; + + // Skip the init module itself (already loaded) + let name = core::str::from_utf8(&module.name).unwrap_or(""); + if name.matches("init") { + continue; + } + + // Create a read-only buffer cap for the module's memory - FIXME: this is how process images are passed on + let module_cap = + BufferCap::create_for_phys_range(module.phys_start, module.size, Rights::READ); + + cspace.insert(slot, module_cap); + + // Also store module metadata in a well-known location + // (Init can query its DCB for module info) + + semi_println!(" Module '{}' at slot {:#x}", name, slot); + slot += 1; + } + + init_domain.dcb_mut().module_cap_count = slot - MODULE_SLOT_START; +} + +/// Mark init thread memory as reclaimable +/// TODO: this memory should've been first removed from the DTB memory map! +fn mark_init_memory_reclaimable(boot_info: &BootInfo, untypeds_list: ) { + // The init thread area is entirely one blob of reclaimable memory. Convert it to untyped and donate to init domain. + + // Also mark .init sections in kernel image + extern "C" { + static __init_start: u8; + static __init_end: u8; + } + + unsafe { + let init_start = &__init_start as *const _ as u64; + let init_end = &__init_end as *const _ as u64; + let init_size = (init_end - init_start) as usize; + + if init_size > 0 { + untypeds_list.push(ReclaimableRegion { + phys: kernel_virt_to_phys(VirtAddr::new(init_start)), + size: init_size, + kind: ReclaimableKind::InitCode, + }); + + semi_println!( + " Init code memory {:#x}-{:#x} ({init_size} bytes) reclaimed", + init_start, + init_end + ); + } + } +} + +// ───────────────────────────────────────────────────────────────────── +// Switch to init domain +// ───────────────────────────────────────────────────────────────────── + +/// Final step: switch to init domain with initial time budget +fn switch_to_domain(domain: DomainRef, time: TimeCap) -> ! { + // Update domain state + { + let dcb = domain.dcb_mut(); + dcb.state + .store(DomainState::Running as u32, Ordering::Release); + dcb.time_remaining_ns + .store(time.remaining_us() * 1000, Ordering::Relaxed); + dcb.activation_count.fetch_add(1, Ordering::Relaxed); + } + + // Set current time cap in init's cspace + domain.cspace().insert(CSPACE_SLOT_CURRENT_TIME, time); + + // Setup TTBR0 for user space + let ttbr0 = domain.page_tables().root_phys(); + + // Get entry context + let entry_point = domain.entry_point(); + let stack_pointer = domain.stack_pointer(); + + // Record this as current domain -- why percpu?? + percpu::set_current_domain(domain); + + semi_println!( + "Entering init at {:#x} with SP={:#x}", + entry_point.as_u64(), + stack_pointer.as_u64() + ); + + // Do the context switch - this never returns + unsafe { + context_switch_to_user(ttbr0, entry_point, stack_pointer); + } +} + +/// Low-level context switch to user mode +#[unsafe(naked)] +unsafe extern "C" fn context_switch_to_user(ttbr0: PhysAddr, entry: VirtAddr, sp: VirtAddr) -> ! { + core::arch::naked_asm!( + // Set user page tables (TTBR0) + "msr ttbr0_el1, x0", + "isb", + // Invalidate TLB for ASID 0 (init) + "tlbi aside1is, xzr", + "dsb sy", + "isb", + // Set up ELR (return address) and SPSR (return state) + "msr elr_el1, x1", + // SPSR: EL0t, all interrupts enabled + "mov x3, #0", // EL0t + "msr spsr_el1, x3", + // Set user stack pointer + "msr sp_el0, x2", + // Clear all general-purpose registers for security + "mov x0, #0", + "mov x1, #0", + "mov x2, #0", + "mov x3, #0", + "mov x4, #0", + "mov x5, #0", + "mov x6, #0", + "mov x7, #0", + "mov x8, #0", + "mov x9, #0", + "mov x10, #0", + "mov x11, #0", + "mov x12, #0", + "mov x13, #0", + "mov x14, #0", + "mov x15, #0", + "mov x16, #0", + "mov x17, #0", + "mov x18, #0", + "mov x19, #0", + "mov x20, #0", + "mov x21, #0", + "mov x22, #0", + "mov x23, #0", + "mov x24, #0", + "mov x25, #0", + "mov x26, #0", + "mov x27, #0", + "mov x28, #0", + "mov x29, #0", + "mov x30, #0", + // Enter user mode + "eret", + options(noreturn), + ); +} +*/ diff --git a/kernel/kickstart/src/memory.rs b/kernel/kickstart/src/memory.rs new file mode 100644 index 000000000..32eca8e25 --- /dev/null +++ b/kernel/kickstart/src/memory.rs @@ -0,0 +1,236 @@ +// Boot allocator, Section mapping, memory permissions + +use { + crate::{boot_info::BOOT_INFO, loader::LoadableSection}, + libaddress::{PhysAddr, VirtAddr}, + liblocking::interface::Mutex, + libmapping::AttributeFields, +}; + +// Memory region translation. +// #[allow(dead_code)] +// #[derive(Copy, Clone)] +// pub enum Translation { +// /// One-to-one address mapping +// Identity, +// /// Mapping with a specified offset +// Offset(usize), +// } + +pub struct BootAllocator { + current: PhysAddr, + end: PhysAddr, +} + +#[derive(PartialEq, Copy, Clone)] +pub enum Alloc { + /// This allocation should be entered into mappings and stay around after `kickstart` finishes + Persistent, + /// This allocation will perish and be added to Untypeds after `kickstart` finishes + Droppable, +} + +impl BootAllocator { + pub fn new(start: PhysAddr, size: usize) -> Self { + Self { + current: start, + end: PhysAddr::new(start.as_u64() + size as u64), + } + } + + pub fn alloc_pages(&mut self, count: usize, usage: (&'static str, Alloc)) -> Option { + self.alloc_aligned(count * 4096, 4096, usage) + } + + pub fn alloc_aligned( + &mut self, + size: usize, + align: usize, + usage: (&'static str, Alloc), + ) -> Option { + let aligned = self.current.aligned_up(align as u64); + let new_current = PhysAddr::new(aligned.as_u64() + size as u64); + + #[cfg(feature = "qemu")] + libqemu::semi_println!( + "alloc_aligned {:#016x} => {:#016x} (wrt {:#016x})", + aligned.as_u64(), + new_current.as_u64(), + self.end.as_u64() + ); + + if !usage.0.is_empty() { + BOOT_INFO.lock(|bi| { + bi.insert_used_region( + aligned, + new_current, + AttributeFields { + droppable: usage.1 == Alloc::Droppable, + ..Default::default() // TODO: better RWX flags + }, + usage.0, + ); + }); + } + + if new_current > self.end { + return None; + } + self.current = new_current; + Some(aligned) + } + + pub fn current(&self) -> PhysAddr { + self.current + } + pub fn end(&self) -> PhysAddr { + self.end + } + pub fn remaining(&self) -> usize { + self.end - self.current + } +} + +/// Memory protection flags +#[derive(Debug, Clone, Copy)] +pub struct MemoryPermissions { + pub readable: bool, + pub writable: bool, + pub executable: bool, +} + +impl core::fmt::Display for MemoryPermissions { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + write!( + f, + "{}{}{}", + if self.readable { "R" } else { "-" }, + if self.writable { "W" } else { "-" }, + if self.executable { "X" } else { "-" } + ) + } +} + +impl MemoryPermissions { + /// Convert to `AArch64` page table flags + pub const fn as_pte_flags(self) -> u64 { + let mut flags = 0_u64; + + // Access flag (must be set) + flags |= 1 << 10; // AF + + // Shareability (inner shareable for normal memory) + flags |= 0b11 << 8; // SH + + // Access permissions + if !self.writable { + flags |= 0b10 << 6; // AP[2:1] = read-only + } + + // Execute never flags + if !self.executable { + flags |= 1 << 53; // PXN + flags |= 1 << 54; // UXN + } + + flags + } +} + +/// Layout of the loaded kernel in physical memory +#[derive(Debug)] +pub struct KernelLayout { + /// Physical address where kernel is loaded + pub phys_base: PhysAddr, + /// Virtual base address (higher-half) + pub virt_base: VirtAddr, + /// Total size of kernel in physical memory + pub total_size: usize, + /// Section metadata (for page table setup) + pub sections: &'static [LoadableSection], + /// BSS physical address (for zeroing) + pub bss_phys: PhysAddr, + /// BSS virtual address (for kernel mapping) + pub bss_virt: VirtAddr, + /// BSS size + pub bss_size: usize, + /// Stack virtual address (for kernel mapping) + pub stack_virt_bottom: VirtAddr, + /// Exception vector table virtual address (for `VBAR_EL1`) + pub vectors_virt: VirtAddr, +} + +impl KernelLayout { + pub fn virt_to_phys(&self, virt: VirtAddr) -> PhysAddr { + let offset = virt.as_u64() - self.virt_base.as_u64(); + PhysAddr::new(self.phys_base.as_u64() + offset) + } + + /// Get the `VBAR_EL1` value (virtual address for use after MMU enable) + /// + /// This is what the kernel would set `VBAR_EL1` to after switching to + /// higher-half addresses. + pub fn vbar_el1_virt(&self) -> u64 { + assert!( + self.vectors_virt.as_u64().trailing_zeros() >= 11, + "VBAR_EL1 address 0x{:016X} must be 2KB aligned", + self.vectors_virt.as_u64() + ); + self.vectors_virt.as_u64() + } + + pub fn iter_sections(&self) -> impl Iterator + '_ { + self.sections.iter().map(move |section| { + let offset = section.meta.offset_from_base(self.virt_base.as_u64()); + SectionMapping { + name: section.meta.name, + phys_start: PhysAddr::new(self.phys_base.as_u64() + offset), + virt_start: VirtAddr::new(section.meta.virt_addr), + size: section.meta.size, + permissions: section.meta.permissions, + } + }) + } + + pub fn bss_mapping(&self) -> Option { + (self.bss_size > 0).then_some(SectionMapping { + name: ".bss", + phys_start: self.bss_phys, + virt_start: self.bss_virt, + size: self.bss_size, + permissions: MemoryPermissions { + readable: true, + writable: true, + executable: false, + }, + }) + } +} + +#[derive(Debug, Clone, Copy)] +pub struct SectionMapping { + pub name: &'static str, + pub phys_start: PhysAddr, + pub virt_start: VirtAddr, + pub size: usize, + pub permissions: MemoryPermissions, +} + +impl SectionMapping { + pub fn page_count(&self) -> usize { + self.size.div_ceil(0x1000) + } + + pub fn pages(&self) -> impl Iterator { + let phys_start = self.phys_start.as_u64(); + let virt_start = self.virt_start.as_u64(); + let count = self.page_count(); + (0..count).map(move |i| { + let offset = (i * 0x1000) as u64; + ( + PhysAddr::new(phys_start + offset), + VirtAddr::new(virt_start + offset), + ) + }) + } +} diff --git a/kernel/kickstart/src/paging.rs b/kernel/kickstart/src/paging.rs new file mode 100644 index 000000000..25882445b --- /dev/null +++ b/kernel/kickstart/src/paging.rs @@ -0,0 +1,449 @@ +// kickstart/src/paging.rs - Page table management with section-aware mapping + +/// Kernel virtual address space layout (TTBR1 region: `0xFFFF_xxxx_xxxx_xxxx`) +/// +/// ``` +/// 0xFFFF_FFFF_FFFF_FFFF ┌─────────────────────┐ +/// │ Kernel stacks │ Per-CPU kernel stacks +/// 0xFFFF_FFFF_8000_0000 ├─────────────────────┤ +/// │ Kernel heap │ (if any - we try to avoid) +/// 0xFFFF_FFFF_0000_0000 ├─────────────────────┤ +/// │ DCB shared pages │ Read-only mapped to user too +/// 0xFFFF_FF00_0000_0000 ├─────────────────────┤ +/// │ Device MMIO │ 1:1 mapped device regions +/// 0xFFFF_0080_0000_0000 ├─────────────────────┤ +/// │ Physical memory │ Linear map of all RAM +/// │ (kernel direct) │ Kernel can access any phys through this offset +/// 0xFFFF_0000_0000_0000 └─────────────────────┘ +/// ``` +#[cfg(feature = "qemu")] +use libqemu::semi_println; +use { + crate::memory::{Alloc, BootAllocator, KernelLayout, MemoryPermissions, SectionMapping}, + core::ptr, + libaddress::{PhysAddr, VirtAddr}, +}; + +const KERNEL_BASE: u64 = 0xFFFF_8000_0000_0000; +const KERNEL_PHYS_MAP: u64 = 0xFFFF_8000_0000_0000; // Linear map base +const KERNEL_DEVICE_BASE: u64 = 0xFFFF_8080_0000_0000; +const KERNEL_DCB_BASE: u64 = 0xFFFF_FF00_0000_0000; +const KERNEL_HEAP_BASE: u64 = 0xFFFF_FFFF_0000_0000; +const KERNEL_STACK_BASE: u64 = 0xFFFF_FFFF_8000_0000; + +/// Page table entry flags for `AArch64` Stage 1 +pub mod flags { + pub const VALID: u64 = 1 << 0; + pub const TABLE: u64 = 1 << 1; + pub const PAGE: u64 = 1 << 1; + + // Access permissions (AP[2:1]) + pub const AP_RW_EL1: u64 = 0b00 << 6; + pub const AP_RO_EL1: u64 = 0b10 << 6; + + // Shareability + pub const SH_INNER: u64 = 0b11 << 8; + + // Access flag + pub const AF: u64 = 1 << 10; + + // Memory attributes index + pub const ATTR_NORMAL: u64 = 0 << 2; + pub const ATTR_DEVICE: u64 = 1 << 2; + + // Execute never + pub const PXN: u64 = 1 << 53; + pub const UXN: u64 = 1 << 54; + + // Block descriptor + pub const BLOCK: u64 = 0; +} + +/// Page table (512 entries × 8 bytes = 4KB) +#[repr(C, align(4096))] +pub struct PageTable { + entries: [u64; 512], +} + +impl PageTable { + pub const fn new() -> Self { + Self { entries: [0; 512] } + } +} + +/// Which TTBR to use +#[derive(Debug, Clone, Copy)] +pub enum Ttbr { + Ttbr0, + Ttbr1, +} + +/// MMU configuration builder +pub struct MmuSetup<'a> { + allocator: &'a mut BootAllocator, + ttbr0_l0: PhysAddr, + ttbr1_l0: PhysAddr, +} + +impl<'a> MmuSetup<'a> { + pub fn new(allocator: &'a mut BootAllocator) -> Result { + let ttbr0_l0 = allocator + .alloc_pages(1, ("user L0", Alloc::Droppable)) + .ok_or("Failed to allocate TTBR0 L0 table")?; + + let ttbr1_l0 = allocator + .alloc_pages(1, ("kernel L0", Alloc::Persistent)) + .ok_or("Failed to allocate TTBR1 L0 table")?; + + // SAFETY: Unsafe + unsafe { + ptr::write_bytes(ttbr0_l0.as_mut_ptr::(), 0, 4096); + ptr::write_bytes(ttbr1_l0.as_mut_ptr::(), 0, 4096); + } + + Ok(Self { + allocator, + ttbr0_l0, + ttbr1_l0, + }) + } + + pub fn memory_top(&self) -> PhysAddr { + self.allocator.current() + } + + /// Map a 4KB page with specific permissions + pub fn map_page( + &mut self, + ttbr: Ttbr, + virt: VirtAddr, + phys: PhysAddr, + perms: MemoryPermissions, + usage: (&'static str, Alloc), + ) -> Result<(), &'static str> { + let pte_flags = perms.as_pte_flags() | flags::ATTR_NORMAL; + self.map_page_with_flags(ttbr, virt, phys, pte_flags, perms, usage) + } + + /// Map a 4KB page with raw PTE flags + fn map_page_with_flags( + &mut self, + ttbr: Ttbr, + virt: VirtAddr, + phys: PhysAddr, + pte_flags: u64, + perms: MemoryPermissions, // Only for Display + usage: (&'static str, Alloc), + ) -> Result<(), &'static str> { + let l0_phys = match ttbr { + Ttbr::Ttbr0 => self.ttbr0_l0, + Ttbr::Ttbr1 => self.ttbr1_l0, + }; + + let va = virt.as_u64(); + let l0_idx = ((va >> 39) & 0x1FF) as usize; + let l1_idx = ((va >> 30) & 0x1FF) as usize; + let l2_idx = ((va >> 21) & 0x1FF) as usize; + let l3_idx = ((va >> 12) & 0x1FF) as usize; + + let l1_phys = self.ensure_table(l0_phys, l0_idx, usage)?; + let l2_phys = self.ensure_table(l1_phys, l1_idx, usage)?; + let l3_phys = self.ensure_table(l2_phys, l2_idx, usage)?; + + // SAFETY: Unsafe + let l3_table = unsafe { &mut *(l3_phys.as_mut_ptr::()) }; + l3_table.entries[l3_idx] = phys.as_u64() | flags::VALID | flags::PAGE | pte_flags; + + #[cfg(feature = "qemu")] + semi_println!( + "Mapped 4K page {} frame {} in {} with {}", + virt, + phys, + match ttbr { + Ttbr::Ttbr0 => "TTBR0(user)", + Ttbr::Ttbr1 => "TTBR1(kernel)", + }, + perms + ); + + Ok(()) + } + + /// Map a 2MB block with specific permissions + pub fn map_block_2mb( + &mut self, + ttbr: Ttbr, + virt: VirtAddr, + phys: PhysAddr, + perms: MemoryPermissions, + usage: (&'static str, Alloc), + ) -> Result<(), &'static str> { + if virt.as_u64() & 0x1F_FFFF != 0 || phys.as_u64() & 0x1F_FFFF != 0 { + return Err("2MB block mapping requires 2MB alignment"); + } + + let pte_flags = perms.as_pte_flags() | flags::ATTR_NORMAL; + + let l0_phys = match ttbr { + Ttbr::Ttbr0 => self.ttbr0_l0, + Ttbr::Ttbr1 => self.ttbr1_l0, + }; + + let va = virt.as_u64(); + let l0_idx = ((va >> 39) & 0x1FF) as usize; + let l1_idx = ((va >> 30) & 0x1FF) as usize; + let l2_idx = ((va >> 21) & 0x1FF) as usize; + + let l1_phys = self.ensure_table(l0_phys, l0_idx, usage)?; + let l2_phys = self.ensure_table(l1_phys, l1_idx, usage)?; + + // SAFETY: Unsafe + let l2_table = unsafe { &mut *(l2_phys.as_mut_ptr::()) }; + l2_table.entries[l2_idx] = phys.as_u64() | flags::VALID | flags::BLOCK | pte_flags; + + #[cfg(feature = "qemu")] + semi_println!( + "Mapped 2M page {} frame {} in {} with {}", + virt, + phys, + match ttbr { + Ttbr::Ttbr0 => "TTBR0(user)", + Ttbr::Ttbr1 => "TTBR1(kernel)", + }, + perms + ); + + Ok(()) + } + + fn ensure_table( + &mut self, + table_phys: PhysAddr, + index: usize, + usage: (&'static str, Alloc), + ) -> Result { + // SAFETY: Unsafe + let table = unsafe { &mut *(table_phys.as_mut_ptr::()) }; + let entry = table.entries[index]; + + if entry & flags::VALID != 0 { + Ok(PhysAddr::new(entry & 0x0000_FFFF_FFFF_F000)) + } else { + let new_table = self + .allocator + .alloc_pages(1, usage) + .ok_or("Failed to allocate page table")?; + + // SAFETY: Unsafe + unsafe { + ptr::write_bytes(new_table.as_mut_ptr::(), 0, 4096); + } + + table.entries[index] = new_table.as_u64() | flags::VALID | flags::TABLE; + Ok(new_table) + } + } + + pub fn ttbr0(&self) -> u64 { + self.ttbr0_l0.as_u64() + } + + pub fn ttbr1(&self) -> u64 { + self.ttbr1_l0.as_u64() + } +} + +/// Create identity mapping for `kickstart` +pub fn create_identity_mapping( + setup: &mut MmuSetup, + start: PhysAddr, + end: PhysAddr, +) -> Result<(), &'static str> { + let start_aligned = start.aligned_down(2_u64 * 1024 * 1024); + let end_aligned = end.aligned_up(2_u64 * 1024 * 1024); + + let perms = MemoryPermissions { + readable: true, + writable: true, + executable: true, + }; // for init code -- FIXME: not necessarily! + + let mut addr = start_aligned; + while addr.as_u64() < end_aligned.as_u64() { + setup.map_block_2mb( + Ttbr::Ttbr0, + VirtAddr::new(addr.as_u64()), + addr, + perms, + ("Kickstart identity mapping", Alloc::Droppable), + )?; + addr = PhysAddr::new(addr.as_u64() + 2 * 1024 * 1024); + } + + Ok(()) +} + +/// Create higher-half mapping for kernel with proper per-section permissions +pub fn create_kernel_mapping( + setup: &mut MmuSetup, + layout: &KernelLayout, + max_ram_bytes: u64, + el1_stack: u64, + el1_stack_size: usize, +) -> Result<(u64,), &'static str> { + // Map each section with its specific permissions + for section in layout.iter_sections() { + map_section(setup, §ion)?; + } + + // Map BSS section + if let Some(bss) = layout.bss_mapping() { + map_section(setup, &bss)?; + } + + // ───────────────────────────────────────────────────────────────── + // Setup linear physical map (all RAM accessible to nucleus) + // TODO: exclude physical memory that covers the kernel image itself! + // ───────────────────────────────────────────────────────────────── + + let perms = MemoryPermissions { + readable: true, + writable: true, + executable: false, + }; + + // Map all physical memory using 2MB blocks with a specific offset + // Kernel mapping for phys memory starts at + for i in 0..max_ram_bytes.div_ceil(2 * 1024 * 1024) { + setup.map_block_2mb( + Ttbr::Ttbr1, + VirtAddr::new(libaddress::PHYSICAL_KERNEL_WINDOW + i * 2 * 1024 * 1024), + PhysAddr::new(i * 2 * 1024 * 1024), + perms, + ("Nucleus phys memory mapping", Alloc::Persistent), + ); + } + + // Map kernel stack + let stack_bottom = layout.stack_virt_bottom; + + for i in 0..el1_stack_size.div_ceil(4 * 1024) as u64 { + setup.map_page( + Ttbr::Ttbr1, + VirtAddr::new(stack_bottom.as_u64() + i * 4 * 1024), + PhysAddr::new(el1_stack + i * 4 * 1024), + perms, + ("Nucleus stack mapping", Alloc::Persistent), + ); + } + + let stack_virt_top = stack_bottom + el1_stack_size; + + // ───────────────────────────────────────────────────────────────── + // Setup device MMIO mappings + // ───────────────────────────────────────────────────────────────── + // RPi4 peripherals at 0xFE00_0000 - 0xFF00_0000 + // let device_base_phys = 0xFE00_0000_u64; + // Map as device memory (non-cacheable, no speculation) + // create_device_mapping(setup, device_base_phys, VIRT_MMIO, 0x100_0000); // 16MiB + + Ok((stack_virt_top.into(),)) +} + +/// Map a single section with proper permissions +fn map_section(setup: &mut MmuSetup, section: &SectionMapping) -> Result<(), &'static str> { + if !section.phys_start.is_aligned(4096_u64) { + #[cfg(feature = "qemu")] + semi_println!("!! Section {} not aligned to 4K boundary!", section.name); + return Err("Section not aligned"); + } + + // Check if we can use 2MB blocks (section must be 2MB aligned and sized) + let can_use_2mb = section.phys_start.is_aligned(2_u64 * 1024 * 1024) + && section.virt_start.as_u64().is_multiple_of(2 * 1024 * 1024) + && section.size >= 2 * 1024 * 1024; + + if can_use_2mb { + // Use 2MB blocks for large aligned sections + let mut phys = section.phys_start; + let mut virt = section.virt_start; + let mut remaining = section.size; + + while remaining >= 2 * 1024 * 1024 { + setup.map_block_2mb( + Ttbr::Ttbr1, + virt, + phys, + section.permissions, + ("Nucleus section mapping", Alloc::Persistent), + )?; + phys = PhysAddr::new(phys.as_u64() + 2 * 1024 * 1024); + virt = VirtAddr::new(virt.as_u64() + 2 * 1024 * 1024); + remaining -= 2 * 1024 * 1024; + } + + // Map remaining pages + while remaining > 0 { + setup.map_page( + Ttbr::Ttbr1, + virt, + phys, + section.permissions, + ("Nucleus section mapping", Alloc::Persistent), + )?; + phys = PhysAddr::new(phys.as_u64() + 0x1000); + virt = VirtAddr::new(virt.as_u64() + 0x1000); + remaining = remaining.saturating_sub(0x1000); + } + } else { + // Use 4KB pages + for (phys, virt) in section.pages() { + setup.map_page( + Ttbr::Ttbr1, + virt, + phys, + section.permissions, + ("Nucleus section mapping", Alloc::Persistent), + )?; + } + } + + Ok(()) +} + +/// Create device memory mapping +pub fn create_device_mapping( + setup: &mut MmuSetup, + phys: PhysAddr, + virt: VirtAddr, + size: usize, +) -> Result<(), &'static str> { + let pages = size.div_ceil(0x1000); + let perms = MemoryPermissions { + readable: true, + writable: true, + executable: false, + }; + + for i in 0..pages { + let offset = (i * 0x1000) as u64; + + // let _l0_phys = setup.ttbr1_l0; + let va = virt.as_u64() + offset; + let pa = phys.as_u64() + offset; + + // Use device memory attributes + let pte_flags = perms.as_pte_flags() | flags::ATTR_DEVICE; + // PageFlags::KERNEL_RW | PageFlags::DEVICE_nGnRnE, + setup.map_page_with_flags( + Ttbr::Ttbr1, + VirtAddr::new(va), + PhysAddr::new(pa), + pte_flags, + perms, + ("Nucleus device mapping", Alloc::Persistent), + )?; + } + + Ok(()) +} diff --git a/kernel/kickstart/src/qsort.rs b/kernel/kickstart/src/qsort.rs new file mode 100644 index 000000000..e2386d79a --- /dev/null +++ b/kernel/kickstart/src/qsort.rs @@ -0,0 +1,42 @@ +use core::cmp::Ordering; + +// In-place quicksort from https://github.com/jlkiri/rust-sorting-algorithms/blob/master/src/quick.rs +fn partition( + array: &mut [T], + l: isize, + h: isize, + compare: fn(&T, &T) -> Ordering, +) -> isize { + let pivot = array[h.cast_unsigned()]; + let mut i = l - 1; // Index of the smaller element + + for j in l..h { + if compare(&array[j.cast_unsigned()], &pivot) != Ordering::Greater { + i += 1; + array.swap(i.cast_unsigned(), j.cast_unsigned()); + } + } + + array.swap((i + 1).cast_unsigned(), h.cast_unsigned()); + + i + 1 +} + +fn quick_sort_partition( + array: &mut [T], + start: isize, + end: isize, + compare: fn(&T, &T) -> Ordering, +) { + if start < end && end - start >= 1 { + let pivot = partition(array, start, end, compare); + quick_sort_partition(array, start, pivot - 1, compare); + quick_sort_partition(array, pivot + 1, end, compare); + } +} + +pub fn sort(array: &mut [T], compare: fn(&T, &T) -> Ordering) { + let start = 0; + let end = array.len() - 1; + quick_sort_partition(array, start, end.cast_signed(), compare); +} diff --git a/nucleus/Cargo.toml b/kernel/nucleus/Cargo.toml similarity index 73% rename from nucleus/Cargo.toml rename to kernel/nucleus/Cargo.toml index 7f8f63312..42016b95a 100644 --- a/nucleus/Cargo.toml +++ b/kernel/nucleus/Cargo.toml @@ -1,6 +1,8 @@ [package] name = "nucleus" -description = "Vesper nanokernel binary" + +description = "Vesper nanokernel privileged nucleus binary." + authors = { workspace = true } categories = { workspace = true } documentation = { workspace = true } @@ -20,36 +22,30 @@ default = [] # Enable JTAG debugging of kernel - enable jtag helpers and # block waiting for JTAG probe attach at the start of kernel main. jtag = ["libmachine/jtag"] -noserial = ["libplatform/noserial"] +noserial = [] #"libplatform/noserial" # Build for running under QEMU with semihosting, so various halt/reboot options would for example quit QEMU instead. qemu = [ "dep:libqemu", - "libconsole/qemu", + # "libconsole/qemu", "libexception/qemu", "libmachine/qemu", - "libplatform/qemu", + # "libplatform/qemu", ] [dependencies] aarch64-cpu = { workspace = true } -bit_field = { workspace = true } bitflags = { workspace = true } cfg-if = { workspace = true } -libboot = { workspace = true } -libconsole = { workspace = true } +libaddress = { workspace = true } libcpu = { workspace = true } libexception = { workspace = true } -libkernel-state = { workspace = true } +liblocking = { workspace = true } liblog = { workspace = true } libmachine = { workspace = true } -libmemory = { workspace = true } -libplatform = { workspace = true } +libmapping = { workspace = true } +libobject = { workspace = true } +libprint = { workspace = true } libqemu = { workspace = true, optional = true } -libtime = { workspace = true } -snafu = { workspace = true } -tock-registers = { workspace = true } -usize_conversions = { workspace = true } -ux = { workspace = true } [dev-dependencies] libexception = { workspace = true, features = ["test_build"] } @@ -57,6 +53,9 @@ liblocking = { workspace = true } libqemu = { workspace = true } libtest = { workspace = true } +[build-dependencies] +build-rs = { workspace = true } + [[bin]] name = "nucleus" test = false # test are all concentrated in integration tests/ diff --git a/nucleus/build.rs b/kernel/nucleus/build.rs similarity index 79% rename from nucleus/build.rs rename to kernel/nucleus/build.rs index b3aafde7a..5030da575 100644 --- a/nucleus/build.rs +++ b/kernel/nucleus/build.rs @@ -1,6 +1,6 @@ //! This build script is used to link main kernel binary. -const LINKER_SCRIPT: &str = "libs/platform/src/platform/raspberrypi/linker/kernel.ld"; +const LINKER_SCRIPT: &str = "libs/platform/src/raspberrypi/linker/nucleus.ld"; const LINKER_SCRIPT_AUX: &str = "libs/exception/src/arch/aarch64/linker/aarch64-exceptions.ld"; fn main() { diff --git a/kernel/nucleus/design.md b/kernel/nucleus/design.md new file mode 100644 index 000000000..28ac132ce --- /dev/null +++ b/kernel/nucleus/design.md @@ -0,0 +1,23 @@ +Key Design Points: + +Aspect | Design Choice | Rationale +Type numbering | Core 0-15, Arch 16-63 | Clear separation, room for growth +Arch trait | Associated types | Compile-time type safety per arch +Pools | Separate core + arch | Different lifecycles, easier to reason about +Frame sizes | Enum, not raw bits | Type-safe, arch-specific validation +VSpace | Wraps root PT + ASID | Clean abstraction for address space +Dispatch | Single match on ObjectType | Uniform handling, arch types get own handlers + + +Kernel API surface: + +Wait options (open wait, like servers waiting; closed wait, like waiting one client's response specifically) +Timeouts for send and recv phases separately + +Structure: + +Key - a capability index in the keytable - this key is per-domain because it indexes a local domain table +KeyEntry - a capability "value" - entry in the keytable, capabilities are same-size values carrying type, + pointer to kernel object and access rights. +A kernel object corresponding to the capability is allocated in the object pool. +Untyped does not have an "object", it points to memory region to allocate from, and amount of allocations already done. diff --git a/kernel/nucleus/src/api/arch/asid.rs b/kernel/nucleus/src/api/arch/asid.rs new file mode 100644 index 000000000..58d302773 --- /dev/null +++ b/kernel/nucleus/src/api/arch/asid.rs @@ -0,0 +1,9 @@ +pub fn invoke( + asid: &mut A::ASID, + rights: Rights, + op: u32, + args: &[u64; 6], +) -> Result<(u64, u64), CapError> { + // ASIDs mostly just exist; operations are minimal + todo!("asid invoke") +} diff --git a/kernel/nucleus/src/api/arch/asid_pool.rs b/kernel/nucleus/src/api/arch/asid_pool.rs new file mode 100644 index 000000000..fbc8a70b9 --- /dev/null +++ b/kernel/nucleus/src/api/arch/asid_pool.rs @@ -0,0 +1,9 @@ +pub fn invoke( + pool: &mut A::ASIDPool, + rights: Rights, + op: u32, + args: &[u64; 6], + kernel: &mut Kernel, +) -> Result<(u64, u64), CapError> { + todo!("asid_pool invoke") +} diff --git a/kernel/nucleus/src/api/arch/frame.rs b/kernel/nucleus/src/api/arch/frame.rs new file mode 100644 index 000000000..74ba565f6 --- /dev/null +++ b/kernel/nucleus/src/api/arch/frame.rs @@ -0,0 +1,69 @@ +use { + crate::{ + api::key_entry::KeyEntry, + objects::{ArchObjects, nucleus::Nucleus}, + }, + libobject::{CapError, Rights, arch::frame::FrameOp}, +}; + +/// Invoke a frame capability operation. +/// +/// The frame data is stored inline in the KeyEntry as a RegionPayload. +/// Each frame cap tracks its own single mapping (seL4-style). +/// To map the same physical frame at two addresses, duplicate the cap first. +pub fn invoke( + entry: &mut KeyEntry, + op: u32, + args: &[u64; 6], + nucleus: &mut Nucleus, +) -> Result<(u64, u64), CapError> { + let op = FrameOp::try_from(op as u8).map_err(|_| CapError::InvalidOperation)?; + let rights = entry.rights(); + + match op { + FrameOp::Map => { + // args[0] = vspace_slot + // args[1] = virt_addr + // args[2] = rights (R/W/X bits) + // args[3] = attrs (cacheability, etc.) + + if !rights.contains(Rights::READ) { + return Err(CapError::InsufficientRights); + } + + let frame = entry.as_frame_mut()?; + if frame.is_mapped() { + return Err(CapError::AlreadyMapped); + } + + let vaddr = args[1]; + // ... perform the mapping via arch-specific page table code ... + frame.set_mapped(vaddr); + + Ok((0, 0)) + } + + FrameOp::Unmap => { + let frame = entry.as_frame_mut()?; + if !frame.is_mapped() { + return Err(CapError::NotMapped); + } + // ... perform the unmapping using frame.mapped_vaddr() ... + frame.clear_mapped(); + Ok((0, 0)) + } + + FrameOp::GetAddress => { + if !rights.contains(Rights::GRANT) { + return Err(CapError::InsufficientRights); + } + let frame = entry.as_frame()?; + Ok((frame.paddr, frame.size() as u64)) + } + + FrameOp::Remap => { + // Change attributes on existing mapping + todo!("frame remap"); + } + } +} diff --git a/kernel/nucleus/src/api/arch/mod.rs b/kernel/nucleus/src/api/arch/mod.rs new file mode 100644 index 000000000..870cc1ec5 --- /dev/null +++ b/kernel/nucleus/src/api/arch/mod.rs @@ -0,0 +1 @@ +pub mod frame; diff --git a/kernel/nucleus/src/api/arch/page_table.rs b/kernel/nucleus/src/api/arch/page_table.rs new file mode 100644 index 000000000..609821852 --- /dev/null +++ b/kernel/nucleus/src/api/arch/page_table.rs @@ -0,0 +1,23 @@ +pub fn invoke( + pt: &mut A::PageTable, + rights: Rights, + op: u32, + args: &[u64; 6], + kernel: &mut Kernel, +) -> Result<(u64, u64), CapError> { + match op { + PageTableOp::Map => { + // args[0] = vspace_slot + // args[1] = virt_addr (determines which slot in parent) + let vspace_slot = KeySlot(args[0] as u16); + let virt_addr = VirtAddr::new(args[1]); + + // ... mapping logic + todo!("page_table map") + } + + PageTableOp::Unmap => { + todo!("page_table unmap") + } + } +} diff --git a/kernel/nucleus/src/api/arch/vspace.rs b/kernel/nucleus/src/api/arch/vspace.rs new file mode 100644 index 000000000..790b8f566 --- /dev/null +++ b/kernel/nucleus/src/api/arch/vspace.rs @@ -0,0 +1,36 @@ +pub fn invoke( + vspace: &mut A::VSpace, + rights: Rights, + op: u32, + args: &[u64; 6], + kernel: &mut Kernel, +) -> Result<(u64, u64), CapError> { + let op = VSpaceOp::try_from(op as u8).map_err(|_| CapError::InvalidOperation)?; + + match op { + VSpaceOp::SetRoot => { + // args[0] = page_table_slot + todo!("vspace set_root") + } + VSpaceOp::AssignASID => { + // args[0] = asid_pool_slot + let pool_slot = KeySlot(args[0] as u16); + + let domain = kernel.current_domain_mut()?; + let pool_entry = domain.keytable.lookup_mut(pool_slot)?; + let pool = pool_entry.as_object_mut::()?; + + let asid = pool.allocate().ok_or(CapError::ASIDPoolExhausted)?; + + vspace.asid = Some(asid); + Ok((asid as u64, 0)) + } + VSpaceOp::Activate => { + todo!("vspace activate") + } + VSpaceOp::GetASID => { + let asid = vspace.asid.ok_or(CapError::NoASIDAssigned)?; + Ok((asid as u64, 0)) + } + } +} diff --git a/kernel/nucleus/src/api/buffer.rs b/kernel/nucleus/src/api/buffer.rs new file mode 100644 index 000000000..9094cee99 --- /dev/null +++ b/kernel/nucleus/src/api/buffer.rs @@ -0,0 +1,106 @@ +// ===================== +// == Syscall handler == +// ===================== + +#[inline] +pub fn invoke(cap: &KeyEntry, op: u32, arg0: u64, arg1: u64) -> SyscallResult { + let buffer = cap.as_buffer()?; + let caller = current_domain(); + + match BufferOp::try_from(op)? { + BufferOp::Map => { + // Check MAP right + if !cap.rights.contains(Rights::MAP) { + return Err(SyscallError::PermissionDenied); + } + + // Check not already mapped by this domain + if buffer.is_mapped_by(caller.id) { + return Err(SyscallError::AlreadyMapped); + } + + let hint = if arg0 != 0 { + Some(VirtAddr::new(arg0)) + } else { + None + }; + let flags = MapFlags::from_bits_truncate(arg1 as u32); + + // Allocate virtual address range + let vaddr = caller.address_space.allocate_range( + hint, + buffer.size, + flags.contains(MapFlags::FIXED), + )?; + + // Compute page table permissions from cap rights + let pte_flags = cap.rights.to_pte_flags() | buffer.flags.to_pte_flags(); + + // Install page table mappings + for offset in (0..buffer.size).step_by(PAGE_SIZE) { + let paddr = buffer.phys_base + offset; + let vaddr_page = vaddr + offset; + + caller + .address_space + .map_page(vaddr_page, paddr, pte_flags)?; + } + + // Record mapping for revocation + buffer.mappings.push(Mapping { + domain_id: caller.id, + virt_addr: vaddr, + permissions: cap.rights, + }); + + Ok(vaddr.as_u64()) + } + + BufferOp::Unmap => { + // Find and remove mapping for this domain + let mapping = buffer + .mappings + .iter() + .position(|m| m.domain_id == caller.id) + .ok_or(SyscallError::NotMapped)?; + + let mapping = buffer.mappings.remove(mapping); + + // Remove page table entries + for offset in (0..buffer.size).step_by(PAGE_SIZE) { + caller + .address_space + .unmap_page(mapping.virt_addr + offset)?; + } + + // TLB invalidation (in single-address-space, this is local) + tlb_invalidate_range(mapping.virt_addr, buffer.size); + + Ok(0) + } + + BufferOp::Query => { + let info_ptr = arg0 as *mut BufferInfo; + + // Validate user pointer + if !caller.address_space.is_valid_user_ptr(info_ptr) { + return Err(SyscallError::InvalidPointer); + } + + let mapping = buffer.mappings.iter().find(|m| m.domain_id == caller.id); + + let info = BufferInfo { + size: buffer.size, + flags: buffer.flags, + is_mapped: mapping.is_some(), + mapped_addr: mapping.map(|m| m.virt_addr), + }; + + unsafe { + info_ptr.write(info); + } + + Ok(0) + } + } +} diff --git a/kernel/nucleus/src/api/debug_console.rs b/kernel/nucleus/src/api/debug_console.rs new file mode 100644 index 000000000..2e4f59335 --- /dev/null +++ b/kernel/nucleus/src/api/debug_console.rs @@ -0,0 +1,27 @@ +use { + crate::{api::KeyEntry, objects::DebugConsole}, + libaddress::PhysAddr, + libobject::{CapError, Key, SyscallResult, debug_console::DebugConsoleOp}, +}; + +// ===================== +// == Syscall handler == +// ===================== + +#[inline] +pub fn invoke(cap: &mut KeyEntry, op: u32, arg0: u64, arg1: u64) -> SyscallResult { + let console = cap.as_object_mut::()?; + let op = DebugConsoleOp::try_from(op).map_err(|_unused| CapError::InvalidOperation)?; + + #[cfg(feature = "qemu")] + libqemu::semi_println!("DebugConsole:invoke"); + + match op { + // DebugConsoleOp::Write => console.handle_write(arg0, arg1), + DebugConsoleOp::Write => { + crate::objects::debug_console::DebugConsole::handle_write(PhysAddr::new(arg0), arg1)?; + Ok((0, 0)) + } + _ => Err(CapError::InvalidOperation), + } +} diff --git a/kernel/nucleus/src/api/domain.rs b/kernel/nucleus/src/api/domain.rs new file mode 100644 index 000000000..4620ad848 --- /dev/null +++ b/kernel/nucleus/src/api/domain.rs @@ -0,0 +1,22 @@ +// ===================== +// == Syscall handler == +// ===================== + +pub fn invoke(cap: &Cap, op: u32, arg0: u64, arg1: u64) -> SyscallResult { + let domain = cap.as_domain()?; + match op { + DomainOp::Activate => { + // Make domain runnable (usually combined with TimeCap donation) + domain.activate() + } + DomainOp::Grant => { + // Grant a capability to this domain's cspace + let src_slot = arg0 as CapSlot; + let dest_slot = arg1 as CapSlot; + domain.grant_cap(src_slot, dest_slot) + } + DomainOp::Suspend => domain.suspend(), + DomainOp::Resume => domain.resume(), + _ => Err(SyscallError::InvalidOp), + } +} diff --git a/kernel/nucleus/src/api/endpoint.rs b/kernel/nucleus/src/api/endpoint.rs new file mode 100644 index 000000000..631abf508 --- /dev/null +++ b/kernel/nucleus/src/api/endpoint.rs @@ -0,0 +1,52 @@ +// Client Domain Server Domain +// ───────────── ───────────── +// │ │ +// │ ep.call(msg) │ ep.recv() +// │ │ +// ▼ ▼ +// ┌─────────┐ ┌─────────┐ +// │ BLOCKED │ │ BLOCKED │ +// │ waiting │ │ waiting │ +// │ for │ │ for │ +// │ reply │ │ sender │ +// └────┬────┘ └────┬────┘ +// │ │ +// │ ┌───────────────────┐ │ +// │ │ KERNEL SWITCH │ │ +// ├───────►│ │◄────────────┤ +// │ │ 1. Copy msg regs │ │ +// │ │ 2. Set badge │ │ +// │ │ 3. Switch domain │ │ +// │ └───────────────────┘ │ +// │ │ +// │ ▼ +// │ (running) +// │ process msg +// │ │ +// │ ┌───────────────────┐ │ +// │ │ KERNEL SWITCH │ │ +// │◄───────│ │◄────────────┤ +// │ │ 1. Copy reply │ │ ep.reply(response) +// │ │ 2. Switch back │ │ +// │ └───────────────────┘ │ +// ▼ +// (running) +// got reply + +// ===================== +// == Syscall handler == +// ===================== + +#[inline] +pub fn invoke(cap: &Cap, op: u32, arg0: u64, arg1: u64) -> SyscallResult { + let ep = cap.as_endpoint()?; + match op { + EndpointOp::Call => ep.handle_call(), + EndpointOp::Send => ep.handle_send(), + EndpointOp::Recv => ep.handle_recv(), + EndpointOp::Reply => { /* invoke the reply_cap */ } + EndpointOp::ReplyRecv => ep.handle_replyrecv(), + EndpointOp::Forward => ep.handle_forward(), + _ => Err(SyscallError::InvalidOp), + } +} diff --git a/kernel/nucleus/src/api/event_count.rs b/kernel/nucleus/src/api/event_count.rs new file mode 100644 index 000000000..2a73ba244 --- /dev/null +++ b/kernel/nucleus/src/api/event_count.rs @@ -0,0 +1,19 @@ +// ===================== +// == Syscall handler == +// ===================== + +pub fn invoke(cap: &Cap, op: u32, arg0: u64) -> SyscallResult { + let ec = cap.as_event_count()?; + match op { + EventOp::Advance => { + Ok(ec.advance(arg0)) // atomic ADD, returns new value + } + EventOp::Await => { + Ok(ec.await_ge(arg0)) // blocks until >= arg0 + } + EventOp::Read => { + Ok(ec.read()) // non-blocking read + } + _ => Err(SyscallError::InvalidOp), + } +} diff --git a/kernel/nucleus/src/api/key_entry.rs b/kernel/nucleus/src/api/key_entry.rs new file mode 100644 index 000000000..65f86de7d --- /dev/null +++ b/kernel/nucleus/src/api/key_entry.rs @@ -0,0 +1,405 @@ +// ═══════════════════════════════════════════════════════════════════ +// KEY ENTRY (CAPABILITY TABLE ENTRY) +// ═══════════════════════════════════════════════════════════════════ +// +// Tagged union: most types store a pointer to a pool-allocated object, +// but region types (Untyped, Frame) store metadata inline — the +// capability IS the object, no indirection needed. +// Revocation tree is external (userspace CapManager, Composite-style). +// +// ┌──────────────────────────────────────────────┐ +// │ Common header — 4 bytes │ +// │ obj_type: ObjectType (1 byte) │ +// │ rights: Rights (1 byte) │ +// │ badge: u16 (2 bytes) │ +// ├──────────────────────────────────────────────┤ +// │ Payload — 16 bytes (union on obj_type) │ +// │ │ +// │ VARIANT A: Pointer-based (most types) │ +// │ ptr: NonNull<()> (8 bytes) │ +// │ generation: u32 (4 bytes) │ +// │ _pad: u32 (4 bytes) │ +// │ │ +// │ VARIANT B: Inline Region (Untyped, Frame) │ +// │ paddr: u64 (8 bytes) │ +// │ state: u32 (4 bytes) │ +// │ Untyped → watermark (>> MIN_ALIGN_BITS) │ +// │ Frame → map_count (low 16 bits) │ +// │ size_bits: u8 (1 byte) │ +// │ is_device: bool (1 byte) │ +// │ _pad: u16 (2 bytes) │ +// │ │ +// │ VARIANT C: Null │ +// │ (all zeros) │ +// └──────────────────────────────────────────────┘ +// Total: 20 bytes used, 32-byte aligned slot + +use { + crate::objects::{NucleusObject, object_ref::ObjectRef}, + core::ptr::NonNull, + libobject::{CapError, ObjectType, Rights}, +}; + +/// Payload for pointer-based capabilities (most object types). +#[repr(C)] +#[derive(Clone, Copy)] +struct ObjectPayload { + ptr: NonNull<()>, + generation: u32, + _pad: u32, +} + +/// Payload for inline region capabilities (Untyped, Frame). +/// No indirection — the capability IS the object. +/// +/// The `state` field is dual-use: +/// - **Untyped**: watermark (next free byte offset, shifted right by `MIN_ALIGN_BITS`) +/// - **Frame**: mapped virtual address >> 12 (0 = unmapped). +/// Each frame cap copy tracks its own single mapping (seL4-style). +/// To map the same physical frame twice, duplicate the cap first. +#[repr(C)] +#[derive(Clone, Copy)] +pub struct RegionPayload { + /// Physical base address of the region + pub paddr: u64, + /// Dual-use state field (see type docs) + pub state: u32, + /// Size as log2 (region = `2^size_bits`) + pub size_bits: u8, + /// Is this device memory (not normal RAM)? + pub is_device: bool, + pub _pad: u16, +} + +/// 16-byte payload union, discriminated by `obj_type` in the header. +#[repr(C)] +union KeyPayload { + obj: ObjectPayload, + region: RegionPayload, + null: [u8; 16], +} + +/// A single entry in a domain's capability table (`KeyTable`). +/// +/// 20 bytes used in a 32-byte aligned slot. +/// Discriminated union: `obj_type` selects the payload variant. +#[repr(C, align(32))] +pub struct KeyEntry { + obj_type: ObjectType, + rights: Rights, + badge: u16, + payload: KeyPayload, +} + +// Verify sizes at compile time +const _: () = assert!(core::mem::size_of::() == 32); // same as seL4 +const _: () = assert!(core::mem::size_of::() == 16); +const _: () = assert!(core::mem::size_of::() == 16); + +/// Minimum alignment bits for watermark shift (16-byte alignment). +const MIN_ALIGN_BITS: u32 = 4; + +impl KeyEntry { + /// Create a null/empty entry. + pub const fn null() -> Self { + Self { + obj_type: ObjectType::NULL, + rights: Rights::empty(), + badge: 0, + payload: KeyPayload { null: [0_u8; 16] }, + } + } + + /// Create a pointer-based capability entry (most object types). + pub fn new(object: &T, rights: Rights, badge: u16) -> Self { + Self { + obj_type: T::TYPE, + rights, + badge, + payload: KeyPayload { + obj: ObjectPayload { + ptr: NonNull::from(object).cast(), + generation: 0, + _pad: 0, + }, + }, + } + } + + /// Create a capability entry from a pre-built `ObjectRef` (for arch objects). + pub fn from_ref(obj_ref: ObjectRef, rights: Rights, badge: u16) -> Self { + Self { + obj_type: obj_ref.object_type(), + rights, + badge, + payload: KeyPayload { + obj: ObjectPayload { + ptr: obj_ref.as_raw_ptr(), + generation: 0, + _pad: 0, + }, + }, + } + } + + /// Create an inline Untyped capability (no pool allocation). + pub fn new_untyped(paddr: u64, size_bits: u8, is_device: bool, rights: Rights) -> Self { + Self { + obj_type: ObjectType::UNTYPED, + rights, + badge: 0, + payload: KeyPayload { + region: RegionPayload { + paddr, + state: 0, // watermark starts at 0 + size_bits, + is_device, + _pad: 0, + }, + }, + } + } + + /// Create an inline Frame capability (no pool allocation). + pub fn new_frame(paddr: u64, size_bits: u8, is_device: bool, rights: Rights) -> Self { + Self { + obj_type: ObjectType::FRAME, + rights, + badge: 0, + payload: KeyPayload { + region: RegionPayload { + paddr, + state: 0, // map_count starts at 0 + size_bits, + is_device, + _pad: 0, + }, + }, + } + } + + /// Check if this entry is valid (not null). + #[inline] + pub fn is_valid(&self) -> bool { + self.obj_type != ObjectType::NULL + } + + /// Check if this is an inline region type (Untyped or Frame). + #[inline] + pub fn is_region(&self) -> bool { + self.obj_type == ObjectType::UNTYPED || self.obj_type == ObjectType::FRAME + } + + /// Get the object type. + #[inline] + pub fn object_type(&self) -> ObjectType { + self.obj_type + } + + /// Get access rights. + #[inline] + pub fn rights(&self) -> Rights { + self.rights + } + + /// Get badge value. + #[inline] + pub fn badge(&self) -> u16 { + self.badge + } + + /// Get generation counter (pointer-based caps only). + #[inline] + pub fn generation(&self) -> u32 { + debug_assert!(!self.is_region() && self.obj_type != ObjectType::NULL); + // SAFETY: We checked the object is valid and is of correct type. + unsafe { self.payload.obj.generation } + } + + /// Access the underlying object with type checking. + /// Returns error if called on a region type — use `as_region()` instead. + #[inline] + pub fn as_object(&self) -> Result<&T, CapError> { + if self.obj_type != T::TYPE || self.is_region() { + return Err(CapError::TypeMismatch { + expected: T::TYPE, + found: self.obj_type, + }); + } + // SAFETY: type verified, pointer-based variant guaranteed + Ok(unsafe { self.payload.obj.ptr.cast::().as_ref() }) + } + + /// Access the underlying object mutably with type checking. + /// Returns error if called on a region type — use `as_region_mut()` instead. + #[inline] + pub fn as_object_mut(&mut self) -> Result<&mut T, CapError> { + if self.obj_type != T::TYPE || self.is_region() { + return Err(CapError::TypeMismatch { + expected: T::TYPE, + found: self.obj_type, + }); + } + // SAFETY: type verified, pointer-based variant guaranteed + Ok(unsafe { self.payload.obj.ptr.cast::().as_mut() }) + } + + /// Access the inline region payload (Untyped or Frame, read-only). + #[inline] + pub fn as_region(&self) -> Result<&RegionPayload, CapError> { + if !self.is_region() { + return Err(CapError::TypeMismatch { + expected: ObjectType::UNTYPED, + found: self.obj_type, + }); + } + // SAFETY: We checked the object is valid and is of the right type. + Ok(unsafe { &self.payload.region }) + } + + /// Access the inline region payload (Untyped or Frame, mutable). + #[inline] + pub fn as_region_mut(&mut self) -> Result<&mut RegionPayload, CapError> { + if !self.is_region() { + return Err(CapError::TypeMismatch { + expected: ObjectType::UNTYPED, + found: self.obj_type, + }); + } + // SAFETY: We checked the object is valid and is of the right type. + Ok(unsafe { &mut self.payload.region }) + } + + /// Access the inline region payload, but only if this is an Untyped. + #[inline] + pub fn as_untyped(&self) -> Result<&RegionPayload, CapError> { + if self.obj_type != ObjectType::UNTYPED { + return Err(CapError::TypeMismatch { + expected: ObjectType::UNTYPED, + found: self.obj_type, + }); + } + // SAFETY: We checked the object is valid and is of the right type. + Ok(unsafe { &self.payload.region }) + } + + /// Access the inline region payload mutably, but only if this is an Untyped. + #[inline] + pub fn as_untyped_mut(&mut self) -> Result<&mut RegionPayload, CapError> { + if self.obj_type != ObjectType::UNTYPED { + return Err(CapError::TypeMismatch { + expected: ObjectType::UNTYPED, + found: self.obj_type, + }); + } + // SAFETY: We checked the object is valid and is of the right type. + Ok(unsafe { &mut self.payload.region }) + } + + /// Access the inline region payload, but only if this is a Frame. + #[inline] + pub fn as_frame(&self) -> Result<&RegionPayload, CapError> { + if self.obj_type != ObjectType::FRAME { + return Err(CapError::TypeMismatch { + expected: ObjectType::FRAME, + found: self.obj_type, + }); + } + // SAFETY: We checked the object is valid and is of the right type. + Ok(unsafe { &self.payload.region }) + } + + /// Access the inline region payload mutably, but only if this is a Frame. + #[inline] + pub fn as_frame_mut(&mut self) -> Result<&mut RegionPayload, CapError> { + if self.obj_type != ObjectType::FRAME { + return Err(CapError::TypeMismatch { + expected: ObjectType::FRAME, + found: self.obj_type, + }); + } + // SAFETY: We checked the object is valid and is of the right type. + Ok(unsafe { &mut self.payload.region }) + } +} + +// ═══════════════════════════════════════════════════════════════════ +// REGION PAYLOAD OPERATIONS +// ═══════════════════════════════════════════════════════════════════ + +impl RegionPayload { + // ── Common ── + + /// Get the total size of the region in bytes. + #[inline] + pub fn size(&self) -> usize { + 1_usize << self.size_bits + } + + /// Check if the state field is zero (no allocations / no mappings). + #[inline] + pub fn is_free(&self) -> bool { + self.state == 0 + } + + /// Reset the state field to zero. + #[inline] + pub fn reset(&mut self) { + self.state = 0; + } + + // ── Untyped-specific ── + + /// Get the watermark (next free byte offset) in bytes. + /// Only meaningful when this is an Untyped region. + #[inline] + pub fn watermark_bytes(&self) -> usize { + (self.state as usize) << MIN_ALIGN_BITS + } + + /// Set the watermark from a byte offset. + /// The offset must be aligned to `MIN_ALIGN_BITS`. + /// Only meaningful when this is an Untyped region. + #[inline] + pub fn set_watermark_bytes(&mut self, offset: usize) { + debug_assert!(offset & ((1 << MIN_ALIGN_BITS) - 1) == 0); + self.state = u32::try_from(offset >> MIN_ALIGN_BITS).unwrap(); + } + + /// Get the remaining free bytes in this untyped region. + #[inline] + pub fn free_bytes(&self) -> usize { + self.size() - self.watermark_bytes() + } + + // ── Frame-specific ── + // Each frame cap tracks its own single mapping (seL4-style). + // state = mapped vaddr >> 12 (0 = unmapped). + + /// Check if this frame cap is currently mapped. + #[inline] + pub fn is_mapped(&self) -> bool { + self.state != 0 + } + + /// Get the virtual address this frame is mapped at (if any). + #[inline] + pub fn mapped_vaddr(&self) -> Option { + (self.state != 0).then_some(u64::from(self.state) << 12) + } + + /// Record that this frame cap was mapped at `vaddr`. + /// The vaddr must be page-aligned. + #[inline] + pub fn set_mapped(&mut self, vaddr: u64) { + debug_assert!(vaddr.trailing_zeros() >= 12); + debug_assert!(vaddr != 0, "cannot map at vaddr 0"); + self.state = u32::try_from(vaddr >> 12).unwrap(); + } + + /// Clear the mapping (frame was unmapped). + #[inline] + pub fn clear_mapped(&mut self) { + self.state = 0; + } +} diff --git a/kernel/nucleus/src/api/key_table.rs b/kernel/nucleus/src/api/key_table.rs new file mode 100644 index 000000000..ab1e7c077 --- /dev/null +++ b/kernel/nucleus/src/api/key_table.rs @@ -0,0 +1,26 @@ +use libobject::key_table::KeyTableOp; + +// ===================== +// == Syscall handler == +// ===================== + +pub fn invoke(key: &Key, op: u32, args: &[u64]) -> SyscallResult { + let captbl = key.as_keytable()?; + match KeyTableOp::try_from(op)? { + KeyTableOp::CopyDerive => { + let (src, dst_captbl, dst_slot) = (args[0], args[1], args[2]); + // Copy cap from this captbl[src] to dst_captbl[dst_slot] + //... + } + KeyTableOp::Move => { + // Copy cap from captbl[src] to dst_captbl[dst_slot] + // Delete cap in captbl[src] + } + KeyTableOp::Delete => { + // Delete cap in captbl[src] + } + KeyTableOp::Revoke => { + // Revoke cap, by bumping it's epoch and making derived accesses invalid + } + } +} diff --git a/kernel/nucleus/src/api/mod.rs b/kernel/nucleus/src/api/mod.rs new file mode 100644 index 000000000..9d1b2f704 --- /dev/null +++ b/kernel/nucleus/src/api/mod.rs @@ -0,0 +1,211 @@ +use { + crate::objects::{ArchObjects, DebugConsole, Nucleus}, + libobject::{ArchType, CapError, CoreType, KeySlot, ObjectType}, +}; + +// pub mod arch; +pub mod debug_console; +pub mod key_entry; +// pub mod key_table; + +pub use key_entry::KeyEntry; + +// ═════════════════ +// SYSCALL DISPATCH +// ═════════════════ + +/// Main capability invocation handler with two-level dispatch. +/// +/// First: single bit test to separate arch vs core +/// Then: smaller match within each category +/// +/// This is more branch-predictor friendly because: +/// 1. The arch bit test is highly predictable (most calls are core) +/// 2. Each sub-match has fewer cases +#[inline] +pub fn handle_cap_invoke( + nucleus: &mut Nucleus, + cap_slot: u32, + op: u32, + args: &[u64; 6], +) -> Result<(u64, u64), CapError> { + let slot = KeySlot(cap_slot); + #[cfg(feature = "qemu")] + libqemu::semi_println!( + "handle_cap_invoke(slot {cap_slot}:op {op}:args[{},{},{},{},{},{}])", + args[0], + args[1], + args[2], + args[3], + args[4], + args[5] + ); + let obj_type = { + let domain = nucleus + .current_domain_mut() + .ok_or(CapError::InvalidDomain)?; + #[cfg(feature = "qemu")] + libqemu::semi_println!("handle_cap_invoke(got domain)"); + let entry = domain.keytable.lookup_mut(slot)?; + #[cfg(feature = "qemu")] + libqemu::semi_println!("handle_cap_invoke(got entry)"); + entry.object_type() + }; + + #[cfg(feature = "qemu")] + libqemu::semi_println!("handle_cap_invoke(resolved obj_type {})", obj_type.as_u8()); + + if obj_type.is_arch() { + // Architecture-specific dispatch (less common path) + arch_invoke::(nucleus, slot, obj_type, op, args) + } else { + // Core dispatch (common path) + core_invoke::(nucleus, slot, obj_type, op, args) + } +} + +/// Core object dispatch +#[inline(always)] +fn core_invoke( + nucleus: &mut Nucleus, + entry_slot: KeySlot, + obj_type: ObjectType, + op: u32, + args: &[u64; 6], +) -> Result<(u64, u64), CapError> { + let core_type = CoreType::try_from(obj_type)?; + + let domain = nucleus + .current_domain_mut() + .ok_or(CapError::InvalidDomain)?; + let entry = domain.keytable.lookup_mut(entry_slot)?; + + #[cfg(feature = "qemu")] + libqemu::semi_println!("core_invoke"); + + match core_type { + CoreType::Null => Err(CapError::NullCapability), + + // CoreType::Untyped => { + // let untyped = entry.as_object_mut::()?; + // // Untyped::invoke(untyped, ....) + // api::untyped::invoke(untyped, entry.rights(), op, args, &mut nucleus.pools) + // } + CoreType::DebugConsole => { + #[cfg(feature = "qemu")] + libqemu::semi_println!("core_invoke: DebugConsole"); + let debug_console = entry.as_object_mut::()?; + // DebugConsole::invoke(debug_console, entry.rights(), op, args, nucleus) + crate::api::debug_console::invoke(entry, op, args[0], args[1]) + } // CoreType::Domain => { + // let domain = entry.as_object_mut::()?; + // api::domain::invoke(domain, entry.rights(), op, args) + // } + + // CoreType::KeyTable => { + // let kt = entry.as_object_mut::()?; + // api::keytable::invoke(kt, entry.rights(), op, args) + // } + + // CoreType::Notification => { + // let notify = entry.as_object_mut::()?; + // api::notification::invoke(notify, entry.rights(), entry.badge(), op, args) + // } + + // CoreType::EventCount => { + // let ec = entry.as_object_mut::()?; + // api::event_count::invoke(ec, entry.rights(), op, args) + // } + + // CoreType::Endpoint => { + // let ep = entry.as_object_mut::()?; + // api::endpoint::invoke(ep, entry.rights(), entry.badge(), op, args, nucleus) + // } + + // CoreType::Time => { + // let time = entry.as_object_mut::()?; + // api::time::invoke(time, entry.rights(), op, args, nucleus) + // } + + // CoreType::Buffer => { + // let buf = entry.as_object_mut::()?; + // api::buffer::invoke(buf, entry.rights(), op, args) + // } + + // CoreType::Reply => { + // let reply = entry.as_object_mut::()?; + // api::reply::invoke(reply, op, args, nucleus) + // } + _ => Err(CapError::UnsupportedCoreType(core_type)), + } +} + +/// Architecture-specific dispatch - defined per architecture +#[inline(always)] +fn arch_invoke( + nucleus: &mut Nucleus, + entry_slot: KeySlot, + obj_type: ObjectType, + op: u32, + args: &[u64; 6], +) -> Result<(u64, u64), CapError> { + let arch_type = ArchType::try_from(obj_type)?; + + let domain = nucleus + .current_domain_mut() + .ok_or(CapError::InvalidDomain)?; + let entry = domain.keytable.lookup_mut(entry_slot)?; + + #[expect( + clippy::match_single_binding, + reason = "All other arms are commented out" + )] + match arch_type { + // ArchType::Frame => { + // A::invoke_frame(entry, op, args, nucleus) + // } + + // ArchType::PageTable => { + // let pt = entry.as_object_mut::()?; + // A::invoke_page_table(pt, entry.rights(), op, args, nucleus) + // } + + // ArchType::VSpace => { + // let vspace = entry.as_object_mut::()?; + // A::invoke_vspace(vspace, entry.rights(), op, args, nucleus) + // } + + // ArchType::ASIDPool => { + // let pool = entry.as_object_mut::()?; + // A::invoke_asid_pool(pool, entry.rights(), op, args, nucleus) + // } + + // ArchType::ASID => { + // let asid = entry.as_object_mut::()?; + // A::invoke_asid(asid, entry.rights(), op, args) + // } + + // ArchType::IOSpace => { + // // May not be supported on all architectures + // A::invoke_io_space(entry, op, args, nucleus) + // } + + // ArchType::IOPort => { + // // x86 only + // #[cfg(target_arch = "x86_64")] + // { + // let port = entry.as_object_mut::()?; + // x86_64::invoke_io_port(port, entry.rights(), op, args) + // } + // #[cfg(not(target_arch = "x86_64"))] + // { + // Err(CapError::UnsupportedArchType(arch_type)) + // } + // } + + // ArchType::IRQHandler => A::invoke_irq_handler(entry, op, args, nucleus), + + // ArchType::IRQControl => A::invoke_irq_control(entry, op, args, nucleus), + x => Err(CapError::UnsupportedArchType(x)), + } +} diff --git a/kernel/nucleus/src/api/notification.rs b/kernel/nucleus/src/api/notification.rs new file mode 100644 index 000000000..2ed860a5b --- /dev/null +++ b/kernel/nucleus/src/api/notification.rs @@ -0,0 +1,46 @@ +// ===================== +// == Syscall handler == +// ===================== + +pub fn invoke( + notify: &mut Notification, + rights: Rights, + badge: u32, + op: u32, + args: &[u64; 6], +) -> Result<(u64, u64), CapError> { + let op = NotifyOp::try_from(op as u8).map_err(|_| CapError::InvalidOperation)?; + + match op { + NotifyOp::Signal => { + // Check we have send rights + if !rights.contains(Rights::SEND) { + return Err(CapError::InsufficientRights); + } + + // Signal using badge (or args[0] if badge is 0) + let bits = if badge != 0 { badge as u64 } else { args[0] }; + notify.signal(bits); + Ok((0, 0)) + } + + NotifyOp::Wait => { + // Check we have receive rights + if !rights.contains(Rights::RECV) { + return Err(CapError::InsufficientRights); + } + + let bits = notify.wait(current_domain_mut()); + Ok((bits, 0)) + } + + NotifyOp::Poll => { + if !rights.contains(Rights::RECV) { + return Err(CapError::InsufficientRights); + } + + let bits = notify.poll(); + Ok((bits, 0)) + } + } +} diff --git a/kernel/nucleus/src/api/reply.rs b/kernel/nucleus/src/api/reply.rs new file mode 100644 index 000000000..433420d69 --- /dev/null +++ b/kernel/nucleus/src/api/reply.rs @@ -0,0 +1,33 @@ +// CLIENT DOMAIN KERNEL SERVER DOMAIN +// ───────────── ────── ───────────── +// +// CSpace: Reply Pool: CSpace: +// ┌──────────────┐ ┌──────────────┐ ┌──────────────┐ +// │ ... │ │ Reply #0 │ │ ... │ +// │ ep_cap ──────┼──┐ │ Reply #1 ◄───┼─────────┼─ reply_slot │ +// │ ... │ │ │ Reply #2 │ │ ... │ +// └──────────────┘ │ │ ... │ └──────────────┘ +// │ └──────────────┘ +// │ ▲ +// │ │ +// ▼ │ +// ┌────────────┐ │ +// │ Endpoint │ │ +// │ │ kernel creates +// │ state: │ Reply object +// │ Idle │ on Call arrival +// │ │ +// │ waiters: │ +// │ (empty) │ +// └────────────┘ + +// ===================== +// == Syscall handler == +// ===================== + +pub fn invoke(cap: &CapEntry, op: u32, arg0: u64, arg1: u64) -> SyscallResult { + match op { + ReplyOp::Send => handle_send(), + _ => Err(SyscallError::InvalidOp), + } +} diff --git a/kernel/nucleus/src/api/time.rs b/kernel/nucleus/src/api/time.rs new file mode 100644 index 000000000..3fbae6b1e --- /dev/null +++ b/kernel/nucleus/src/api/time.rs @@ -0,0 +1,26 @@ +// ===================== +// == Syscall handler == +// ===================== + +pub fn invoke(key: &TimeKey, op: u32, args: [u64; 4]) -> SyscallResult { + match op { + TimeOp::Donate => { + let target = args[0] as KeySlot; + let target_domain = lookup_domain_key(target)?; + // Transfer time + switch to target + key.donate(target_domain) // activate_domain + } + TimeOp::Split => { + let amount_us = args[0]; + // Create new TimeCap with 'amount_us' + // Reduce current cap by same + key.split(amount_us) + } + TimeOp::Merge => { + let other = args[0] as CapSlot; + key.merge(other) + } + TimeOp::Query => Ok([key.remaining_us, 0]), + _ => Err(SyscallError::InvalidOp), + } +} diff --git a/kernel/nucleus/src/api/untyped.rs b/kernel/nucleus/src/api/untyped.rs new file mode 100644 index 000000000..fa6a0b413 --- /dev/null +++ b/kernel/nucleus/src/api/untyped.rs @@ -0,0 +1,53 @@ +// ===================== +// == Syscall handler == +// ===================== + +#[inline] +pub fn invoke(cap: u32, op: u32, args: &[u64]) -> SyscallResult { + // fn captbl_activate(captbl: u32, op: KeyTableOp, slot: u32) -> Result<()> { + // CAPTBL_ACTIVATE + let ct = lookup_captbl(CAPTBL_SELF)?; + if ct.slots[slot].is_valid() { + return Err(SyscallError::SlotOccupied); + } + // ... create object at slot via retype.. + // } +} + +// =========== +// == Tests == +// =========== + +// #[cfg(test)] +// mod untyped_tests { +// use {super::*, crate::buffer::BufferCap}; + +// #[test_case] +// fn create_notification() { +// let mem = UntypedKey::new(0); +// // (~16 bytes) +// mem.retype(ObjectType::Notification, 4, slot_a)?; +// let notify = NotifyCap::from_slot(slot_a); +// } + +// #[test_case] +// fn create_event_count() { +// // (~24 bytes) +// untyped_retype(mem, ObjectType::EventCount, 5, slot_b)?; +// let ec = EventCountCap::from_slot(slot_b); +// } + +// #[test_case] +// fn create_domain() { +// // (~4KB typically, includes nucleus stack + metadata) +// untyped_retype(mem, ObjectType::Domain, 12, slot_c)?; +// let domain = DomainCap::from_slot(slot_c); +// } + +// #[test_case] +// fn create_buffer() { +// // (64KB buffer) +// untyped_retype(mem, ObjectType::Buffer, 16, slot_d)?; +// let buf = BufferCap::::from_slot(slot_d, 1 << 16); +// } +// } diff --git a/kernel/nucleus/src/main.rs b/kernel/nucleus/src/main.rs new file mode 100644 index 000000000..041426711 --- /dev/null +++ b/kernel/nucleus/src/main.rs @@ -0,0 +1,269 @@ +/* + * SPDX-License-Identifier: BlueOak-1.0.0 + * Copyright (c) Berkus Decker + */ + +//! Vesper single-address-space nanokernel. +//! +//! This crate implements the kernel binary proper. + +#![no_std] +#![no_main] +#![feature(decl_macro)] +#![feature(allocator_api)] +#![feature(format_args_nl)] +#![feature(stmt_expr_attributes)] +#![feature(slice_ptr_get)] +#![deny(missing_docs)] +#![deny(warnings)] +#![allow(unused)] +#![allow(internal_features)] +#![allow(linker_messages)] +#![feature(ptr_internals)] +#![feature(core_intrinsics)] + +#[cfg(feature = "qemu")] +use libqemu::{semi_print, semi_println}; +use { + crate::objects::{Nucleus, ObjectPool, arch::ArchPools, domain::DcbPages}, + cfg_if::cfg_if, + core::{ + arch::asm, + cell::{LazyCell, UnsafeCell}, + panic::PanicInfo, + time::Duration, + }, + libcpu::endless_sleep, + libexception::arch::aarch64::ExceptionContext, + liblocking::{IRQSafeNullLock, interface::Mutex}, + liblog::{info, println, warn}, + libmapping::AccessPermissions, + libobject::{ArchType, CapError, KeySlot}, +}; + +/// Syscall API - capability invocation handlers +mod api; +/// Nucleus objects implementations +mod objects; + +// TODO: Split this into read-only part, that does not need locks, per-cpu mutable part that does not need locks, +// TODO: Shared atomic counters that do not need locks and shared mutable collections that DO need locks (but should be minority) +/// Global kernel state, protected by The Great Kernel Lock +static mut NUCLEUS: IRQSafeNullLock>> = + IRQSafeNullLock::new(LazyCell::new(|| { + let mut n = Nucleus:: { + current_domain: None, + dcb_pages: DcbPages::new(), + pools: objects::nucleus::NucleusPools { + /// SAFETY: Not very safe thing at all. + domains: unsafe { ObjectPool::new(0x1000 as *mut u8, 16384) }, // TODO: proper alloc... + /// SAFETY: Not very safe thing at all. + arch: unsafe { ArchPools::new() }, + }, + }; + n.create_domain(); + n + })); + +#[panic_handler] +fn panicked(info: &PanicInfo) -> ! { + libmachine::panic::handler(info) +} + +//------------------------------------------------------------------------------ +//------------------------------------------------------------------------------ +// Exception handlers +//------------------------------------------------------------------------------ +//------------------------------------------------------------------------------ + +/// The default exception handler, invoked for every exception type unless the handler +/// is overridden. +/// Prints verbose information about the exception and then panics. +/// +/// Default pointer is configured in the linker script. +#[unsafe(no_mangle)] +extern "C" fn default_exception_handler(exc: &ExceptionContext) { + panic!( + "Unexpected CPU Exception!\n\n\ + {}", + exc + ); +} + +//------------------------------------------------------------------------------ +// Current, EL0 +//------------------------------------------------------------------------------ + +#[unsafe(no_mangle)] +extern "C" fn current_el0_synchronous(_e: &mut ExceptionContext) { + panic!("Should not be here. Use of SP_EL0 in EL1 is not supported.") +} + +#[unsafe(no_mangle)] +extern "C" fn current_el0_irq(_e: &mut ExceptionContext) { + panic!("Should not be here. Use of SP_EL0 in EL1 is not supported.") +} + +#[unsafe(no_mangle)] +extern "C" fn current_el0_serror(_e: &mut ExceptionContext) { + panic!("Should not be here. Use of SP_EL0 in EL1 is not supported.") +} + +//------------------------------------------------------------------------------ +// Current, ELx +//------------------------------------------------------------------------------ + +#[cfg(not(any(test, feature = "test_build")))] +#[unsafe(no_mangle)] +extern "C" fn current_elx_synchronous(e: &mut ExceptionContext) { + cap_invoke_handler(e); +} + +#[cfg(any(test, feature = "test_build"))] +#[unsafe(no_mangle)] +extern "C" fn current_elx_synchronous(e: &mut ExceptionContext) { + { + use aarch64_cpu::registers::{ESR_EL1, Readable}; + + const TEST_SVC_ID: u64 = 0x1337; + + let esr_el1 = libexception::arch::esr_el1::EsrEL1(LocalRegisterCopy::new(ESR_EL1.get())); + + if let Some(ESR_EL1::EC::Value::SVC64) = esr_el1.exception_class() + && esr_el1.iss() == TEST_SVC_ID + { + liblog::println!("Serving syscall {TEST_SVC_ID}"); + return; + } + } + + if libdebug::exception_dump(e) { + return; + } + + default_exception_handler(e); +} + +#[unsafe(no_mangle)] +extern "C" fn current_elx_irq(e: &mut ExceptionContext) { + // -- @todo + // let token = unsafe { &exception::asynchronous::IRQContext::new() }; + // exception::asynchronous::irq_manager().handle_pending_irqs(token); + default_exception_handler(e); +} + +#[unsafe(no_mangle)] +extern "C" fn current_elx_serror(e: &mut ExceptionContext) { + default_exception_handler(e); +} + +//------------------------------------------------------------------------------ +// Lower, AArch64 +//------------------------------------------------------------------------------ + +#[unsafe(no_mangle)] +extern "C" fn lower_aarch64_synchronous(e: &mut ExceptionContext) { + cap_invoke_handler(e); +} + +#[unsafe(no_mangle)] +extern "C" fn lower_aarch64_irq(e: &mut ExceptionContext) { + default_exception_handler(e); +} + +#[unsafe(no_mangle)] +extern "C" fn lower_aarch64_serror(e: &mut ExceptionContext) { + default_exception_handler(e); +} + +//------------------------------------------------------------------------------ +// Lower, AArch32 +//------------------------------------------------------------------------------ + +#[unsafe(no_mangle)] +extern "C" fn lower_aarch32_synchronous(e: &mut ExceptionContext) { + default_exception_handler(e); +} + +#[unsafe(no_mangle)] +extern "C" fn lower_aarch32_irq(e: &mut ExceptionContext) { + default_exception_handler(e); +} + +#[unsafe(no_mangle)] +extern "C" fn lower_aarch32_serror(e: &mut ExceptionContext) { + default_exception_handler(e); +} + +//------------------------------------------------------------------------------ +// Kernel entry point +//------------------------------------------------------------------------------ + +#[unsafe(no_mangle)] +extern "C" fn cap_invoke_handler(frame: &mut ExceptionContext) { + let cap_slot = u32::try_from(frame.gpr[0]).unwrap(); + let op = u32::try_from(frame.gpr[1]).unwrap(); + #[cfg(feature = "qemu")] + semi_println!( + "CapInvoke SYSCALL(cap: {cap_slot}, op: {op}) happened, we're at PC {:#016X}, SP {:#016X}, exception frame @ {:#016X}", + get_pc(), + get_sp(), + core::ptr::from_mut(frame) as u64, + ); + + // semi_println!("{}", frame); + + let args: &[u64; 6] = &frame.gpr[2..=7].try_into().unwrap(); + + // SAFETY: Unsafe. + let result = unsafe { + #[allow(static_mut_refs)] + NUCLEUS.lock(|nucleus| api::handle_cap_invoke(nucleus, cap_slot, op, args)) + }; + + // let cap = current_domain().keytable.lookup(cap_slot)?; + // let args = &[arg0, arg1, arg2, arg3, arg4, arg5]; // FIXME temp + + // let result = match cap.cap_type() { + // ObjectType::Untyped => api::untyped::invoke(cap, op, args), // retype, split + // ObjectType::Domain => api::domain::invoke(cap, op, args), // activate, suspend... + // ObjectType::KeyTable => api::key_table::invoke(cap, op, args), + // ObjectType::Time => api::time::invoke(cap, op, args), // donate, split, merge + // ObjectType::Endpoint => api::endpoint::invoke(cap, op, args), + // ObjectType::Notification => api::notification::invoke(cap, op, args), + // ObjectType::EventCount => api::event_count::invoke(cap, op, args), + // ObjectType::Buffer => api::buffer::invoke(cap, op, args), // map, unmap, query + // ObjectType::None => Err(SyscallError::InvalidSlot), + // }; + + let (x0, x1, x2) = match result { + Ok((v0, v1)) => (0, v0, v1), + Err(e) => e.code(), + }; + // Return values + #[cfg(feature = "qemu")] + semi_println!("CapInvoke SYSCALL(Return {x0:#x}, {x1:#x}, {x2:#x})",); + // SAFETY: Not safe. + unsafe { + frame.gpr[0] = x0; + frame.gpr[1] = x1; + frame.gpr[2] = x2; + } +} + +fn get_pc() -> u64 { + let pc: u64; + // SAFETY: Safe. + unsafe { + asm!( + "adr {}, .", + out(reg) pc, + ); + } + pc +} + +fn get_sp() -> u64 { + use aarch64_cpu::registers::Readable; + aarch64_cpu::registers::SP.get() +} diff --git a/kernel/nucleus/src/objects/arch/aarch64_objects.rs b/kernel/nucleus/src/objects/arch/aarch64_objects.rs new file mode 100644 index 000000000..25647c0e0 --- /dev/null +++ b/kernel/nucleus/src/objects/arch/aarch64_objects.rs @@ -0,0 +1,175 @@ +use { + crate::{ + Nucleus, + api::key_entry::KeyEntry, + objects::{ + ArchObjects, + arch::{AArch64ASID, AArch64ASIDPool, AArch64PageTable, AArch64VSpace, ArchPools}, + arch_objects::FrameSize, + object_ref::ObjectRef, + }, + }, + libaddress::PhysAddr, + libobject::{ArchType, CapError, Rights}, +}; + +// ═══════════════════════════════════════════════════════════════════ +// AARCH64 IMPLEMENTATION +// ═══════════════════════════════════════════════════════════════════ + +pub struct AArch64; + +impl ArchObjects for AArch64 { + type PageTable = AArch64PageTable; + type VSpace = AArch64VSpace; + type ASIDPool = AArch64ASIDPool; + type ASID = AArch64ASID; + + const FRAME_SIZES: &'static [FrameSize] = + &[FrameSize::Small, FrameSize::Large, FrameSize::Huge]; + + const PT_LEVELS: usize = 4; + const PT_INDEX_BITS: usize = 9; + + fn validate_frame_size(size_bits: u8) -> Result { + match size_bits { + 12 => Ok(4096), // 4KB + 21 => Ok(2 * 1024 * 1024), // 2MB + 30 => Ok(1024 * 1024 * 1024), // 1GB + _ => Err(CapError::InvalidFrameSize(size_bits as usize)), + } + } + + fn validate_retype(arch_type: ArchType, size_bits: u8) -> Result { + match arch_type { + ArchType::PageTable => { + if size_bits == 12 { + Ok(4096) + } else { + Err(CapError::InvalidSize(size_bits as usize)) + } + } + ArchType::VSpace => Ok(core::mem::size_of::()), + ArchType::ASIDPool => Ok(core::mem::size_of::()), + ArchType::ASID => Ok(core::mem::size_of::()), + _ => Err(CapError::UnsupportedArchType(arch_type)), + } + } + + fn create_arch_object( + arch_type: ArchType, + phys_addr: PhysAddr, + size_bits: u8, + pools: &mut ArchPools, + ) -> Result { + // match arch_type { + // ArchType::Frame => { + // let frame_size = FrameSize::from_bits(size_bits as usize) + // .map_err(|_| CapError::InvalidSize(size_bits as usize))?; + // let frame = AArch64Frame::new(phys_addr, frame_size); + // let obj = pools + // .frames + // .allocate(frame) + // .ok_or(CapError::PoolExhausted)?; + // Ok(ObjectRef::new(obj)) + // } + // ArchType::PageTable => { + // let pt = AArch64PageTable::new(phys_addr); + // let obj = pools + // .page_tables + // .allocate(pt) + // .ok_or(CapError::PoolExhausted)?; + // Ok(ObjectRef::new(obj)) + // } + // ArchType::VSpace => { + // let vspace = AArch64VSpace::new(); + // let obj = pools + // .vspaces + // .allocate(vspace) + // .ok_or(CapError::PoolExhausted)?; + // Ok(ObjectRef::new(obj)) + // } + // ArchType::ASIDPool => { + // let pool = AArch64ASIDPool::new(); + // let obj = pools + // .asid_pools + // .allocate(pool) + // .ok_or(CapError::PoolExhausted)?; + // Ok(ObjectRef::new(obj)) + // } + // _ => Err(CapError::UnsupportedArchType(arch_type)), + // } + Err(CapError::UnsupportedArchType(arch_type)) + } + + // ───────────────────────────────────────────────────────────────── + // Frame Operations + // ───────────────────────────────────────────────────────────────── + + fn invoke_frame( + entry: &mut KeyEntry, + op: u32, + args: &[u64; 6], + nucleus: &mut Nucleus, + ) -> Result<(u64, u64), CapError> { + // crate::api::arch::frame::invoke(entry, op, args, nucleus) + Err(CapError::InvalidOperation) + } + + // ───────────────────────────────────────────────────────────────── + // Page Table Operations + // ───────────────────────────────────────────────────────────────── + + fn invoke_page_table( + pt: &mut AArch64PageTable, + rights: Rights, + op: u32, + args: &[u64; 6], + nucleus: &mut Nucleus, + ) -> Result<(u64, u64), CapError> { + // crate::api::arch::page_table::invoke(pt, rights, op, args) + Err(CapError::InvalidOperation) + } + + // ───────────────────────────────────────────────────────────────── + // VSpace Operations + // ───────────────────────────────────────────────────────────────── + + fn invoke_vspace( + vspace: &mut AArch64VSpace, + rights: Rights, + op: u32, + args: &[u64; 6], + nucleus: &mut Nucleus, + ) -> Result<(u64, u64), CapError> { + // crate::api::arch::vspace::invoke(vspace, rights, op, args) + Err(CapError::InvalidOperation) + } + + // ───────────────────────────────────────────────────────────────── + // ASID Pool Operations + // ───────────────────────────────────────────────────────────────── + + fn invoke_asid_pool( + pool: &mut AArch64ASIDPool, + rights: Rights, + op: u32, + args: &[u64; 6], + _nucleus: &mut Nucleus, + ) -> Result<(u64, u64), CapError> { + // Most ASID operations go through VSpace.AssignASID + // Direct pool operations are rare + Err(CapError::InvalidOperation) + } + + fn invoke_asid( + asid: &mut AArch64ASID, + rights: Rights, + op: u32, + args: &[u64; 6], + ) -> Result<(u64, u64), CapError> { + // ASID capabilities are mostly just tokens + // Operations would be for explicit invalidation + Err(CapError::InvalidOperation) + } +} diff --git a/kernel/nucleus/src/objects/arch/arch_pools.rs b/kernel/nucleus/src/objects/arch/arch_pools.rs new file mode 100644 index 000000000..2840f6471 --- /dev/null +++ b/kernel/nucleus/src/objects/arch/arch_pools.rs @@ -0,0 +1,37 @@ +use crate::objects::{ArchObjects, ObjectPool}; + +// ═══════════════════════════════════════════════════════════════════ +// ARCHITECTURE-SPECIFIC OBJECT POOLS +// ═══════════════════════════════════════════════════════════════════ + +/// Pools for architecture-specific objects +pub struct ArchPools { + // pub frames: ObjectPool, + // pub page_tables: ObjectPool, + // pub vspaces: ObjectPool, + // pub asid_pools: ObjectPool, + // pub asids: ObjectPool, + _marker: core::marker::PhantomData, // FIXME temp +} + +impl ArchPools { + /// Create pools backed by untyped memory + /// + /// # Safety + /// Memory regions must be valid and non-overlapping + pub unsafe fn new(// frame_mem: (*mut u8, usize), + // pt_mem: (*mut u8, usize), + // vspace_mem: (*mut u8, usize), + // asid_pool_mem: (*mut u8, usize), + // asid_mem: (*mut u8, usize), + ) -> Self { + Self { + // frames: unsafe { ObjectPool::new(frame_mem.0, frame_mem.1) }, + // page_tables: unsafe { ObjectPool::new(pt_mem.0, pt_mem.1) }, + // vspaces: unsafe { ObjectPool::new(vspace_mem.0, vspace_mem.1) }, + // asid_pools: unsafe { ObjectPool::new(asid_pool_mem.0, asid_pool_mem.1) }, + // asids: unsafe { ObjectPool::new(asid_mem.0, asid_mem.1) }, + _marker: core::marker::PhantomData, + } + } +} diff --git a/kernel/nucleus/src/objects/arch/asid.rs b/kernel/nucleus/src/objects/arch/asid.rs new file mode 100644 index 000000000..51980d6ce --- /dev/null +++ b/kernel/nucleus/src/objects/arch/asid.rs @@ -0,0 +1,7 @@ +use {crate::objects::NucleusObject, libobject::ObjectType}; + +pub struct AArch64ASID; + +impl NucleusObject for AArch64ASID { + const TYPE: ObjectType = ObjectType::ASID; +} diff --git a/kernel/nucleus/src/objects/arch/asid_pool.rs b/kernel/nucleus/src/objects/arch/asid_pool.rs new file mode 100644 index 000000000..5bcb9b482 --- /dev/null +++ b/kernel/nucleus/src/objects/arch/asid_pool.rs @@ -0,0 +1,13 @@ +use {crate::objects::NucleusObject, libobject::ObjectType}; + +pub struct AArch64ASIDPool; + +impl AArch64ASIDPool { + pub fn new() -> Self { + Self + } +} + +impl NucleusObject for AArch64ASIDPool { + const TYPE: ObjectType = ObjectType::ASID_POOL; +} diff --git a/kernel/nucleus/src/objects/arch/frame.rs b/kernel/nucleus/src/objects/arch/frame.rs new file mode 100644 index 000000000..240294f65 --- /dev/null +++ b/kernel/nucleus/src/objects/arch/frame.rs @@ -0,0 +1,8 @@ +// ───────────────────────────────────────────────────────────────── +// Frame capabilities are now inline in KeyEntry as RegionPayload. +// No separate AArch64Frame struct needed — the capability IS the object. +// +// See: api/key_entry.rs (RegionPayload, KeyEntry::new_frame) +// objects/untyped.rs (retype creates inline Frame caps) +// api/arch/frame.rs (frame invocation handler) +// ───────────────────────────────────────────────────────────────── diff --git a/kernel/nucleus/src/objects/arch/mod.rs b/kernel/nucleus/src/objects/arch/mod.rs new file mode 100644 index 000000000..527d50a39 --- /dev/null +++ b/kernel/nucleus/src/objects/arch/mod.rs @@ -0,0 +1,30 @@ +#[cfg(target_arch = "aarch64")] +pub mod aarch64_objects; +#[cfg(target_arch = "aarch64")] +pub use aarch64_objects::AArch64 as ArchObjectsImpl; + +pub mod arch_pools; +pub use arch_pools::ArchPools; + +#[cfg(target_arch = "aarch64")] +pub mod asid; +#[cfg(target_arch = "aarch64")] +pub use asid::AArch64ASID; + +#[cfg(target_arch = "aarch64")] +pub mod asid_pool; +#[cfg(target_arch = "aarch64")] +pub use asid_pool::AArch64ASIDPool; + +#[cfg(target_arch = "aarch64")] +pub mod frame; + +#[cfg(target_arch = "aarch64")] +pub mod page_table; +#[cfg(target_arch = "aarch64")] +pub use page_table::AArch64PageTable; + +#[cfg(target_arch = "aarch64")] +pub mod vspace; +#[cfg(target_arch = "aarch64")] +pub use vspace::AArch64VSpace; diff --git a/kernel/nucleus/src/objects/arch/page_table.rs b/kernel/nucleus/src/objects/arch/page_table.rs new file mode 100644 index 000000000..8c1dae8b2 --- /dev/null +++ b/kernel/nucleus/src/objects/arch/page_table.rs @@ -0,0 +1,13 @@ +use {crate::objects::NucleusObject, libaddress::PhysAddr, libobject::ObjectType}; + +pub struct AArch64PageTable; + +impl AArch64PageTable { + pub fn new(_addr: PhysAddr) -> Self { + Self + } +} + +impl NucleusObject for AArch64PageTable { + const TYPE: ObjectType = ObjectType::PAGE_TABLE; +} diff --git a/kernel/nucleus/src/objects/arch/vspace.rs b/kernel/nucleus/src/objects/arch/vspace.rs new file mode 100644 index 000000000..5de5e290f --- /dev/null +++ b/kernel/nucleus/src/objects/arch/vspace.rs @@ -0,0 +1,13 @@ +use {crate::objects::NucleusObject, libobject::ObjectType}; + +pub struct AArch64VSpace; + +impl AArch64VSpace { + pub fn new() -> Self { + Self + } +} + +impl NucleusObject for AArch64VSpace { + const TYPE: ObjectType = ObjectType::VSPACE; +} diff --git a/kernel/nucleus/src/objects/arch_objects.rs b/kernel/nucleus/src/objects/arch_objects.rs new file mode 100644 index 000000000..f9ae961d5 --- /dev/null +++ b/kernel/nucleus/src/objects/arch_objects.rs @@ -0,0 +1,152 @@ +use { + crate::{ + api::key_entry::KeyEntry, + objects::{NucleusObject, arch::ArchPools, nucleus::Nucleus, object_ref::ObjectRef}, + }, + libaddress::PhysAddr, + libobject::{ArchType, CapError, Rights}, +}; + +// ═══════════════════════════════════════════════════════════════════ +// ARCH OBJECTS TRAIT WITH INVOKE METHODS +// ═══════════════════════════════════════════════════════════════════ + +/// Frame size enumeration (common across architectures) +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub enum FrameSize { + /// 4KB (standard page) + Small, // 12 bits + /// 2MB (large page / section) + Large, // 21 bits + /// 1GB (huge page / supersection) + Huge, // 30 bits +} + +impl FrameSize { + pub const fn bits(self) -> u8 { + match self { + FrameSize::Small => 12, + FrameSize::Large => 21, + FrameSize::Huge => 30, + } + } + + pub fn from_bits(bits: usize) -> Result { + match bits { + 12 => Ok(FrameSize::Small), + 21 => Ok(FrameSize::Large), + 30 => Ok(FrameSize::Huge), + _ => Err(()), + } + } + + pub const fn size(self) -> usize { + 1 << self.bits() + } +} + +/// Architecture abstraction trait - extended with invoke methods. +/// +/// Frame capabilities are arch-independent (inline `RegionPayload` in `KeyEntry`), +/// so there is no `type Frame` associated type. Frame size validation is +/// arch-specific via `validate_frame_size`. +pub trait ArchObjects: Sized + 'static { + // ─── Associated Types (pool-backed arch objects only) ─── + type PageTable: NucleusObject; + type VSpace: NucleusObject; + type ASIDPool: NucleusObject; + type ASID: NucleusObject; + + // ─── Constants ─── + const FRAME_SIZES: &'static [FrameSize]; + const PT_LEVELS: usize; + const PT_INDEX_BITS: usize; + + // ─── Validation ─── + + /// Validate frame `size_bits` for this architecture. + /// Returns the frame size in bytes on success. + fn validate_frame_size(size_bits: u8) -> Result; + + /// Validate and return object size for pool-backed arch types. + fn validate_retype(arch_type: ArchType, size_bits: u8) -> Result; + + // ─── Object Creation (pool-backed arch types only) ─── + /// Create a pool-backed arch object. Frame is NOT handled here — + /// it is created inline via `KeyEntry::new_frame()` in the retype path. + fn create_arch_object( + arch_type: ArchType, + phys_addr: PhysAddr, + size_bits: u8, + pools: &mut ArchPools, + ) -> Result; + + // ─── Invocation Handlers ─── + + /// Handle frame operations. The frame data is inline in `entry` as a `RegionPayload`. + fn invoke_frame( + entry: &mut KeyEntry, + op: u32, + args: &[u64; 6], + nucleus: &mut Nucleus, + ) -> Result<(u64, u64), CapError>; + + fn invoke_page_table( + pt: &mut Self::PageTable, + rights: Rights, + op: u32, + args: &[u64; 6], + nucleus: &mut Nucleus, + ) -> Result<(u64, u64), CapError>; + + fn invoke_vspace( + vspace: &mut Self::VSpace, + rights: Rights, + op: u32, + args: &[u64; 6], + nucleus: &mut Nucleus, + ) -> Result<(u64, u64), CapError>; + + fn invoke_asid_pool( + pool: &mut Self::ASIDPool, + rights: Rights, + op: u32, + args: &[u64; 6], + nucleus: &mut Nucleus, + ) -> Result<(u64, u64), CapError>; + + fn invoke_asid( + asid: &mut Self::ASID, + rights: Rights, + op: u32, + args: &[u64; 6], + ) -> Result<(u64, u64), CapError>; + + // Optional - default implementations return UnsupportedArchType + fn invoke_io_space( + _entry: &mut KeyEntry, + _op: u32, + _args: &[u64; 6], + _nucleus: &mut Nucleus, + ) -> Result<(u64, u64), CapError> { + Err(CapError::UnsupportedArchType(ArchType::IOSpace)) + } + + fn invoke_irq_handler( + _entry: &mut KeyEntry, + _op: u32, + _args: &[u64; 6], + _nucleus: &mut Nucleus, + ) -> Result<(u64, u64), CapError> { + Err(CapError::UnsupportedArchType(ArchType::IRQHandler)) + } + + fn invoke_irq_control( + _entry: &mut KeyEntry, + _op: u32, + _args: &[u64; 6], + _nucleus: &mut Nucleus, + ) -> Result<(u64, u64), CapError> { + Err(CapError::UnsupportedArchType(ArchType::IRQControl)) + } +} diff --git a/kernel/nucleus/src/objects/buffer.rs b/kernel/nucleus/src/objects/buffer.rs new file mode 100644 index 000000000..ad725b3dc --- /dev/null +++ b/kernel/nucleus/src/objects/buffer.rs @@ -0,0 +1,33 @@ +// ==================== +// == Nucleus object == +// ==================== + +/// Buffer kernel object - represents a contiguous memory region +pub struct Buffer { + phys_base: PhysAddr, // Physical address (fixed at creation) + size: usize, // Size in bytes (fixed at creation) + flags: BufferFlags, // CACHED, DEVICE, SHARED, etc. + + // Mapping tracking (for single-address-space) + // mappings: SmallVec, // Who has it mapped where +} + +struct Mapping { + domain_id: DomainId, + virt_addr: VirtAddr, + permissions: Rights, // May be less than cap rights (derived cap) +} + +bitflags::bitflags! { + pub struct BufferFlags: u32 { + const CACHED = 1 << 0; // Normal cacheable memory + const DEVICE = 1 << 1; // Device memory (uncached, no speculative) + const SHARED = 1 << 2; // Multi-domain sharing expected + const DMA = 1 << 3; // DMA-capable (physically contiguous) + const EXEC = 1 << 4; // Executable (if supported) + } +} + +impl NucleusObject for Buffer { + const TYPE: ObjectType = ObjectType::Buffer; +} diff --git a/kernel/nucleus/src/objects/debug_console.rs b/kernel/nucleus/src/objects/debug_console.rs new file mode 100644 index 000000000..2d031bdd1 --- /dev/null +++ b/kernel/nucleus/src/objects/debug_console.rs @@ -0,0 +1,45 @@ +use { + crate::objects::NucleusObject, + core::slice, + libaddress::PhysAddr, + libobject::{CapError, ObjectType}, +}; + +// ==================== +// == Nucleus object == +// ==================== + +pub struct DebugConsole; + +impl DebugConsole { + pub fn handle_write(ptr: PhysAddr, len: u64) -> Result<(), CapError> { + // libqemu::semi_println!( + // "DebugConsole::handle_write(user ptr {ptr:?}, kernel ptr {:?}, size {})", + // ptr.user_to_kernel(), + // len + // ); + let len = usize::try_from(len).unwrap(); + // SAFETY: Unsafe, need to check user pointers. + let slice = unsafe { slice::from_raw_parts(ptr.user_to_kernel().as_ptr(), len) }; + let mut buf = [0_u8; 4096]; + // libqemu::semi_println!( + // "DebugConsole::copy from user to {:#08x}", + // &buf as *const _ as u64 + // ); + buf[..len].copy_from_slice(slice); + buf[slice.len()] = 0; + let cstr = + // SAFETY: Need to validate user pointer is valid, need to copy via kernel physmem mapping. + unsafe { core::ffi::CStr::from_bytes_with_nul(&buf[..=slice.len()]) }.map_err(|e| { + // libqemu::semi_println!("{e}"); + CapError::Unknown + })?; + #[cfg(feature = "qemu")] + libqemu::semihosting::sys_write0_call(cstr); + Ok(()) + } +} + +impl NucleusObject for DebugConsole { + const TYPE: ObjectType = ObjectType::DEBUG_CONSOLE; +} diff --git a/kernel/nucleus/src/objects/domain.rs b/kernel/nucleus/src/objects/domain.rs new file mode 100644 index 000000000..d52501778 --- /dev/null +++ b/kernel/nucleus/src/objects/domain.rs @@ -0,0 +1,248 @@ +use { + crate::objects::{KeyTable, NucleusObject}, + core::{ptr::NonNull, sync::atomic::Ordering}, + libaddress::{PhysAddr, VirtAddr}, + libobject::{ + ObjectType, + domain::{DcbPage, DomainControlBlock, DomainId, DomainState}, + }, +}; + +// ==================== +// == Nucleus object == +// ==================== + +/// This is a nucleus-visible half of domain structure. +/// The `DomainControlBlock` is user-visible and is defined in libobject. +pub struct Domain { + // ═══════════════════════════════════════════════════════════ + // PRIVATE SECTION (kernel only, NOT mapped to userspace) + // ═══════════════════════════════════════════════════════════ + // + // This would be in a separate structure or after a page boundary + // - Saved register context + // - Capability space (keytable) + // - Kernel stack pointer + // - Etc. + pub keytable: KeyTable, +} + +// Verify size for cache alignment +// TODO const _: () = assert!(core::mem::size_of::() == 4096); + +impl NucleusObject for Domain { + const TYPE: ObjectType = ObjectType::DOMAIN; +} + +impl Domain { + // Initialize new domain's cspace + // fn init_cspace(&mut self) { + // // Slot 0: capability to this captbl itself + // self.cspace[CAPTBL_SELF] = Cap::new(ObjectType::KeyTable, self.cspace_id); + // // Now domain can manipulate its own caps + // } +} + +// ## Memory Ordering Considerations +// +// KERNEL (writer) USERSPACE (reader) +// ─────────────── ────────────────── +// +// // Update multiple fields +// dcb.time_used.store(x, Relaxed); +// dcb.time_remaining.store(y, Relaxed); +// dcb.state.store(z, Release); ──────────────────────┐ +// │ │ +// │ Release ensures all │ +// │ prior writes visible │ +// ▼ ▼ +// let state = dcb.state.load(Acquire); +// // Acquire ensures we see +// // all writes before the Release +// let used = dcb.time_used.load(Relaxed); +// let rem = dcb.time_remaining.load(Relaxed); +// +// Protocol: +// - Kernel does Release store on state LAST +// - Userspace does Acquire load on state FIRST +// - Then can safely read other fields with Relaxed + +// ═══════════════════════════════════════════════════════════════════ +// DCB PAGES MANAGER +// ═══════════════════════════════════════════════════════════════════ + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum DcbError { + TooManyPages, + PageNotAllocated, + NoFreeDomains, + NotAllocated, + InvalidDomainId, +} + +/// Manager for all DCB pages in the system. +/// +/// Provides: +/// - Kernel-side mutable access for state updates +/// - Physical addresses for user-space mapping +/// - Domain ID allocation +pub struct DcbPages { + /// Array of DCB pages (kernel virtual addresses) + pages: [Option<&'static mut DcbPage>; Self::MAX_PAGES], + /// Physical addresses of each page (for user mapping) + phys_addrs: [Option; Self::MAX_PAGES], // TODO: Option> + /// Number of allocated pages + num_pages: usize, + /// Next domain ID to allocate + next_domain_id: u32, + /// Bitmap of allocated domain IDs + allocated: [u64; Self::MAX_DOMAINS / 64], +} + +impl DcbPages { + /// Maximum number of DCB pages (supports up to 8192 domains) + pub const MAX_PAGES: usize = 256; + /// Maximum domains (256 pages × 32 DCBs/page) + pub const MAX_DOMAINS: usize = Self::MAX_PAGES * DcbPage::DCBS_PER_PAGE as usize; + + /// Well-known user-space base address for DCB mapping + /// This is mapped read-only into all domains + /// SAFETY: Safe, manual fixed address. + pub const USER_BASE: VirtAddr = unsafe { VirtAddr::new_unchecked(0x0000_7FFF_FE00_0000) }; + + /// Create empty DCB pages manager + pub const fn new() -> Self { + Self { + pages: [const { None }; Self::MAX_PAGES], + phys_addrs: [const { None }; Self::MAX_PAGES], + num_pages: 0, + next_domain_id: 0, + allocated: [0; Self::MAX_DOMAINS / 64], + } + } + + /// Add a new DCB page (called during kernel init) + /// + /// # Safety + /// - `page` must be valid, aligned, and not aliased + /// - `phys_addr` must be the correct physical address + pub unsafe fn add_page( + &mut self, + page: *mut DcbPage, + phys_addr: PhysAddr, + ) -> Result { + if self.num_pages >= Self::MAX_PAGES { + return Err(DcbError::TooManyPages); + } + + let idx = self.num_pages; + // SAFETY: Unsafe. + self.pages[idx] = unsafe { Some(&mut *page) }; + self.phys_addrs[idx] = Some(phys_addr); + self.num_pages += 1; + + Ok(idx) + } + + /// Allocate a new domain ID and initialize its DCB + pub fn allocate_domain(&mut self, scheduler_id: DomainId) -> Result { + // Find free slot + let id = self.find_free_slot()?; + + // Mark as allocated + let word = id as usize / 64; + let bit = id as usize % 64; + self.allocated[word] |= 1 << bit; + + // Initialize DCB + let domain_id = DomainId(id); + let dcb = self.get_mut(domain_id).ok_or(DcbError::PageNotAllocated)?; + *dcb = DomainControlBlock::new(domain_id, scheduler_id); + + Ok(domain_id) + } + + /// Release a domain ID + pub fn release_domain(&mut self, id: DomainId) -> Result<(), DcbError> { + let word = id.0 as usize / 64; + let bit = id.0 as usize % 64; + + if self.allocated[word] & (1 << bit) == 0 { + return Err(DcbError::NotAllocated); + } + + // Mark as free + self.allocated[word] &= !(1 << bit); + + // Clear DCB + if let Some(dcb) = self.get_mut(id) { + dcb.state + .store(DomainState::Inactive as u32, Ordering::Release); + dcb.id = DomainId::INVALID; + } + + Ok(()) + } + + /// Get a DCB by domain ID (immutable) + #[inline] + pub fn get(&self, id: DomainId) -> Option<&DomainControlBlock> { + let page_idx = id.page_index(); + let slot = id.slot_in_page(); + + self.pages.get(page_idx)?.as_ref()?.get(slot) + } + + /// Get a DCB by domain ID (mutable) - kernel only + #[inline] + pub fn get_mut(&mut self, id: DomainId) -> Option<&mut DomainControlBlock> { + let page_idx = id.page_index(); + let slot = id.slot_in_page(); + + self.pages.get_mut(page_idx)?.as_mut()?.get_mut(slot) + } + + /// Get physical address of a DCB page (for user mapping) + pub fn page_phys_addr(&self, page_idx: usize) -> Option { + self.phys_addrs.get(page_idx).copied().flatten() + } + + /// Get user-space virtual address for a domain's DCB + #[expect(clippy::unused_self)] + pub fn user_addr(&self, id: DomainId) -> VirtAddr { + VirtAddr::new(Self::USER_BASE.as_u64() + (u64::from(id.0) * 128)) + } + + /// Iterate over all allocated domains + pub fn iter_allocated(&self) -> impl Iterator + '_ { + self.allocated + .iter() + .enumerate() + .flat_map(|(word_idx, &word)| { + (0..64) + .filter(move |&bit| (word & (1 << bit) != 0)) + .map(move |bit| DomainId(u32::try_from(word_idx * 64 + bit).unwrap())) + }) + } + + /// Number of allocated domains + pub fn num_allocated(&self) -> usize { + self.allocated.iter().map(|w| w.count_ones() as usize).sum() + } + + fn find_free_slot(&self) -> Result { + let max_id = u32::try_from(self.num_pages * DcbPage::DCBS_PER_PAGE as usize).unwrap(); + + for (word_idx, &word) in self.allocated.iter().enumerate() { + if word != !0 { + let bit = word.trailing_ones(); + let id = u32::try_from(word_idx * 64).unwrap() + bit; + if id < max_id { + return Ok(id); + } + } + } + + Err(DcbError::NoFreeDomains) + } +} diff --git a/kernel/nucleus/src/objects/endpoint.rs b/kernel/nucleus/src/objects/endpoint.rs new file mode 100644 index 000000000..4b4bb30cc --- /dev/null +++ b/kernel/nucleus/src/objects/endpoint.rs @@ -0,0 +1,196 @@ +// ==================== +// == Nucleus object == +// ==================== + +/// Message passed through endpoints +/// Fits in registers for fast IPC (~200-500 cycles) +#[repr(C)] +pub struct Message { + /// Message label/opcode - receiver dispatches on this + pub label: u64, + + /// 5 general-purpose data words + pub data: [u64; 5], + + /// Optional capability to transfer (None = no transfer) + /// Sender's cap is moved (not copied) to receiver + pub cap: Option, +} + +impl Message { + pub const fn new(label: u64) -> Self { + Self { + label, + data: [0; 5], + cap: None, + } + } + + pub fn with_data(label: u64, d0: u64, d1: u64, d2: u64, d3: u64, d4: u64) -> Self { + Self { + label, + data: [d0, d1, d2, d3, d4], + cap: None, + } + } + + pub fn with_cap(mut self, cap_slot: CapSlot) -> Self { + self.cap = Some(cap_slot); + self + } +} + +/// Kernel object +struct Endpoint { + /// Domain that can receive on this endpoint (None = anyone) + receiver: Option, + + /// Current state + state: EndpointState, + + /// Waiting senders (if state == Recving, this is empty) + /// Waiting receivers (if state == Sending, this is empty) + wait_queue: WaitQueue, + + /// Message buffer (kernel memory) + msg_regs: [u64; 6], + + /// Badge of sender (filled when message delivered) + sender_badge: u64, + + /// Cap transfer slot + transfer_cap: Option, +} + +enum EndpointState { + Idle, + /// Someone is blocked sending + Sending, + /// Someone is blocked receiving + Recving, +} + +impl Endpoint { + fn handle_call( + &mut self, + caller: &mut Domain, + msg: &[u64; 6], + cap_slot: Option, + badge: u64, + ) -> SyscallResult { + match self.state { + EndpointState::Recving => { + // Receiver waiting! Fast path - direct switch + let receiver = self.wait_queue.pop_front().unwrap(); + + // Copy message to receiver + self.msg_regs = *msg; + self.sender_badge = badge; + self.transfer_cap = cap_slot; + + // Block caller waiting for reply + caller.state = DomainState::Blocked; + caller.block_reason = BlockReason::Endpoint; + + // Wake receiver + receiver.state = DomainState::Runnable; + + // Switch to receiver (donate remaining time) + switch_to(receiver); + + Ok(0) + } + + EndpointState::Idle | EndpointState::Sending => { + // No receiver - block caller + caller.state = DomainState::Blocked; + caller.block_reason = BlockReason::Endpoint; + + self.wait_queue.push_back(caller.id); + self.state = EndpointState::Sending; + + // Store message for when receiver arrives + self.msg_regs = *msg; + self.sender_badge = badge; + self.transfer_cap = cap_slot; + + // Schedule someone else + schedule_next(); + + Ok(0) + } + } + } + + fn handle_recv(&mut self, receiver: &mut Domain, reply_dest_slot: CapSlot) -> SyscallResult { + match self.state { + EndpointState::Sending => { + let sender_info = self.wait_queue.pop_front().unwrap(); + + // Create Reply object for this call + let reply = Reply { + caller: sender_info.domain_id, + state: ReplyState::Pending, + }; + + // Allocate kernel memory for Reply object - FIXME: kernel memory allocation! + // and place capability in receiver's cspace + let reply_cap = kernel_alloc_reply(reply)?; + receiver.cspace.insert(reply_dest_slot, reply_cap)?; + + // Copy message to receiver + receiver.regs.x0 = 0; // Success + receiver.regs.x1 = sender_info.badge; + receiver.regs.x2 = self.msg_regs[0]; // label + receiver.regs.x3 = self.msg_regs[1]; + receiver.regs.x4 = self.msg_regs[2]; + receiver.regs.x5 = self.msg_regs[3]; + receiver.regs.x6 = self.msg_regs[4]; + receiver.regs.x7 = self.msg_regs[5]; + // x8 = transferred cap slot (if any) + + if self.wait_queue.is_empty() { + self.state = EndpointState::Idle; + } + + Ok(0) + } + + EndpointState::Idle | EndpointState::Recving => { + // Block receiver + receiver.state = DomainState::Blocked; + receiver.block_reason = BlockReason::Endpoint; + receiver.blocked_data = reply_dest_slot as u64; // Remember where to put reply cap + + self.wait_queue.push_back(receiver.id); + self.state = EndpointState::Recving; + + schedule_next(); + Ok(0) + } + } + } + + fn handle_reply_recv( + &mut self, + server: &mut Domain, + reply_cap_slot: CapSlot, + next_reply_slot: CapSlot, + reply_msg: &[u64; 6], + ) -> SyscallResult { + // 1. Send reply via the provided reply cap + let reply_cap = server.cspace.take(reply_cap_slot)?; + let reply = reply_cap.as_reply()?; + reply.handle_send(reply_msg, None)?; + + // Reply object is consumed, free kernel memory + kernel_free_reply(reply); + + // 2. Receive next message + self.handle_recv(server, next_reply_slot) + } +} + +impl NucleusObject for Endpoint { + const TYPE: ObjectType = ObjectType::Endpoint; +} diff --git a/kernel/nucleus/src/objects/event_count.rs b/kernel/nucleus/src/objects/event_count.rs new file mode 100644 index 000000000..607744500 --- /dev/null +++ b/kernel/nucleus/src/objects/event_count.rs @@ -0,0 +1,12 @@ +// ==================== +// == Nucleus object == +// ==================== + +struct EventCount { + value: u64, // Monotonically increasing counter + waiters: WaitQueue, // Domains waiting for value >= target +} + +impl NucleusObject for EventCount { + const TYPE: ObjectType = ObjectType::EventCount; +} diff --git a/kernel/nucleus/src/objects/key_table.rs b/kernel/nucleus/src/objects/key_table.rs new file mode 100644 index 000000000..060c41b24 --- /dev/null +++ b/kernel/nucleus/src/objects/key_table.rs @@ -0,0 +1,105 @@ +use { + crate::{api::key_entry::KeyEntry, objects::NucleusObject}, + libobject::{CapError, KeySlot, ObjectType, domain::DomainId}, +}; + +// ==================== +// == Nucleus object == +// ==================== + +/// A capability table for a domain. +/// +/// This is what seL4 calls a `CNode`. Each domain has one. +/// The table itself is a kernel object that can be referenced +/// by capabilities (for capability space manipulation). +pub struct KeyTable { + /// The actual capability entries + entries: [KeyEntry; Self::NUM_SLOTS], + /// Domain that owns this table + owner: DomainId, + /// Number of valid entries (for iteration) + count: u16, +} + +impl KeyTable { + /// Number of slots per table (power of 2 for fast indexing) + pub const NUM_SLOTS: usize = 256; + + /// Create a new empty capability table + pub fn new(owner: DomainId) -> Self { + Self { + entries: [const { KeyEntry::null() }; Self::NUM_SLOTS], + owner, + count: 0, + } + } + + /// Lookup a capability by slot index + #[inline] + pub fn lookup(&self, slot: KeySlot) -> Result<&KeyEntry, CapError> { + let idx = slot.0 as usize; + if idx >= Self::NUM_SLOTS { + return Err(CapError::InvalidSlot(slot)); + } + + let entry = &self.entries[idx]; + if !entry.is_valid() { + return Err(CapError::EmptySlot(slot)); + } + + Ok(entry) + } + + /// Lookup a capability mutably + #[inline] + pub fn lookup_mut(&mut self, slot: KeySlot) -> Result<&mut KeyEntry, CapError> { + let idx = slot.0 as usize; + if idx >= Self::NUM_SLOTS { + return Err(CapError::InvalidSlot(slot)); + } + + let entry = &mut self.entries[idx]; + if !entry.is_valid() { + return Err(CapError::EmptySlot(slot)); + } + + Ok(entry) + } + + /// Insert a capability at a specific slot + pub fn insert(&mut self, slot: KeySlot, entry: KeyEntry) -> Result<(), CapError> { + let idx = slot.0 as usize; + if idx >= Self::NUM_SLOTS { + return Err(CapError::InvalidSlot(slot)); + } + + if self.entries[idx].is_valid() { + return Err(CapError::SlotOccupied(slot)); + } + + self.entries[idx] = entry; + self.count += 1; + Ok(()) + } + + /// Remove a capability from a slot + pub fn remove(&mut self, slot: KeySlot) -> Result { + let idx = slot.0 as usize; + if idx >= Self::NUM_SLOTS { + return Err(CapError::InvalidSlot(slot)); + } + + let entry = core::mem::replace(&mut self.entries[idx], KeyEntry::null()); + if entry.is_valid() { + self.count -= 1; + Ok(entry) + } else { + Err(CapError::EmptySlot(slot)) + } + } +} + +// KeyTable is itself a kernel object +impl NucleusObject for KeyTable { + const TYPE: ObjectType = ObjectType::KEY_TABLE; +} diff --git a/kernel/nucleus/src/objects/mod.rs b/kernel/nucleus/src/objects/mod.rs new file mode 100644 index 000000000..909498a54 --- /dev/null +++ b/kernel/nucleus/src/objects/mod.rs @@ -0,0 +1,14 @@ +pub mod arch; +pub mod arch_objects; +pub mod debug_console; +pub mod domain; +pub mod key_table; +pub mod nucleus; +pub mod nucleus_object; +pub mod object_pool; +pub mod object_ref; + +pub use { + arch::ArchObjectsImpl, arch_objects::ArchObjects, debug_console::DebugConsole, domain::Domain, + key_table::KeyTable, nucleus::Nucleus, nucleus_object::NucleusObject, object_pool::ObjectPool, +}; diff --git a/kernel/nucleus/src/objects/notification.rs b/kernel/nucleus/src/objects/notification.rs new file mode 100644 index 000000000..ba46e3ef3 --- /dev/null +++ b/kernel/nucleus/src/objects/notification.rs @@ -0,0 +1,22 @@ +// ==================== +// == Nucleus object == +// ==================== + +struct Notification { + state: u64, // Bitmap + waiters: WaitQueue, // Blocked domains + bound: Option, // Optional bound domain for fast wakeup +} + +impl Notification { + fn signal(&mut self, bits: u64) {} + fn wait() { + // if bits are already set, clear and immediately return + // otherwise block the domain.. + } + fn poll() {} +} + +impl NucleusObject for Notification { + const TYPE: ObjectType = ObjectType::Notification; +} diff --git a/kernel/nucleus/src/objects/nucleus.rs b/kernel/nucleus/src/objects/nucleus.rs new file mode 100644 index 000000000..d06ed8b9c --- /dev/null +++ b/kernel/nucleus/src/objects/nucleus.rs @@ -0,0 +1,236 @@ +use { + crate::{ + api::key_entry::KeyEntry, + objects::{ + ArchObjects, DebugConsole, Domain, KeyTable, ObjectPool, arch::ArchPools, + domain::DcbPages, + }, + }, + core::sync::atomic::Ordering, + libobject::{ + KeySlot, + domain::{BlockReason, DomainControlBlock, DomainId, DomainState}, + }, +}; + +// ┌─────────────────────────────────────────────────────────────────────┐ +// │ KERNEL TYPE STRUCTURE │ +// ├─────────────────────────────────────────────────────────────────────┤ +// │ │ +// │ Kernel │ +// │ │ │ +// │ ├── pools: KernelPools │ +// │ │ ├── untypeds: ObjectPool │ +// │ │ ├── domains: ObjectPool │ +// │ │ ├── keytables: ObjectPool │ +// │ │ ├── notifications: ObjectPool │ +// │ │ ├── event_counts: ObjectPool │ +// │ │ ├── endpoints: ObjectPool │ +// │ │ ├── time_slices: ObjectPool │ +// │ │ ├── buffers: ObjectPool │ +// │ │ ├── replies: ObjectPool │ +// │ │ │ │ +// │ │ └── arch: ArchPools │ +// │ │ ├── frames: ObjectPool │ +// │ │ ├── page_tables: ObjectPool │ +// │ │ ├── vspaces: ObjectPool │ +// │ │ ├── asid_pools: ObjectPool │ +// │ │ └── asids: ObjectPool │ +// │ │ │ +// │ ├── current_domain: Option │ +// │ └── dcb_pages: DcbPages │ +// │ │ +// └─────────────────────────────────────────────────────────────────────┘ + +// ═══════════════════════════════════════════════════════════════════ +// UNIFIED KERNEL OBJECT MANAGEMENT +// ═══════════════════════════════════════════════════════════════════ + +/// All kernel object pools - both core and architecture-specific +pub struct NucleusPools { + // ─── Core Object Pools ─── + // pub untypeds: ObjectPool, + pub domains: ObjectPool, + // pub keytables: ObjectPool, + // pub notifications: ObjectPool, + // pub event_counts: ObjectPool, + // pub endpoints: ObjectPool, + // pub time_slices: ObjectPool, + // pub buffers: ObjectPool, + // pub replies: ObjectPool, + + // ─── Architecture-Specific Pools ─── + pub arch: ArchPools, +} + +/// Complete nucleus state (parameterized by architecture) +pub struct Nucleus { + /// All object pools + pub pools: NucleusPools, + /// Currently running domain + pub current_domain: Option, // FIXME: not option, always something (Idle or other) + /// DCB shared pages + pub dcb_pages: DcbPages, +} + +// ═══════════════════════════════════════════════════════════════════ +// KERNEL INTEGRATION +// ═══════════════════════════════════════════════════════════════════ + +impl Nucleus { + #[expect(clippy::unused_self)] + pub fn current_cpu(&self) -> usize { + 0 + } + + #[expect(clippy::unused_self)] + pub fn current_time_ns(&self) -> u64 { + 0 + } + + /// Nucleus-private domain data, like keytables + pub fn current_domain_mut(&mut self) -> Option<&mut Domain> { + // need objects::Domain here, not DCB! or a tuple + self.pools + .domains + .get_mut(self.current_domain.unwrap_or(0) as usize) + } + + /// User-visible DCB + pub fn current_dcb_mut(&mut self) -> Option<&mut DomainControlBlock> { + // need objects::Domain here, not DCB! or a tuple + self.dcb_pages + .get_mut(DomainId(self.current_domain.unwrap_or(0))) + } + + // TODO: Testing fixture + pub fn create_domain(&mut self) { + self.pools + .domains + .allocate(Domain { + keytable: KeyTable::new(DomainId(0)), + }) + .and_then(|dom| { + dom.keytable + .insert( + libobject::KeySlot(127), + KeyEntry::new(&DebugConsole, libobject::Rights::all(), 0), + ) + .ok() + }) + .expect("Poof"); + } + + /// Update DCB when domain is activated + pub fn activate_domain(&mut self, id: DomainId, time_budget_ns: u64) { + let cpu = self.current_cpu(); + let time = self.current_time_ns(); + if let Some(dcb) = self.dcb_pages.get_mut(id) { + // Update time budget + dcb.time_remaining_ns + .store(time_budget_ns, Ordering::Relaxed); + dcb.last_activated_ns.store(time, Ordering::Relaxed); + dcb.activation_count.fetch_add(1, Ordering::Relaxed); + dcb.cpu + .store(u32::try_from(cpu).unwrap(), Ordering::Relaxed); + + // Set state last (Release ensures all above writes are visible) + dcb.state + .store(DomainState::Running as u32, Ordering::Release); + } + } + + /// Update DCB when domain yields/blocks/faults + pub fn deactivate_domain(&mut self, id: DomainId, reason: DeactivateReason) { + let elapsed = 0; //self.time_since_activation(id); + + if let Some(dcb) = self.dcb_pages.get_mut(id) { + // Update time accounting + dcb.time_consumed_ns.fetch_add(elapsed, Ordering::Relaxed); + dcb.time_remaining_ns.fetch_sub( + elapsed.min(dcb.time_remaining_ns.load(Ordering::Relaxed)), + Ordering::Relaxed, + ); + + // Update state based on reason + match reason { + DeactivateReason::TimeExhausted | DeactivateReason::Yielded => { + dcb.state + .store(DomainState::Runnable as u32, Ordering::Release); + } + + DeactivateReason::Blocked { reason, slot } => { + dcb.block_reason.store(reason as u32, Ordering::Relaxed); + dcb.blocked_on_slot.store(slot.0, Ordering::Relaxed); + dcb.state + .store(DomainState::Blocked as u32, Ordering::Release); + } + + DeactivateReason::Faulted { + fault_type, + code, + addr, + slot, + } => { + dcb.fault_type.store(fault_type as u32, Ordering::Relaxed); + dcb.fault_code.store(code, Ordering::Relaxed); + dcb.fault_addr.store(addr, Ordering::Relaxed); + dcb.fault_slot.store(slot.0, Ordering::Relaxed); + dcb.state + .store(DomainState::Faulted as u32, Ordering::Release); + } + + DeactivateReason::Suspended => { + dcb.state + .store(DomainState::Suspended as u32, Ordering::Release); + } + } + } + } + + /// Update DCB when notification is signaled to a domain + pub fn signal_notification(&mut self, id: DomainId, slot: KeySlot, bits: u64) { + if let Some(dcb) = self.dcb_pages.get_mut(id) { + // OR the notification bits + dcb.pending_notifications + .fetch_or(1 << slot.0, Ordering::Release); + + // If domain was blocked on notifications, make it runnable + let state = dcb.state.load(Ordering::Acquire); + let block_reason = dcb.block_reason.load(Ordering::Relaxed); + + if state == DomainState::Blocked as u32 + && block_reason == BlockReason::Notification as u32 + { + dcb.state + .store(DomainState::Runnable as u32, Ordering::Release); + } + } + } +} + +pub enum DeactivateReason { + TimeExhausted, + Blocked { + reason: BlockReason, + slot: KeySlot, + }, + Faulted { + fault_type: FaultType, + code: u32, + addr: u64, + slot: KeySlot, + }, + Suspended, + Yielded, +} + +#[repr(u32)] +pub enum FaultType { + None = 0, + PageFault = 1, + CapFault = 2, + UnknownSyscall = 3, + UserException = 4, + VMFault = 5, +} diff --git a/kernel/nucleus/src/objects/nucleus_object.rs b/kernel/nucleus/src/objects/nucleus_object.rs new file mode 100644 index 000000000..a87a3ec27 --- /dev/null +++ b/kernel/nucleus/src/objects/nucleus_object.rs @@ -0,0 +1,90 @@ +//! Kernel object storage and capability lookup +//! +//! Design goals: +//! 1. Compact `KeyEntry` (fits in cache line) +//! 2. Type-safe access from handlers +//! 3. Objects live in typed pools (good for allocation) +//! 4. Support for derivation/revocation tree + +// ┌─────────────────────────────────────────────────────────────────────┐ +// │ ARCHITECTURE-SPECIFIC OBJECTS │ +// ├─────────────────────────────────────────────────────────────────────┤ +// │ │ +// │ Generic Kernel Objects Architecture-Specific Objects │ +// │ ───────────────────────── ───────────────────────────────── │ +// │ │ +// │ • Untyped AArch64: │ +// │ • Domain • Frame (4KB, 2MB, 1GB pages) │ +// │ • KeyTable • PageTable (translation table) │ +// │ • Notification • VSpace (TTBR0/TTBR1 root) │ +// │ • EventCount • ASIDPool (ASID allocation) │ +// │ • Endpoint • ASID (address space ID) │ +// │ • Time • IOSpace (SMMU for devices) │ +// │ • Buffer │ +// │ • Reply x86_64: │ +// │ • Frame (4KB, 2MB, 1GB pages) │ +// │ • PageTable (PML4/PDPT/PD/PT) │ +// │ • VSpace (CR3 root) │ +// │ • IOPort (x86 I/O ports) │ +// │ • IOSpace (VT-d for devices) │ +// │ │ +// │ RISC-V: │ +// │ • Frame (4KB, 2MB, 1GB) │ +// │ • PageTable (Sv39/Sv48) │ +// │ • VSpace (satp root) │ +// │ │ +// └─────────────────────────────────────────────────────────────────────┘ + +// ┌─────────────────────────────────────────────────────────────────────┐ +// │ OBJECT TYPE HIERARCHY │ +// ├─────────────────────────────────────────────────────────────────────┤ +// │ │ +// │ ObjectType (u8) │ +// │ ├── Core Types (0-15) │ +// │ │ ├── 0: Null │ +// │ │ ├── 1: Untyped ─→ Untyped struct │ +// │ │ ├── 2: Domain ─→ Domain struct │ +// │ │ ├── 3: KeyTable ─→ KeyTable struct │ +// │ │ ├── 4: Notification ─→ Notification struct │ +// │ │ ├── 5: EventCount ─→ EventCount struct │ +// │ │ ├── 6: Endpoint ─→ Endpoint struct │ +// │ │ ├── 7: Time ─→ TimeSlice struct │ +// │ │ ├── 8: Buffer ─→ Buffer struct │ +// │ │ └── 9: Reply ─→ Reply struct │ +// │ │ │ +// │ └── Arch Types (16-63) ──────────────────────────────────────────┐ │ +// │ │ │ │ +// │ │ ┌─────────────────────────────────────────────────────┐ │ │ +// │ │ │ impl ArchObjects for AArch64 │ │ │ +// │ │ │ type Frame = AArch64Frame │ │ │ +// │ │ │ type PageTable = AArch64PageTable │ │ │ +// │ │ │ type VSpace = AArch64VSpace │ │ │ +// │ │ │ type ASIDPool = AArch64ASIDPool │ │ │ +// │ │ │ type ASID = AArch64ASID │ │ │ +// │ │ └─────────────────────────────────────────────────────┘ │ │ +// │ │ │ │ +// │ ├── 16: Frame ─→ A::Frame │ │ +// │ ├── 17: PageTable ─→ A::PageTable │ │ +// │ ├── 18: VSpace ─→ A::VSpace │ │ +// │ ├── 19: ASIDPool ─→ A::ASIDPool │ │ +// │ ├── 20: ASID ─→ A::ASID │ │ +// │ ├── 21: IOSpace ─→ (SMMU/VT-d specific) │ │ +// │ ├── 22: IOPort ─→ (x86 only) │ │ +// │ ├── 23: IRQHandler ─→ IRQ binding │ │ +// │ └── 24: IRQControl ─→ IRQ management │ │ +// │ │ +// └─────────────────────────────────────────────────────────────────────┘ + +// ═══════════════════════════════════════════════════════════════════ +// KERNEL OBJECT TRAIT +// ═══════════════════════════════════════════════════════════════════ + +/// Marker trait for kernel objects - provides type → `ObjectType` mapping +pub trait NucleusObject: Sized + 'static { + const TYPE: libobject::ObjectType; + + //TODO: add invoke here? + // fn invoke(obj: &Self::TYPE, op: u32, args: &[u64]) -> SyscallResult; +} + +// Should object type live here or in libobject? Seems like here is a better place but CapErrors need to refer to object types. diff --git a/kernel/nucleus/src/objects/object_pool.rs b/kernel/nucleus/src/objects/object_pool.rs new file mode 100644 index 000000000..e29219b24 --- /dev/null +++ b/kernel/nucleus/src/objects/object_pool.rs @@ -0,0 +1,128 @@ +use crate::objects::NucleusObject; + +// ═══════════════════════════════════════════════════════════════════ +// OBJECT POOLS +// ═══════════════════════════════════════════════════════════════════ + +// FIXME: allocate a whole pool (X objects of same type) via untyped retype and then get objects from pool as needed + +/// A pool of kernel objects of type T, backed by untyped memory. +/// +/// Objects are allocated via Untyped.Retype and live until revoked. +pub struct ObjectPool { + /// Base address of the pool + base: *mut T, + /// Bitmap of allocated slots + allocated: [u64; 4], // 256 objects max per pool + /// Number of allocated objects + count: u16, + /// Total capacity + capacity: u16, +} + +impl ObjectPool { + /// Create a new pool backed by untyped memory + /// + /// # Safety + /// The untyped memory must be properly sized and aligned for T + pub unsafe fn new(memory: *mut u8, size: usize) -> Self { + let capacity = u16::try_from(size / core::mem::size_of::()).unwrap(); + assert!(capacity <= 256); + + Self { + base: memory.cast(), + allocated: [0; 4], + count: 0, + capacity, + } + } + + /// Allocate an object in the pool, returning a reference + pub fn allocate(&mut self, init: T) -> Option<&mut T> { + // Find free slot + let slot = self.find_free_slot()?; + + // Mark as allocated + let word = slot / 64; + let bit = slot % 64; + self.allocated[word] |= 1 << bit; + self.count += 1; + + // Initialize the object + // SAFETY: Unsafe + unsafe { + let ptr = self.base.add(slot); + ptr.write(init); + Some(&mut *ptr) + } + } + + /// Get a reference to an allocated object by index + pub fn get(&self, index: usize) -> Option<&T> { + if index >= self.capacity as usize { + return None; + } + + let word = index / 64; + let bit = index % 64; + if self.allocated[word] & (1 << bit) == 0 { + return None; + } + + // SAFETY: Unsafe + unsafe { Some(&*self.base.add(index)) } + } + + /// Get a mutable reference to an allocated object + pub fn get_mut(&mut self, index: usize) -> Option<&mut T> { + if index >= self.capacity as usize { + return None; + } + + let word = index / 64; + let bit = index % 64; + if self.allocated[word] & (1 << bit) == 0 { + return None; + } + + // SAFETY: Unsafe + unsafe { Some(&mut *self.base.add(index)) } + } + + /// Deallocate an object + pub fn deallocate(&mut self, index: usize) -> bool { + if index >= self.capacity as usize { + return false; + } + + let word = index / 64; + let bit = index % 64; + if self.allocated[word] & (1 << bit) == 0 { + return false; + } + + self.allocated[word] &= !(1 << bit); + self.count -= 1; + + // Drop the object + // SAFETY: Unsafe + unsafe { + core::ptr::drop_in_place(self.base.add(index)); + } + + true + } + + fn find_free_slot(&self) -> Option { + for (word_idx, &word) in self.allocated.iter().enumerate() { + if word != !0 { + let bit = word.trailing_ones() as usize; + let slot = word_idx * 64 + bit; + if slot < self.capacity as usize { + return Some(slot); + } + } + } + None + } +} diff --git a/kernel/nucleus/src/objects/object_ref.rs b/kernel/nucleus/src/objects/object_ref.rs new file mode 100644 index 000000000..1eab8f0b5 --- /dev/null +++ b/kernel/nucleus/src/objects/object_ref.rs @@ -0,0 +1,97 @@ +use { + super::NucleusObject, + core::ptr::NonNull, + libobject::{CapError, object_type::ObjectType}, +}; + +// ═══════════════════════════════════════════════════════════════════ +// TYPE-ERASED OBJECT POINTER +// ═══════════════════════════════════════════════════════════════════ + +/// A type-erased pointer to a kernel object, with its type tag. +/// +/// This is the "fat pointer" alternative - we store the type alongside +/// the pointer so we can safely cast it back. +#[derive(Clone, Copy)] +pub struct ObjectRef { + ptr: NonNull<()>, + obj_type: ObjectType, +} + +impl ObjectRef { + /// Create a new object reference from a typed pointer + pub fn new(obj: &T) -> Self { + Self { + ptr: NonNull::from(obj).cast(), + obj_type: T::TYPE, + } + } + + pub const fn null() -> Self { + Self { + ptr: NonNull::dangling(), + obj_type: ObjectType::NULL, + } + } + + /// Create from a mutable pointer (for objects in pools) + /// + /// # Safety + /// Caller must ensure the pointer is valid and properly aligned + pub unsafe fn from_raw(ptr: *mut T) -> Self { + Self { + // SAFETY: Unsafe + ptr: unsafe { NonNull::new_unchecked(ptr.cast()) }, + obj_type: T::TYPE, + } + } + + /// Get the object type + #[inline] + pub fn object_type(&self) -> ObjectType { + self.obj_type + } + + /// Get the raw type-erased pointer (for embedding in `KeyEntry` payload). + #[inline] + pub fn as_raw_ptr(&self) -> NonNull<()> { + self.ptr + } + + /// Attempt to cast to a specific type (immutable) + #[inline] + pub fn try_as(&self) -> Option<&T> { + (self.obj_type == T::TYPE).then( + // SAFETY: We verified the type matches + || unsafe { self.ptr.cast::().as_ref() }, + ) + } + + /// Attempt to cast to a specific type (mutable) + #[inline] + pub fn try_as_mut(&mut self) -> Option<&mut T> { + (self.obj_type == T::TYPE).then( + // SAFETY: We verified the type matches + || unsafe { self.ptr.cast::().as_mut() }, + ) + } + + /// Cast with error on type mismatch + #[inline] + pub fn as_type(&self) -> Result<&T, CapError> { + self.try_as().ok_or(CapError::TypeMismatch { + expected: T::TYPE, + found: self.obj_type, + }) + } + + /// Cast with error on type mismatch (mutable) + #[inline] + pub fn as_type_mut(&mut self) -> Result<&mut T, CapError> { + let found = self.obj_type; + self.try_as_mut().ok_or(CapError::TypeMismatch { + expected: T::TYPE, + found, + }) + } +} diff --git a/kernel/nucleus/src/objects/reply.rs b/kernel/nucleus/src/objects/reply.rs new file mode 100644 index 000000000..4c4d11bf5 --- /dev/null +++ b/kernel/nucleus/src/objects/reply.rs @@ -0,0 +1,70 @@ +// ==================== +// == Nucleus object == +// ==================== + +/// Reply kernel object +struct Reply { + /// Domain waiting for this reply + caller: DomainId, + + /// State of this reply object + state: ReplyState, +} + +#[derive(Clone, Copy, PartialEq)] +enum ReplyState { + /// Caller is blocked waiting + Pending, + /// Reply was sent, object is consumed + Used, + /// Caller cancelled or timed out + Cancelled, +} + +impl Reply { + fn handle_send(&mut self, msg: &[u64; 6], cap_slot: Option) -> SyscallResult { + match self.state { + ReplyState::Pending => { + let caller = get_domain_mut(self.caller); + + // Copy reply to caller's registers + caller.regs.x0 = 0; // Success + caller.regs.x1 = msg[0]; // label + caller.regs.x2 = msg[1]; + caller.regs.x3 = msg[2]; + caller.regs.x4 = msg[3]; + caller.regs.x5 = msg[4]; + caller.regs.x6 = msg[5]; + + // Transfer cap if present + if let Some(slot) = cap_slot { + let cap = current_domain().cspace.take(slot)?; + caller.cspace.insert(RECEIVED_CAP_SLOT, cap)?; + caller.regs.x7 = RECEIVED_CAP_SLOT as u64; + } else { + caller.regs.x7 = u64::MAX; + } + + // Wake caller + caller.state = DomainState::Runnable; + + // Mark reply object as used (one-shot) + self.state = ReplyState::Used; + + Ok(0) + } + + ReplyState::Used => Err(SyscallError::ReplyAlreadyUsed), + + ReplyState::Cancelled => { + // Caller gave up - just consume the reply cap + self.state = ReplyState::Used; + Ok(0) // Not an error, just no-op + } + } + } +} + +impl NucleusObject for Reply { + const TYPE: ObjectType = ObjectType::Reply; +} diff --git a/kernel/nucleus/src/objects/time.rs b/kernel/nucleus/src/objects/time.rs new file mode 100644 index 000000000..0f845dad0 --- /dev/null +++ b/kernel/nucleus/src/objects/time.rs @@ -0,0 +1,13 @@ +// ==================== +// == Nucleus object == +// ==================== + +struct Time { + remaining_us: u64, // Microseconds left + deadline: Instant, // When this slice expires + parent: Option, // For custom revocation tree (to easily return unused time to parent) +} + +impl NucleusObject for TimeSlice { + const TYPE: ObjectType = ObjectType::Time; +} diff --git a/kernel/nucleus/src/objects/untyped.rs b/kernel/nucleus/src/objects/untyped.rs new file mode 100644 index 000000000..3646e092f --- /dev/null +++ b/kernel/nucleus/src/objects/untyped.rs @@ -0,0 +1,170 @@ +// The key insight: region capabilities (Untyped, Frame) stored inline in KeyEntry +// ARE the kernel object. No separate in-kernel structure, no pool allocation, +// no pointer indirection — just the RegionPayload in the capability slot. +// +// ┌─────────────────────────────────────────────────────────────────────┐ +// │ Region capabilities (inline in KeyEntry) │ +// ├──────────────────┬──────────────────────────────────────────────────┤ +// │ obj_type │ Untyped or Frame (in KeyEntry header) │ +// ├──────────────────┼──────────────────────────────────────────────────┤ +// │ rights │ Access rights (in KeyEntry header) │ +// ├──────────────────┼──────────────────────────────────────────────────┤ +// │ paddr │ Physical address of region │ +// ├──────────────────┼──────────────────────────────────────────────────┤ +// │ state │ Untyped: watermark | Frame: map_count │ +// ├──────────────────┼──────────────────────────────────────────────────┤ +// │ size_bits │ Total size in bits (region = 2^size_bits) │ +// ├──────────────────┼──────────────────────────────────────────────────┤ +// │ is_device │ Boolean: is this device memory? │ +// └──────────────────┴──────────────────────────────────────────────────┘ + +use { + crate::{ + api::key_entry::KeyEntry, + objects::{ArchObjects, NucleusPools, key_table::KeyTable}, + }, + libaddress::PhysAddr, + libobject::{ArchType, CapError, KeySlot, ObjectType, Rights}, +}; + +// ═══════════════════════════════════════════════════════════════════ +// RETYPE FROM UNTYPED +// ═══════════════════════════════════════════════════════════════════ +// +// UNTYPED REGION (2^size_bits bytes) +// ┌────────────────────────────────────────────────────────────┐ +// │█████████████████████████░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░│ +// │ ALLOCATED │ FREE │ +// └────────────────────────────────────────────────────────────┘ +// ↑ ↑ ↑ +// paddr paddr + watermark paddr + 2^size_bits + +/// Retype from an untyped KeyEntry into a new nucleus object. +/// +/// `src_entry` must be an Untyped capability. The new object is allocated +/// from the untyped region and a capability is inserted into `dest_keytable` +/// at `dest_slot`. +/// +/// Region types (Untyped, Frame) are created inline — no pool allocation. +/// Other arch types (PageTable, VSpace, etc.) still go through arch pools. +pub fn retype( + src_entry: &mut KeyEntry, + obj_type: ObjectType, + size_bits: u8, + dest_slot: KeySlot, + dest_keytable: &mut KeyTable, + pools: &mut NucleusPools, +) -> Result<(), CapError> { + let ut = src_entry.as_untyped_mut()?; + + // Determine object size based on type + let obj_size = object_size::(obj_type, size_bits)?; + + // Align the watermark to the object size + let wm = ut.watermark_bytes(); + let aligned_wm = (wm + obj_size - 1) & !(obj_size - 1); + + // Check we have enough memory + if aligned_wm + obj_size > ut.size() { + return Err(CapError::InsufficientMemory); + } + + let obj_paddr = ut.paddr + aligned_wm as u64; + let is_device = ut.is_device; + + // Advance watermark + ut.set_watermark_bytes(aligned_wm + obj_size); + + // Create the capability for the new object + let entry = create_object::(obj_type, obj_paddr, size_bits, is_device, pools)?; + + dest_keytable.insert(dest_slot, entry)?; + + Ok(()) +} + +/// Create a capability for a newly retyped object. +/// +/// Inline region types (Untyped, Frame) are written directly into the KeyEntry. +/// Pool-backed types go through their respective allocators. +fn create_object( + obj_type: ObjectType, + paddr: u64, + size_bits: u8, + is_device: bool, + pools: &mut NucleusPools, +) -> Result { + match obj_type { + // ── Inline region types ── + ObjectType::Untyped => Ok(KeyEntry::new_untyped( + paddr, + size_bits, + is_device, + Rights::all(), + )), + + ObjectType::FRAME => { + let rights = Rights::READ | Rights::WRITE; + Ok(KeyEntry::new_frame(paddr, size_bits, is_device, rights)) + } + + // ── Pool-backed core types ── + + // ObjectType::Notification => { + // let obj = pools.notifications.allocate(Notification::new()) + // .ok_or(CapError::PoolExhausted)?; + // Ok(KeyEntry::new(obj, Rights::all(), 0)) + // } + // ObjectType::Endpoint => { + // let obj = pools.endpoints.allocate(Endpoint::new()) + // .ok_or(CapError::PoolExhausted)?; + // Ok(KeyEntry::new(obj, Rights::all(), 0)) + // } + // ... etc for other pool-backed core types + + // ── Pool-backed arch types (PageTable, VSpace, ASID, etc.) ── + _ if obj_type.is_arch() => { + let arch_type = ArchType::try_from(obj_type)?; + let obj_ref = A::create_arch_object( + arch_type, + PhysAddr::from(paddr), + size_bits, + &mut pools.arch, + )?; + Ok(KeyEntry::from_ref(obj_ref, Rights::all(), 0)) + } + + _ => Err(CapError::InvalidObjectType), + } +} + +/// Get the physical size of an object to be created. +fn object_size(obj_type: ObjectType, size_bits: u8) -> Result { + match obj_type { + // Variable-size region types + ObjectType::Buffer | ObjectType::Untyped => { + if size_bits > 30 { + Err(CapError::InvalidSize) + } else { + Ok(1usize << size_bits) + } + } + + // Frame size is validated by the arch layer + ObjectType::FRAME => A::validate_frame_size(size_bits), + + // Fixed-size core types + // ObjectType::Notification => Ok(core::mem::size_of::()), + // ObjectType::Endpoint => Ok(core::mem::size_of::()), + // ObjectType::Domain => Ok(4096), + // ... etc + + // Arch types validated by arch layer + _ if obj_type.is_arch() => { + let arch_type = ArchType::try_from(obj_type)?; + A::validate_retype(arch_type, size_bits) + } + + _ => Err(CapError::InvalidObjectType), + } +} diff --git a/libs/address/Cargo.toml b/libs/address/Cargo.toml new file mode 100644 index 000000000..95046acfa --- /dev/null +++ b/libs/address/Cargo.toml @@ -0,0 +1,38 @@ +[package] +name = "vesper-address" + +description = "Vesper nanokernel memory address types." + +authors = { workspace = true } +categories = { workspace = true } +documentation = { workspace = true } +edition = { workspace = true } +homepage = { workspace = true } +license = { workspace = true } +readme = { workspace = true } +repository = { workspace = true } +version = { workspace = true } + +publish = false + +[badges] +maintenance = { status = "experimental" } + +[dependencies] +arbitrary-int = { workspace = true } +bitbybit = { workspace = true } +num = { workspace = true } +usize_conversions = { workspace = true } + +[dev-dependencies] +libboot = { workspace = true } +libconsole = { workspace = true } +liblog = { workspace = true } +libqemu = { workspace = true } +libtest = { workspace = true } + +# [lib] +# test = false + +[lints] +workspace = true diff --git a/libs/address/README.md b/libs/address/README.md new file mode 100644 index 000000000..6cecaea86 --- /dev/null +++ b/libs/address/README.md @@ -0,0 +1,31 @@ +# libaddress + +The types Address and Address represent the addresses before and after the mapping in the MMU. + +## Arch-independent + +Address is a 64-bit unsigned number (_address_ of a location in memory) + +It can be aligned and checked for alignment. +There are not other limitations on the address. + +## Arch-dependent + +Address may have a size limitation, e.g. a 40-bits physical address on some platforms. - could be platform-dependent, not arch-dependent! + +Address may contain additional information payload, for example ASID tags. + +## Exports + +This library should export the following: + +- Type `Address` representing the physical memory address, independent of the target platform + (plus some shared operations like NumOps). +- Type `PhysAddr` specific to the target platform, with size and content limitations. +- Type `Address` representing the virtual memory address, independent of the target platform +- plus some shared operations like NumOps or ToPointer/FromPointer ops. +- Type `VirtAddr` specific to the target platform (e.g. with_asid() on aarch64) + +---- + +For more information please re-read. diff --git a/libs/memory/src/mm/mod.rs b/libs/address/src/align.rs similarity index 50% rename from libs/memory/src/mm/mod.rs rename to libs/address/src/align.rs index f328c83af..c3ec2cd6e 100644 --- a/libs/memory/src/mm/mod.rs +++ b/libs/address/src/align.rs @@ -1,17 +1,18 @@ -/* - * SPDX-License-Identifier: BlueOak-1.0.0 - * Copyright (c) Berkus Decker - */ - -mod bump_allocator; -pub use bump_allocator::BumpAllocator; +/// Align address downwards. +/// +/// Returns the greatest x with alignment `align` so that x <= addr. +/// The alignment must be a power of 2. +#[inline(always)] +pub const fn align_down_bits(addr: u64, alignment_bits: u64) -> u64 { + addr & !((1 << alignment_bits) - 1) +} /// Align address downwards. /// /// Returns the greatest x with alignment `align` so that x <= addr. /// The alignment must be a power of 2. #[inline(always)] -pub const fn align_down(addr: usize, alignment: usize) -> usize { +pub const fn align_down(addr: u64, alignment: u64) -> u64 { assert!( alignment.is_power_of_two(), "`alignment` must be a power of two" @@ -24,7 +25,21 @@ pub const fn align_down(addr: usize, alignment: usize) -> usize { /// Returns the smallest x with alignment `align` so that x >= addr. /// The alignment must be a power of 2. #[inline(always)] -pub const fn align_up(value: usize, alignment: usize) -> usize { +pub const fn align_up_bits(value: u64, alignment_bits: u64) -> u64 { + let align_mask = (1 << alignment_bits) - 1; + if value & align_mask == 0 { + value // already aligned + } else { + (value | align_mask) + 1 + } +} + +/// Align address upwards. +/// +/// Returns the smallest x with alignment `align` so that x >= addr. +/// The alignment must be a power of 2. +#[inline(always)] +pub const fn align_up(value: u64, alignment: u64) -> u64 { assert!( alignment.is_power_of_two(), "`alignment` must be a power of two" @@ -38,11 +53,17 @@ pub const fn align_up(value: usize, alignment: usize) -> usize { } } +/// Check if a value is aligned to a given alignment. +#[inline(always)] +pub const fn is_aligned_bits(value: u64, alignment_bits: u64) -> bool { + (value & ((1 << alignment_bits) - 1)) == 0 +} + /// Check if a value is aligned to a given alignment. /// The alignment must be a power of 2. #[inline(always)] -pub const fn is_aligned(value: usize, alignment: usize) -> bool { - assert!( +pub const fn is_aligned(value: u64, alignment: u64) -> bool { + debug_assert!( alignment.is_power_of_two(), "`alignment` must be a power of two" ); @@ -50,26 +71,9 @@ pub const fn is_aligned(value: usize, alignment: usize) -> bool { (value & (alignment - 1)) == 0 } -/// Convert a size into human readable format. -pub const fn size_human_readable_ceil(size: usize) -> (usize, &'static str) { - const KIB: usize = 1024; - const MIB: usize = 1024 * 1024; - const GIB: usize = 1024 * 1024 * 1024; - - if (size / GIB) > 0 { - (size.div_ceil(GIB), "GiB") - } else if (size / MIB) > 0 { - (size.div_ceil(MIB), "MiB") - } else if (size / KIB) > 0 { - (size.div_ceil(KIB), "KiB") - } else { - (size, "Byte") - } -} - /// Calculate the next possible aligned address without sanity checking the /// input parameters. #[inline] -fn aligned_addr_unchecked(addr: usize, alignment: usize) -> usize { +pub fn aligned_addr_unchecked(addr: u64, alignment: u64) -> u64 { (addr + (alignment - 1)) & !(alignment - 1) } diff --git a/libs/address/src/lib.rs b/libs/address/src/lib.rs new file mode 100644 index 000000000..86c689df4 --- /dev/null +++ b/libs/address/src/lib.rs @@ -0,0 +1,639 @@ +// +// SPDX-License-Identifier: BlueOak-1.0.0 +// Copyright (c) Berkus Decker +// + +#![no_std] +#![no_main] +#![feature(const_trait_impl)] +#![feature(const_ops)] +#![feature(const_convert)] +#![feature(custom_test_frameworks)] +#![test_runner(libtest::test_runner)] +#![reexport_test_harness_main = "test_main"] + +use { + arbitrary_int::{i47, i52, u9, u12, u17}, + bitbybit::bitfield, + core::{ + convert::{From, TryInto}, + fmt, + marker::PhantomData, + ops::{Add, AddAssign, Rem, RemAssign, Shl, Shr, Sub, SubAssign}, + }, + usize_conversions::FromUsize, +}; + +//-------------------------------------------------------------------------------------------------- +// Public Definitions +//-------------------------------------------------------------------------------------------------- + +pub mod align; + +/// A 64-bit physical memory address. +/// +/// This is a wrapper type around an `u64`, so it is always 8 bytes, even when compiled +/// on non 64-bit systems. The `UsizeConversions` trait can be used for performing conversions +/// between `u64` and `usize`. +/// +/// On `aarch64`, only the 52 lower bits of a physical address can be used. The top 12 bits need +/// to be zero. This type guarantees that it always represents a valid physical address. +pub type PhysAddr = Address; + +pub type PhysAddrNotValid = AddressNotValid; + +/// A canonical 64-bit virtual memory address. +/// +/// This is a wrapper type around an `u64`, so it is always 8 bytes, even when compiled +/// on non 64-bit systems. The `UsizeConversions` trait can be used for performing conversions +/// between `u64` and `usize`. +/// +/// On `x86_64`, only the 48 lower bits of a virtual address can be used. The top 16 bits need +/// to be copies of bit 47, i.e. the most significant bit. Addresses that fulfil this criterium +/// are called “canonical”. This type guarantees that it always represents a canonical address. +pub type VirtAddr = Address; + +pub type VirtAddrNotValid = AddressNotValid; + +/// Address of all physical memory mapping, so that kernel can operate anywhere. +pub const PHYSICAL_KERNEL_WINDOW: u64 = 0xffff_f000_0000_0000; + +/// Metadata trait for marking the type of an address. +pub const trait AddressType: Copy + Clone + PartialOrd + PartialEq + Ord + Eq { + const NAME: &'static str; + fn validate(addr: u64) -> Result; // Ok(canonical_addr) or Err(raw, reason) +} + +pub const trait PageSize { + fn alignment(&self) -> u64; + fn mask(&self) -> u64; +} + +impl const PageSize for u64 { + fn alignment(&self) -> u64 { + *self + } + fn mask(&self) -> u64 { + assert!( + self.is_power_of_two(), + "PageSize must be a power of two to use as mask" + ); + self - 1 + } +} + +impl const PageSize for usize { + fn alignment(&self) -> u64 { + *self as u64 + } + fn mask(&self) -> u64 { + assert!( + self.is_power_of_two(), + "PageSize must be a power of two to use as mask" + ); + (self - 1) as u64 + } +} + +/// Zero-sized type to mark a physical address. +#[derive(Copy, Clone, Debug, PartialOrd, PartialEq, Ord, Eq)] +pub enum Physical {} + +/// Zero-sized type to mark a virtual address. +#[derive(Copy, Clone, Debug, PartialOrd, PartialEq, Ord, Eq)] +pub enum Virtual {} + +/// Generic address type. +/// +/// This is a wrapper type around an `u64`, so it is always 8 bytes, even when compiled +/// on non 64-bit systems. The `UsizeConversions` trait can be used for performing conversions +/// between `u64` and `usize`. +#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord)] +#[repr(transparent)] +pub struct Address { + pub(crate) value: u64, + pub(crate) _address_type: PhantomData ATYPE>, +} + +const _: () = assert!(core::mem::size_of::>() == core::mem::size_of::()); +const _: () = assert!(core::mem::size_of::>() == core::mem::size_of::()); + +/// A passed `u64` was not a valid address. +/// +/// What this means exactly depends on architecture assumptions. +/// Arch crate is expected to implement this trait and provide +/// additional information about why this address is invalid. +#[derive(Default, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Debug)] +pub struct AddressNotValid { + pub(crate) value: u64, + pub(crate) _address_type: PhantomData ATYPE>, +} + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- + +//================================================================================================= +// AddressNotValid +//================================================================================================= + +impl AddressNotValid { + pub const fn new(value: u64) -> Self { + Self { + value, + _address_type: PhantomData, + } + } +} + +//================================================================================================= +// Physical/Virtual +//================================================================================================= + +#[bitfield(u64)] +struct PhysTopBits { + #[bits(0..=51, r)] + address: i52, + #[bits(52..=63, rw)] + top_bits: u12, +} + +impl const AddressType for Physical { + const NAME: &'static str = "PhysAddr"; + + /// Panics if any bits in the bit position 52 to 64 is set. + /// TODO: this is arch-dependent!. + fn validate(addr: u64) -> Result { + match PhysTopBits::new_with_raw_value(addr).top_bits().value() { + 0 => Ok(addr), + _ => Err(( + addr, + "physical addresses must not have any set bits in positions 52 to 64", + )), + } + } +} + +#[bitfield(u64)] +struct VirtTopBits { + #[bits(0..=46, r)] + address: i47, + #[bits(47..=63, rw)] + top_bits: u17, +} + +impl const AddressType for Virtual { + const NAME: &'static str = "VirtAddr"; + + /// This function tries to performs sign extension of bit 47 to make the address canonical. + /// It succeeds if bits 47 to 64 are a correct sign extension or all null (i.e. copies of bit 47). + /// + /// An error means that bits 48 to 64 are not a valid sign extension and are not null either. + /// So automatic sign extension would have overwritten possibly meaningful bits. + /// This likely indicates a bug, for example an invalid address calculation. + /// TODO: Support ASID byte in top bits of the address. + fn validate(addr: u64) -> Result { + match VirtTopBits::new_with_raw_value(addr).top_bits().value() { + 0 | 0x1ffff => Ok(addr), // address is canonical + 1 => { + // address needs sign extension + let addr = VirtTopBits::new_with_raw_value(addr); + let addr = addr.with_top_bits(u17::from_u32(0x1ffff)); + Ok(addr.raw_value()) + } + _ => Err(( + addr, + "virtual address must not contain any data in bits 48 to 64", + )), + } + } +} + +//================================================================================================= +// Address +//================================================================================================= + +impl Default for Address { + fn default() -> Self { + Self::zero() + } +} + +impl Address { + /// Create an address without checking. + /// # Safety + pub const unsafe fn new_unchecked(value: u64) -> Self { + Self { + value, + _address_type: PhantomData, + } + } + + /// Creates a new address. + /// + /// Panics if address is not representable. + pub const fn new(addr: u64) -> Self { + match ATYPE::validate(addr) { + Ok(addr) => unsafe { + // SAFETY: We've checked address for validity above. + Address::::new_unchecked(addr) + }, + Err((_addr, message)) => panic!("{}", message), + } + } + + /// Tries to create a new address. + pub const fn try_new(addr: u64) -> Result> { + match ATYPE::validate(addr) { + Ok(addr) => Ok( + // SAFETY: We've checked address for validity above. + unsafe { Address::::new_unchecked(addr) }, + ), + Err((addr, _message)) => Err(AddressNotValid::::new(addr)), + } + } + + /// Converts the address to an `u64`. + pub const fn as_u64(&self) -> u64 { + self.value + } + + pub fn as_usize(&self) -> usize { + self.value.try_into().unwrap() + } + + /// Creates an address that points to `0`. + pub const fn zero() -> Address { + Self { + value: 0, + _address_type: PhantomData, + } + } + + /// Converts the address to a raw pointer. + pub const fn into_ptr(self) -> *const T { + self.value as *const T + } + + /// Converts the address to a raw pointer. + pub const fn as_ptr(&self) -> *const T { + self.value as *const T + } + + /// Converts the address to a mutable raw pointer. + pub const fn into_mut_ptr(self) -> *mut T { + self.value as *mut T + } + + /// Converts the address to a mutable raw pointer. + pub const fn as_mut_ptr(&self) -> *mut T { + self.value as *mut T + } + + /// Convenience method for checking if a physical address is null. + pub fn is_null(&self) -> bool { + self.value == 0 + } + + /// Creates an address from the given pointer + pub fn from_ptr(ptr: *const T) -> Self { + Self::new(u64::from_usize(ptr as usize)) + } + + /// Creates a virtual address from the given pointer + pub fn from_mut_ptr(ptr: *mut T) -> Self { + Self::new(u64::from_usize(ptr as usize)) + } + + // TODO: With pageSize parameterized we can move it out to platform-independent code. + + /// Align down to page size. + #[must_use] + pub const fn align_down_page(&self, page_size: &impl const PageSize) -> Self { + let aligned = align::align_down(self.value, page_size.alignment()); + Self::new(aligned) + } + + /// Align up to page size. + #[must_use] + pub const fn align_up_page(&self, page_size: &impl const PageSize) -> Self { + let aligned = align::align_up(self.value, page_size.alignment()); + Self::new(aligned) + } + + /// Checks if the address is page aligned. + pub const fn is_page_aligned(&self, page_size: &impl const PageSize) -> bool { + align::is_aligned(self.value, page_size.alignment()) + } + + /// Return the address' offset into the corresponding page. + pub const fn offset_into_page(&self, page_size: &impl const PageSize) -> u64 { + self.value & page_size.mask() + } + + /// Aligns the address upwards to the given alignment. + /// + /// See the `align_up` function for more information. + #[must_use] + pub fn aligned_up(self, align: U) -> Self + where + U: Into, + { + Self { + value: align::align_up(self.value, align.into()), + _address_type: PhantomData, + } + } + + /// Aligns the address downwards to the given alignment. + /// + /// See the `align_down` function for more information. + #[must_use] + pub fn aligned_down(self, align: U) -> Self + where + U: Into, + { + Self { + value: align::align_down(self.value, align.into()), + _address_type: PhantomData, + } + } + + /// Checks whether the address has the required alignment. + pub fn is_aligned>(self, align: U) -> bool { + align::is_aligned(self.value, align.into()) + } +} + +//================================================================================================= +// Address specifics +//================================================================================================= + +impl Address { + /// Convert physical memory address into a kernel-view virtual address for physical memory. + pub fn user_to_kernel(&self) -> Address { + assert!(self.value < !PHYSICAL_KERNEL_WINDOW); + Address::::new(self.value + PHYSICAL_KERNEL_WINDOW) + } +} + +//================================================================================================= +// Address specifics +//================================================================================================= + +impl Address { + /// Creates a new canonical virtual address without checks (overwriting top bits). + /// + /// This function performs sign extension of bit 47 to make the address canonical, so + /// bits 48 to 64 are overwritten. If you want to check that these bits contain no data, + /// use `new` or `try_new`. + pub const fn new_canonical(addr: u64) -> Self { + let v = VirtTopBits::new_with_raw_value(addr); + let v = if v.top_bits().value() & 1 != 0 { + v.with_top_bits(u17::from_u32(0x1ffff)) + } else { + v.with_top_bits(u17::from_u32(0)) + }; + // SAFETY: We've checked address for validity above. + unsafe { Self::new_unchecked(v.raw_value()) } + } + + // @todo Support ASID byte in top bits of the address. + // pub fn with_asid(addr: u64, asid: ASID) -> Address {} + + // TODO: The following index and page fns should be accessible through a PageSize trait or something? + + /// Returns the 12-bit page offset of this virtual address. + pub fn page_offset(&self) -> u12 { + u12::new((self.value & 0xfff).try_into().unwrap()) + } + // ^ @todo this only works for 4KiB pages + + /// Returns the 9-bit level 3 page table index. + pub fn l3_index(&self) -> u9 { + u9::new(((self.value >> 12) & 0o777).try_into().unwrap()) + } + + /// Returns the 9-bit level 2 page table index. + pub fn l2_index(&self) -> u9 { + u9::new(((self.value >> 12 >> 9) & 0o777).try_into().unwrap()) + } + + /// Returns the 9-bit level 1 page table index. + pub fn l1_index(&self) -> u9 { + u9::new(((self.value >> 12 >> 9 >> 9) & 0o777).try_into().unwrap()) + } + + /// Returns the 9-bit level 0 page table index. + pub fn l0_index(&self) -> u9 { + u9::new( + ((self.value >> 12 >> 9 >> 9 >> 9) & 0o777) + .try_into() + .unwrap(), + ) + } + + pub const fn is_higher_half(self) -> bool { + self.value >= 0xFFFF_0000_0000_0000 + } + + /// Convert kernel-view virtual address of physical memory into a physical memory address. + pub fn kernel_to_user(&self) -> Address { + assert!(self.value >= PHYSICAL_KERNEL_WINDOW); + Address::::new(self.value - PHYSICAL_KERNEL_WINDOW) + } +} + +//================================================================================================= +// From +//================================================================================================= + +impl From for Address { + fn from(value: u64) -> Self { + Self::new(value) + } +} + +impl From for Address { + fn from(value: usize) -> Self { + Self::new(value as u64) + } +} + +impl From> for u64 { + fn from(value: Address) -> Self { + value.as_u64() + } +} + +impl From> for u128 { + fn from(value: Address) -> Self { + u128::from(value.as_u64()) + } +} + +//================================================================================================= +// Add/AddAssign +//================================================================================================= + +impl Add for Address { + type Output = Self; + + /// Add a given offset to the current virtual address. Never wraps. + #[inline(always)] + fn add(self, rhs: T) -> Self::Output { + // @todo runtime cost of unwrap() here + // VirtAddr::new(self.value.saturating_add(num::cast(rhs).unwrap())) + match self.value.checked_add(num::cast(rhs).unwrap()) { + None => panic!("Overflow on Address::add"), + Some(x) => Self::new(x), + } + } +} + +// FIXME: this is already a default impl? +impl AddAssign for Address { + fn add_assign(&mut self, rhs: T) { + *self = *self + rhs; + } +} + +//================================================================================================= +// Sub/SubAssign +//================================================================================================= + +// Difference of two addresses is a size. +impl Sub> for Address { + type Output = usize; + + fn sub(self, rhs: Address) -> Self::Output { + match self.value.checked_sub(rhs.value) { + None => panic!("Overflow on Address::sub"), + Some(x) => usize::try_from(x).unwrap(), + } + } +} + +impl Sub for Address { + type Output = Self; + + fn sub(self, rhs: T) -> Self::Output { + Address::::new(self.value.checked_sub(num::cast(rhs).unwrap()).unwrap()) + } +} + +impl SubAssign for Address { + fn sub_assign(&mut self, rhs: T) { + *self = *self - rhs; + } +} + +//================================================================================================= +// Shr/Shl +//================================================================================================= + +impl Shr for Address { + type Output = Self; + + fn shr(self, shift: usize) -> Self::Output { + Address::::new(self.value >> shift) + } +} + +impl Shl for Address { + type Output = Self; + + fn shl(self, shift: usize) -> Self::Output { + Address::::new(self.value << shift) + } +} + +//================================================================================================= +// Rem/RemAssign +//================================================================================================= + +impl Rem for Address { + type Output = u64; + + fn rem(self, rhs: T) -> Self::Output { + num::traits::CheckedRem::checked_rem(&self.value, &num::cast(rhs).unwrap()).unwrap() + } +} + +// @todo this is not very useful... +impl RemAssign for Address { + fn rem_assign(&mut self, rhs: T) { + *self = Address::::new( + num::traits::CheckedRem::checked_rem(&self.value, &num::cast(rhs).unwrap()).unwrap(), + ); + } +} + +//================================================================================================= +// Display/Debug/fmt +//================================================================================================= + +impl fmt::Debug for Address { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "{}({:#x})", ATYPE::NAME, self.value) + } +} + +impl fmt::Display for Address { + // Don't expect to see physical addresses greater than 40 bit. + #[allow(clippy::cast_possible_truncation)] + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + let q3: u8 = ((self.value >> 32) & 0xff) as u8; + let q2: u16 = ((self.value >> 16) & 0xffff) as u16; + let q1: u16 = (self.value & 0xffff) as u16; + + write!(f, "pa")?; + write!(f, "{q3:02x}_")?; + write!(f, "{q2:04x}_")?; + write!(f, "{q1:04x}") + } +} + +impl fmt::Display for Address { + #[allow(clippy::cast_possible_truncation)] + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + let q4: u16 = ((self.value >> 48) & 0xffff) as u16; + let q3: u16 = ((self.value >> 32) & 0xffff) as u16; + let q2: u16 = ((self.value >> 16) & 0xffff) as u16; + let q1: u16 = (self.value & 0xffff) as u16; + + write!(f, "va")?; + write!(f, "{q4:04x}_")?; + write!(f, "{q3:04x}_")?; + write!(f, "{q2:04x}_")?; + write!(f, "{q1:04x}") + } +} + +impl fmt::Binary for Address { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + self.value.fmt(f) + } +} + +impl fmt::LowerHex for Address { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + self.value.fmt(f) + } +} + +impl fmt::UpperHex for Address { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + self.value.fmt(f) + } +} + +impl fmt::Octal for Address { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + self.value.fmt(f) + } +} + +#[cfg(test)] +#[path = "../../../tests/common/mod.rs"] +mod common; diff --git a/libs/address/tests/address.rs b/libs/address/tests/address.rs new file mode 100644 index 000000000..c3980a01e --- /dev/null +++ b/libs/address/tests/address.rs @@ -0,0 +1,58 @@ +#![no_std] +#![no_main] +#![feature(custom_test_frameworks)] +#![test_runner(libtest::test_runner)] +#![reexport_test_harness_main = "test_main"] + +#[path = "../../../tests/common/mod.rs"] +mod common; + +#[test_case] +pub fn test_invalid_phys_addr() { + use vesper_address::{PhysAddr, PhysAddrNotValid}; + + let result = PhysAddr::try_new(0xfafa_0123_3210_3210); + if let Err(e) = result { + assert_eq!(e, PhysAddrNotValid::new(0xfafa_0123_3210_3210)); + } else { + assert!(false) + } +} + +/// Sanity of [Address] methods. +#[test_case] +fn address_type_method_sanity() { + use vesper_address::{Address, Virtual}; + + const SIZE: u64 = 0x1_0000; + + let addr = Address::::new(SIZE + 100); + + assert_eq!(addr.as_u64(), SIZE + 100); + + assert_eq!(addr.align_down_page(&SIZE), SIZE.into()); + + assert_eq!(addr.align_up_page(&SIZE), (SIZE * 2).into()); + + assert!(!addr.is_page_aligned(&SIZE)); + + assert_eq!(addr.offset_into_page(&SIZE), 100); +} +#[test_case] +pub fn test_align_up() { + use vesper_address::align::align_up; + + // align 1 + assert_eq!(align_up(0, 1), 0); + assert_eq!(align_up(1234, 1), 1234); + assert_eq!(align_up(0xffff_ffff_ffff_ffff, 1), 0xffff_ffff_ffff_ffff); + // align 2 + assert_eq!(align_up(0, 2), 0); + assert_eq!(align_up(1233, 2), 1234); + assert_eq!(align_up(0xffff_ffff_ffff_fffe, 2), 0xffff_ffff_ffff_fffe); + // address 0 + assert_eq!(align_up(0, 128), 0); + assert_eq!(align_up(0, 1), 0); + assert_eq!(align_up(0, 2), 0); + assert_eq!(align_up(0, 0x8000_0000_0000_0000), 0); +} diff --git a/libs/alloc/Cargo.toml b/libs/alloc/Cargo.toml new file mode 100644 index 000000000..e04538096 --- /dev/null +++ b/libs/alloc/Cargo.toml @@ -0,0 +1,31 @@ +[package] +name = "vesper-alloc" + +description = "Vesper nanokernel memory allocator types." + +authors = { workspace = true } +categories = { workspace = true } +documentation = { workspace = true } +edition = { workspace = true } +homepage = { workspace = true } +license = { workspace = true } +readme = { workspace = true } +repository = { workspace = true } +version = { workspace = true } + +publish = false + +[badges] +maintenance = { status = "experimental" } + +[dependencies] +libaddress = { workspace = true } +liblog = { workspace = true } + +# Tests are offloaded to kernel integration tests binary. +[lib] +test = false +harness = false + +[lints] +workspace = true diff --git a/libs/memory/src/mm/bump_allocator.rs b/libs/alloc/src/bump_allocator.rs similarity index 85% rename from libs/memory/src/mm/bump_allocator.rs rename to libs/alloc/src/bump_allocator.rs index d774cf33a..567659031 100644 --- a/libs/memory/src/mm/bump_allocator.rs +++ b/libs/alloc/src/bump_allocator.rs @@ -27,15 +27,18 @@ unsafe impl Allocator for BumpAllocator { fn allocate(&self, layout: Layout) -> Result, AllocError> { let name = self.name; let size = layout.size(); - let start = crate::mm::aligned_addr_unchecked(self.next.get(), layout.align()); - let end = start + layout.size(); + let start = libaddress::align::aligned_addr_unchecked( + self.next.get() as u64, + layout.align() as u64, + ); + let end = start + layout.size() as u64; println!("[i] {name}:\n Allocating Start {start:#010x} End {end:#010x}",); - if end > self.pool_end { + if end > self.pool_end as u64 { return Err(AllocError); } - self.next.set(end); + self.next.set(end.try_into().unwrap()); println!("[i] {name}:\n Allocated Addr {start:#010x} Size {size:#x}",); diff --git a/libs/alloc/src/lib.rs b/libs/alloc/src/lib.rs new file mode 100644 index 000000000..6fa91bd57 --- /dev/null +++ b/libs/alloc/src/lib.rs @@ -0,0 +1,11 @@ +/* + * SPDX-License-Identifier: BlueOak-1.0.0 + * Copyright (c) Berkus Decker + */ + +#![no_std] +#![feature(allocator_api)] +#![feature(format_args_nl)] + +mod bump_allocator; +pub use bump_allocator::BumpAllocator; diff --git a/libs/boot/Cargo.toml b/libs/boot/Cargo.toml index 9758b10d2..1a3709db4 100644 --- a/libs/boot/Cargo.toml +++ b/libs/boot/Cargo.toml @@ -1,7 +1,7 @@ [package] -name = "libboot" +name = "vesper-boot" -description = "Kernel boot code" +description = "Vesper nanokernel boot PIE section." authors = { workspace = true } categories = { workspace = true } @@ -24,5 +24,10 @@ libcpu = { workspace = true } libplatform = { workspace = true } tock-registers = { workspace = true } +# Tests are offloaded to kernel integration tests binary. +[lib] +test = false +harness = false + [lints] workspace = true diff --git a/libs/boot/src/arch/aarch64/boot.rs b/libs/boot/src/arch/aarch64/boot.rs index a6eea7fad..d8977b488 100644 --- a/libs/boot/src/arch/aarch64/boot.rs +++ b/libs/boot/src/arch/aarch64/boot.rs @@ -8,11 +8,14 @@ //! Low-level boot of the ARMv8-A processor. //! +//! Raspi kernel boot helper: +//! In particular, see `dtb_ptr32` + use { - aarch64_cpu::{asm, registers::*}, - core::{arch::global_asm, cell::UnsafeCell}, + aarch64_cpu::registers::*, + core::arch::global_asm, libcpu::endless_sleep, - libplatform::platform::cpu::BOOT_CORE_ID, + libplatform::cpu::BOOT_CORE_ID, tock_registers::interfaces::{Readable, Writeable}, }; @@ -24,11 +27,11 @@ macro_rules! entry { /// Only type-checks! #[unsafe(export_name = "main")] #[inline(always)] - pub unsafe fn __main() -> ! { + pub unsafe fn __main(dtb: u32) -> ! { // type check the given path - let f: unsafe fn() -> ! = $path; + let f: unsafe fn(u32) -> ! = $path; - unsafe { f() } + unsafe { f(dtb) } } }; } @@ -52,32 +55,19 @@ global_asm!( /// # Safety /// /// Totally unsafe! We're in the hardware land. -/// We assume that no statics are accessed before transition to main from `reset()` function. +/// We assume that no statics are accessed before transition to this fn. #[unsafe(no_mangle)] #[unsafe(link_section = ".text.boot")] -pub unsafe extern "C" fn _startup_in_rust() -> ! { - // Can't match values with dots in match, so use intermediate consts. +pub unsafe extern "C" fn _startup_in_rust(dtb: u32) -> ! { + // On entry, w0 should contain the dtb address. + // For non-primary cores it contains 0. + + // Can't match values with dots in names in match, so use intermediate consts. #[cfg(feature = "qemu")] const EL3: u64 = CurrentEL::EL::EL3.value; const EL2: u64 = CurrentEL::EL::EL2.value; - const EL1: u64 = CurrentEL::EL::EL1.value; - - shared_setup_and_enter_pre(); - - match CurrentEL.get() { - #[cfg(feature = "qemu")] - EL3 => setup_and_enter_el1_from_el3(), - EL2 => setup_and_enter_el1_from_el2(), - EL1 => reset(), - // if not core0 or not EL3/EL2/EL1, infinitely wait for events - _ => endless_sleep(), - } -} -#[unsafe(link_section = ".text.boot")] -#[inline(always)] -fn shared_setup_and_enter_pre() { - // Enable timer counter registers for EL1 + // Enable timer counter registers for EL1 - FIXME can be done later after reset CNTHCTL_EL2.write(CNTHCTL_EL2::EL1PCEN::SET + CNTHCTL_EL2::EL1PCTEN::SET); // No virtual offset for reading the counters @@ -96,58 +86,28 @@ fn shared_setup_and_enter_pre() { + SCTLR_EL1::SA0::Disable, ); - // enable_armv6_unaligned_access(); + // Same for SCTLR_EL2? + SCTLR_EL2.write( + SCTLR_EL2::I::NonCacheable + + SCTLR_EL2::C::NonCacheable + + SCTLR_EL2::M::Disable + + SCTLR_EL2::A::Disable + + SCTLR_EL2::SA::Disable, + ); // Set Hypervisor Configuration Register (EL2) // Set EL1 execution state to AArch64 - // @todo Explain the SWIO bit (SWIO hardwired on Pi3) + // @todo Explain the SWIO bit (SWIO is hardwired on RPi3) HCR_EL2.write(HCR_EL2::RW::EL1IsAarch64 + HCR_EL2::SWIO::SET); // @todo disable VM bit to prevent stage 2 MMU translations -} -#[unsafe(link_section = ".text.boot")] -#[inline] -fn shared_setup_and_enter_post() -> ! { - unsafe extern "Rust" { - // Stack top - static __STACK_TOP: UnsafeCell<()>; - } - // Set up SP_EL1 (stack pointer), which will be used by EL1 once - // we "return" to it. - // SAFETY: Pure asm. - unsafe { - SP_EL1.set(__STACK_TOP.get() as u64); + match CurrentEL.get() { + #[cfg(feature = "qemu")] + EL3 => setup_and_enter_el2_from_el3(dtb), + EL2 => reset(dtb), + // Cannot configure memory mappings in all other cases, fail instead! + _ => endless_sleep(), } - - // Use `eret` to "return" to EL1. This will result in execution of - // `reset()` in EL1. - asm::eret() -} - -/// Real hardware boot-up sequence. -/// -/// Prepare and execute transition from EL2 to EL1. -#[unsafe(link_section = ".text.boot")] -#[inline] -fn setup_and_enter_el1_from_el2() -> ! { - // Set Saved Program Status Register (EL2) - // Set up a simulated exception return. - // - // Fake a saved program status, where all interrupts were - // masked and SP_EL1 was used as a stack pointer. - SPSR_EL2.write( - SPSR_EL2::D::Masked - + SPSR_EL2::A::Masked - + SPSR_EL2::I::Masked - + SPSR_EL2::F::Masked - + SPSR_EL2::M::EL1h, // Use SP_EL1 - ); - - // Make the Exception Link Register (EL2) point to reset(). - #[allow(clippy::fn_to_numeric_cast_any)] - ELR_EL2.set(reset as *const () as u64); - - shared_setup_and_enter_post() } /// QEMU boot-up sequence. @@ -157,40 +117,69 @@ fn setup_and_enter_el1_from_el2() -> ! { /// section: 5.5.1 /// However, GPU init code must be switching it down to EL2. /// QEMU can't emulate Raspberry Pi properly (no VC boot code), so it starts in EL3. +/// FIXME: new qemu has seemingly fixed this, so probably not needed anymore, double check! /// -/// Prepare and execute transition from EL3 to EL1. +/// Prepare and execute transition from EL3 to EL2. /// (from https://github.com/s-matyukevich/raspberry-pi-os/blob/master/docs/lesson02/rpi-os.md) #[cfg(feature = "qemu")] #[unsafe(link_section = ".text.boot")] #[inline] -fn setup_and_enter_el1_from_el3() -> ! { - // Set Secure Configuration Register (EL3) +fn setup_and_enter_el2_from_el3(dtb: u32) -> ! { + // Set Secure Configuration Register in EL3. SCR_EL3.write(SCR_EL3::RW::NextELIsAarch64 + SCR_EL3::NS::NonSecure); - // Set Saved Program Status Register (EL3) + // Set Saved Program Status Register in EL3. // Set up a simulated exception return. // // Fake a saved program status, where all interrupts were - // masked and SP_EL1 was used as a stack pointer. + // masked and SP_EL2 was used as a stack pointer. SPSR_EL3.write( SPSR_EL3::D::Masked + SPSR_EL3::A::Masked + SPSR_EL3::I::Masked + SPSR_EL3::F::Masked - + SPSR_EL3::M::EL1h, // Use SP_EL1 + + SPSR_EL3::M::EL2h, // Use SP_EL2 ); - // Make the Exception Link Register (EL3) point to reset(). + // Make the Exception Link Register in EL3 point to reset(). ELR_EL3.set(reset as *const () as u64); - shared_setup_and_enter_post() + unsafe extern "Rust" { + // Stack top + static __STACK_TOP: UnsafeCell<()>; + } + // Set up SP_EL2 (stack pointer), which will be used by EL2 once + // we "return" to it. + // SAFETY: Pure asm. + unsafe { + SP_EL2.set(__STACK_TOP.get() as u64); + } + // Use `eret` to "return" to EL2. This will result in execution of + // `reset()` in EL2. + // Load DTB address into w0 prior to eret. + unsafe { + core::arch::asm!("eret", in("w0") dtb); + core::hint::unreachable_unchecked() + } } -fn reset() -> ! { +// Enter Rust code in EL2. +#[unsafe(link_section = ".text.boot")] +fn reset(dtb: u32) -> ! { unsafe extern "Rust" { - fn main() -> !; + fn main(dtb: u32) -> !; + } + + unsafe extern "Rust" { + // Stack top + static __STACK_TOP: core::cell::UnsafeCell<()>; + } + // Set up SP (stack pointer), which will be used by EL2 rust code. + // SAFETY: Pure asm. + unsafe { + SP.set(__STACK_TOP.get() as u64); } // SAFETY: We're getting to more safety right here! - unsafe { main() } + unsafe { main(dtb) } } diff --git a/libs/boot/src/arch/aarch64/boot.s b/libs/boot/src/arch/aarch64/boot.s index 3e65f874b..b6ea79404 100644 --- a/libs/boot/src/arch/aarch64/boot.s +++ b/libs/boot/src/arch/aarch64/boot.s @@ -1,14 +1,11 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 -// -// Copyright (c) 2021 Andre Richter -// Modifications -// Copyright (c) 2021- Berkus +// SPDX-License-Identifier: BlueOak-1.0.0 +// Copyright (c) Berkus Decker // // Pre-boot code. // Used only because Rust's abstract machine considers UB any access to statics // before statics have been initialized. This is exactly the case for the boot code. -// So we avoid referencing any statics in the Rust code, and delegate the +// So we avoid referencing any linker symbol statics from the Rust code, and delegate the // task to assembly piece instead. // @@ -48,7 +45,7 @@ /// Entrypoint of the processor. /// /// Parks all cores except core0 and checks if we started in EL2/EL3. If -/// so, proceeds with setting up EL1. +/// so, init BSS and enter Rust code. /// /// This is invoked from the linker script, does arch-specific init /// and passes control to the kernel main function in Rust. @@ -70,9 +67,9 @@ _boot_cores: // Initialize BSS - prepare to fearlessly call into Rust code. // Assumptions: BSS start is u128-aligned, BSS end is u128-aligned. - // __BSS_START and __BSS_END are defined in linker script - ADR_REL x1, __BSS_START - ADR_REL x2, __BSS_END + // __BSS_START and __BSS_END are defined in the linker script + ADR_REL x1, __BSS_START // must be physical address!!!1 + ADR_REL x2, __BSS_END // must be physical address!!!1 .L__bss_init_loop: cmp x1, x2 b.eq .L_setup_stack @@ -83,6 +80,7 @@ _boot_cores: ADR_ABS x1, __STACK_TOP mov sp, x1 + // On entry, x0 contains DTB address bl _startup_in_rust .L_parking_loop: diff --git a/libs/console/Cargo.toml b/libs/console/Cargo.toml index ae6393239..c3359591d 100644 --- a/libs/console/Cargo.toml +++ b/libs/console/Cargo.toml @@ -1,7 +1,7 @@ [package] -name = "libconsole" +name = "vesper-console" -description = "Console functions." +description = "Vesper nanokernel console functions." authors = { workspace = true } categories = { workspace = true } @@ -23,14 +23,19 @@ default = [] qemu = [] [dependencies] -aarch64-cpu = { workspace = true } +# aarch64-cpu = { workspace = true } liblocking = { workspace = true } liblog = { workspace = true } +libprint = { workspace = true } libqemu = { workspace = true } -libtime = { workspace = true } +# libtime = { workspace = true } +# libtest = { workspace = true } + +# Tests are offloaded to kernel integration tests binary. [lib] test = false +harness = false [lints] workspace = true diff --git a/libs/console/src/lib.rs b/libs/console/src/lib.rs index cbf4c67dd..d559a20b6 100644 --- a/libs/console/src/lib.rs +++ b/libs/console/src/lib.rs @@ -5,7 +5,6 @@ #![reexport_test_harness_main = "test_main"] pub mod console; -pub mod write_to; pub trait SerialOps { /// Read one byte from serial without translation. @@ -33,11 +32,9 @@ impl liblog::Log for ConsoleLogger { fn log(&self, record: &Record) { #[cfg(any(test, feature = "qemu"))] { - use crate::write_to; - let mut buf = [0_u8; 4096]; // Increase this buffer size to allow dumping larger panic texts. libqemu::semihosting::sys_write0_call( - write_to::c_show(&mut buf, *record.args()).unwrap(), + libprint::format_cstr(&mut buf, *record.args()).unwrap(), ); } diff --git a/libs/cpu/Cargo.toml b/libs/cpu/Cargo.toml index e68714a9f..5326aa7c3 100644 --- a/libs/cpu/Cargo.toml +++ b/libs/cpu/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "libcpu" +name = "vesper-cpu" description = "Vesper nanokernel CPU support." @@ -24,17 +24,11 @@ maintenance = { status = "experimental" } [dependencies] aarch64-cpu = { workspace = true } -bit_field = { workspace = true } -bitflags = { workspace = true } -cfg-if = { workspace = true } -once_cell = { workspace = true } -snafu = { workspace = true } -tock-registers = { workspace = true } -usize_conversions = { workspace = true } -ux = { workspace = true } +# Tests are offloaded to kernel integration tests binary. [lib] test = false +harness = false # For proper testing in libcpu, we build it as a test_runner binary! diff --git a/libs/driver/Cargo.toml b/libs/driver/Cargo.toml index 0436ebab9..19e3fd8b4 100644 --- a/libs/driver/Cargo.toml +++ b/libs/driver/Cargo.toml @@ -1,7 +1,7 @@ [package] -name = "libdriver" +name = "vesper-drivers" -description = "Interfaces for implementing drivers" +description = "Vesper nanokernel interfaces for implementing device drivers." authors = { workspace = true } categories = { workspace = true } @@ -26,8 +26,12 @@ maintenance = { status = "experimental" } liblocking = { workspace = true } liblog = { workspace = true } +# libtest = { workspace = true } + +# Tests are offloaded to kernel integration tests binary. [lib] test = false +harness = false [lints] workspace = true diff --git a/libs/exception/Cargo.toml b/libs/exception/Cargo.toml index 9751f430a..a4b170fbb 100644 --- a/libs/exception/Cargo.toml +++ b/libs/exception/Cargo.toml @@ -1,7 +1,7 @@ [package] -name = "libexception" +name = "vesper-exceptions" -description = "Exceptions handling" +description = "Vesper nanokernel CPU exceptions handling." authors = { workspace = true } categories = { workspace = true } @@ -26,11 +26,13 @@ test_build = [] [dependencies] aarch64-cpu = { workspace = true } liblocal-irq = { workspace = true } -liblocking = { workspace = true } liblog = { workspace = true } -libprimitives = { workspace = true } -snafu = { workspace = true } tock-registers = { workspace = true } +# Tests are offloaded to kernel integration tests binary. +[lib] +test = false +harness = false + [lints] workspace = true diff --git a/libs/exception/src/arch/aarch64/exception/traps.rs b/libs/exception/src/arch/aarch64/debug.rs similarity index 97% rename from libs/exception/src/arch/aarch64/exception/traps.rs rename to libs/exception/src/arch/aarch64/debug.rs index 159113b44..949c0f7e2 100644 --- a/libs/exception/src/arch/aarch64/exception/traps.rs +++ b/libs/exception/src/arch/aarch64/debug.rs @@ -7,7 +7,7 @@ use { tock_registers::{LocalRegisterCopy, interfaces::Readable, register_bitfields}, }; -fn cause_to_string(cause: u64) -> &'static str { +pub fn cause_to_string(cause: u64) -> &'static str { if cause == ESR_EL1::EC::DataAbortCurrentEL.read(ESR_EL1::EC) { "Data Alignment Check" } else { @@ -115,9 +115,9 @@ register_bitfields! { ] } -type IssForDataAbort = LocalRegisterCopy; +pub type IssForDataAbort = LocalRegisterCopy; -fn iss_dfsc_to_string(iss: IssForDataAbort) -> &'static str { +pub fn iss_dfsc_to_string(iss: IssForDataAbort) -> &'static str { match iss.read_as_enum(ISS_DA::DFSC) { Some(ISS_DA::DFSC::Value::AddressSizeTL0) => { "Address size fault, level 0 of translation or translation table base register" @@ -183,7 +183,8 @@ fn iss_dfsc_to_string(iss: IssForDataAbort) -> &'static str { /// Not for production use! /// @todo merge this with default exception handler a bit? /// @returns true if we can skip the instruction and return, false if we need to continue handling the exception. -pub fn synchronous_common(e: &mut super::ExceptionContext) -> bool { +// #[cfg(any(test, feature = "test_build"))] +pub fn exception_dump(e: &mut super::ExceptionContext) -> bool { println!(" ESR_EL1: {:#010x} (syndrome)", ESR_EL1.get()); let cause = ESR_EL1.read(ESR_EL1::EC); println!( diff --git a/libs/exception/src/arch/aarch64/esr_el1.rs b/libs/exception/src/arch/aarch64/esr_el1.rs new file mode 100644 index 000000000..ef5683545 --- /dev/null +++ b/libs/exception/src/arch/aarch64/esr_el1.rs @@ -0,0 +1,41 @@ +use {aarch64_cpu::registers::ESR_EL1, core::fmt, tock_registers::LocalRegisterCopy}; + +// Exception syndrome register. +#[repr(transparent)] +pub struct EsrEL1(pub LocalRegisterCopy); + +impl EsrEL1 { + #[inline(always)] + pub fn exception_class(&self) -> Option { + self.0.read_as_enum(ESR_EL1::EC) + } + + #[cfg(any(test, feature = "test_build"))] + #[inline(always)] + #[allow(dead_code)] + pub fn iss(&self) -> u64 { + self.0.read(ESR_EL1::ISS) + } +} + +/// Human readable `ESR_EL1`. +#[rustfmt::skip] +impl fmt::Display for EsrEL1 { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + // Raw print of whole register. + writeln!(f, "ESR_EL1: {:#010x}", self.0.get())?; + + // Raw print of exception class. + write!(f, " Exception Class (EC) : {:#x}", self.0.read(ESR_EL1::EC))?; + + // Exception class. + let ec_translation = match self.exception_class() { + Some(ESR_EL1::EC::Value::DataAbortCurrentEL) => "Data Abort, current EL", + _ => "N/A", + }; + writeln!(f, " - {ec_translation}")?; + + // Raw print of instruction specific syndrome. + write!(f, " Instr Specific Syndrome (ISS): {:#x}", self.0.read(ESR_EL1::ISS)) + } +} diff --git a/libs/exception/src/arch/aarch64/exception/mod.rs b/libs/exception/src/arch/aarch64/exception/mod.rs deleted file mode 100644 index 2b1b17b87..000000000 --- a/libs/exception/src/arch/aarch64/exception/mod.rs +++ /dev/null @@ -1,414 +0,0 @@ -/* - * SPDX-License-Identifier: BlueOak-1.0.0 - * Copyright (c) Berkus Decker - */ - -//! Interrupt handling -//! -//! The base address is given by `VBAR_ELn` and each entry has a defined offset from this -//! base address. Each table has 16 entries, with each entry being 128 bytes (32 instructions) -//! in size. The table effectively consists of 4 sets of 4 entries. -//! -//! Minimal implementation to help catch MMU traps. -//! Reads `ESR_ELx` to understand why trap was taken. -//! -//! `VBAR_EL1`, `VBAR_EL2`, `VBAR_EL3` -//! -//! `CurrentEL` with `SP0`: +0x0 -//! -//! * Synchronous -//! * IRQ/vIRQ -//! * FIQ -//! * SError/vSError -//! -//! `CurrentEL` with `SPx`: +0x200 -//! -//! * Synchronous -//! * IRQ/vIRQ -//! * FIQ -//! * SError/vSError -//! -//! Lower EL using `AArch64`: +0x400 -//! -//! * Synchronous -//! * IRQ/vIRQ -//! * FIQ -//! * SError/vSError -//! -//! Lower EL using `AArch32`: +0x600 -//! -//! * Synchronous -//! * IRQ/vIRQ -//! * FIQ -//! * SError/vSError -//! -//! When the processor takes an exception to `AArch64` execution state, -//! all of the PSTATE interrupt masks is set automatically. This means -//! that further exceptions are disabled. If software is to support -//! nested exceptions, for example, to allow a higher priority interrupt -//! to interrupt the handling of a lower priority source, then software needs -//! to explicitly re-enable interrupts - -use { - crate::exception::PrivilegeLevel, - aarch64_cpu::{asm::barrier, registers::*}, - core::{ - cell::UnsafeCell, - fmt::{self}, - }, - liblog::info, - snafu::Snafu, - tock_registers::{ - LocalRegisterCopy, - interfaces::{Readable, Writeable}, - }, -}; - -mod traps; - -core::arch::global_asm!(include_str!("vectors.S")); - -//-------------------------------------------------------------------------------------------------- -// Private Definitions -//-------------------------------------------------------------------------------------------------- - -/// The exception context as it is stored on the stack on exception entry. -/// Keep in sync with exception setup code in vectors.S! -#[repr(C)] -struct ExceptionContext { - /// General Purpose Registers, x0-x29 - gpr: [u64; 30], - - /// The link register, aka x30. - lr: u64, - - /// Saved program status. - spsr_el1: SpsrEL1, - - /// Exception link register. The program counter at the time the exception happened. - elr_el1: u64, -} - -//-------------------------------------------------------------------------------------------------- -// Private Code -//-------------------------------------------------------------------------------------------------- - -/// The default exception, invoked for every exception type unless the handler -/// is overridden. -/// Prints verbose information about the exception and then panics. -/// -/// Default pointer is configured in the linker script. -#[unsafe(no_mangle)] -extern "C" fn default_exception_handler(exc: &ExceptionContext) { - panic!( - "Unexpected CPU Exception!\n\n\ - {}", - exc - ); -} - -//------------------------------------------------------------------------------ -// Current, EL0 -//------------------------------------------------------------------------------ - -#[unsafe(no_mangle)] -extern "C" fn current_el0_synchronous(_e: &mut ExceptionContext) { - panic!("Should not be here. Use of SP_EL0 in EL1 is not supported.") -} - -#[unsafe(no_mangle)] -extern "C" fn current_el0_irq(_e: &mut ExceptionContext) { - panic!("Should not be here. Use of SP_EL0 in EL1 is not supported.") -} - -#[unsafe(no_mangle)] -extern "C" fn current_el0_serror(_e: &mut ExceptionContext) { - panic!("Should not be here. Use of SP_EL0 in EL1 is not supported.") -} - -//------------------------------------------------------------------------------ -// Current, ELx -//------------------------------------------------------------------------------ - -#[unsafe(no_mangle)] -extern "C" fn current_elx_synchronous(e: &mut ExceptionContext) { - #[cfg(any(test, feature = "test_build"))] - { - const TEST_SVC_ID: u64 = 0x1337; - - let esr_el1 = EsrEL1(LocalRegisterCopy::new(ESR_EL1.get())); - - if let Some(ESR_EL1::EC::Value::SVC64) = esr_el1.exception_class() - && esr_el1.iss() == TEST_SVC_ID - { - liblog::println!("Serving syscall {TEST_SVC_ID}"); - return; - } - } - - if traps::synchronous_common(e) { - return; - } - - default_exception_handler(e); -} - -#[unsafe(no_mangle)] -extern "C" fn current_elx_irq(_e: &mut ExceptionContext) { - // -- @todo - // let token = unsafe { &exception::asynchronous::IRQContext::new() }; - // exception::asynchronous::irq_manager().handle_pending_irqs(token); -} - -#[unsafe(no_mangle)] -extern "C" fn current_elx_serror(e: &mut ExceptionContext) { - default_exception_handler(e); -} - -//------------------------------------------------------------------------------ -// Lower, AArch64 -//------------------------------------------------------------------------------ - -#[unsafe(no_mangle)] -extern "C" fn lower_aarch64_synchronous(e: &mut ExceptionContext) { - default_exception_handler(e); -} - -#[unsafe(no_mangle)] -extern "C" fn lower_aarch64_irq(e: &mut ExceptionContext) { - default_exception_handler(e); -} - -#[unsafe(no_mangle)] -extern "C" fn lower_aarch64_serror(e: &mut ExceptionContext) { - default_exception_handler(e); -} - -//------------------------------------------------------------------------------ -// Lower, AArch32 -//------------------------------------------------------------------------------ - -#[unsafe(no_mangle)] -extern "C" fn lower_aarch32_synchronous(e: &mut ExceptionContext) { - default_exception_handler(e); -} - -#[unsafe(no_mangle)] -extern "C" fn lower_aarch32_irq(e: &mut ExceptionContext) { - default_exception_handler(e); -} - -#[unsafe(no_mangle)] -extern "C" fn lower_aarch32_serror(e: &mut ExceptionContext) { - default_exception_handler(e); -} - -//------------------------------------------------------------------------------ -// Misc -// Wrappers for memory copies of registers. -//------------------------------------------------------------------------------ - -struct SpsrEL1(pub LocalRegisterCopy); -// Exception syndrome register. -struct EsrEL1(pub LocalRegisterCopy); - -/// Human readable `SPSR_EL1`. -#[rustfmt::skip] -impl fmt::Display for SpsrEL1 { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - // Raw value. - writeln!(f, "SPSR_EL1: {:#010x}", self.0.get())?; - - let to_flag_str = |x| -> _ { - if x { "Set" } else { "Not set" } - }; - - writeln!(f, " Flags:")?; - writeln!(f, " Negative (N): {}", to_flag_str(self.0.is_set(SPSR_EL1::N)))?; - writeln!(f, " Zero (Z): {}", to_flag_str(self.0.is_set(SPSR_EL1::Z)))?; - writeln!(f, " Carry (C): {}", to_flag_str(self.0.is_set(SPSR_EL1::C)))?; - writeln!(f, " Overflow (V): {}", to_flag_str(self.0.is_set(SPSR_EL1::V)))?; - - let to_mask_str = |x| -> _ { - if x { "Masked" } else { "Unmasked" } - }; - - writeln!(f, " Exception handling state:")?; - writeln!(f, " Debug (D): {}", to_mask_str(self.0.is_set(SPSR_EL1::D)))?; - writeln!(f, " SError (A): {}", to_mask_str(self.0.is_set(SPSR_EL1::A)))?; - writeln!(f, " IRQ (I): {}", to_mask_str(self.0.is_set(SPSR_EL1::I)))?; - writeln!(f, " FIQ (F): {}", to_mask_str(self.0.is_set(SPSR_EL1::F)))?; - - write!(f, " Illegal Execution State (IL): {}", - to_flag_str(self.0.is_set(SPSR_EL1::IL)) - ) - } -} - -impl EsrEL1 { - #[inline(always)] - fn exception_class(&self) -> Option { - self.0.read_as_enum(ESR_EL1::EC) - } - - #[cfg(any(test, feature = "test_build"))] - #[inline(always)] - fn iss(&self) -> u64 { - self.0.read(ESR_EL1::ISS) - } -} - -/// Human readable `ESR_EL1`. -#[rustfmt::skip] -impl fmt::Display for EsrEL1 { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - // Raw print of whole register. - writeln!(f, "ESR_EL1: {:#010x}", self.0.get())?; - - // Raw print of exception class. - write!(f, " Exception Class (EC) : {:#x}", self.0.read(ESR_EL1::EC))?; - - // Exception class. - let ec_translation = match self.exception_class() { - Some(ESR_EL1::EC::Value::DataAbortCurrentEL) => "Data Abort, current EL", - _ => "N/A", - }; - writeln!(f, " - {ec_translation}")?; - - // Raw print of instruction specific syndrome. - write!(f, " Instr Specific Syndrome (ISS): {:#x}", self.0.read(ESR_EL1::ISS)) - } -} - -//------------------------------------------------------------------------------ -// Misc -// Exception context. -//------------------------------------------------------------------------------ - -impl ExceptionContext { - // #[inline(always)] - // fn exception_class(&self) -> Option { - // self.esr_el1.exception_class() - // } - - #[inline(always)] - fn fault_address_valid() -> bool { - use ESR_EL1::EC::Value::{ - DataAbortCurrentEL, DataAbortLowerEL, InstrAbortCurrentEL, InstrAbortLowerEL, - PCAlignmentFault, WatchpointCurrentEL, WatchpointLowerEL, - }; - - let esr_el1 = EsrEL1(LocalRegisterCopy::new(ESR_EL1.get())); - - match esr_el1.exception_class() { - None => false, - Some(ec) => matches!( - ec, - InstrAbortLowerEL - | InstrAbortCurrentEL - | PCAlignmentFault - | DataAbortLowerEL - | DataAbortCurrentEL - | WatchpointLowerEL - | WatchpointCurrentEL - ), - } - } - - pub fn write_gprs(&self, f: &mut fmt::Formatter) -> fmt::Result { - writeln!(f, "General purpose registers:")?; - - let alternating = |x| -> _ { if x % 2 == 0 { " " } else { "\n" } }; - - // Print two registers per line. - for (i, reg) in self.gpr.iter().enumerate() { - write!(f, " x{: <2}: {: >#018x}{}", i, reg, alternating(i))?; - } - Ok(()) - } -} - -/// Human readable print of the exception context. -impl fmt::Display for ExceptionContext { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - // writeln!(f, "{}", self.esr_el1)?; - - if Self::fault_address_valid() { - writeln!( - f, - "FAR_EL1: {:#018x}", - usize::try_from(FAR_EL1.get()).unwrap_or(0) - )?; - } - - writeln!(f, "{}", self.spsr_el1)?; - writeln!(f, "ELR_EL1: {:#018x} (return to)", self.elr_el1)?; - writeln!(f)?; - self.write_gprs(f)?; - write!(f, " lr : {:#018x}", self.lr) - } -} - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- - -/// The processor's current privilege level. -pub fn current_privilege_level() -> (PrivilegeLevel, &'static str) { - let el = CurrentEL.read_as_enum(CurrentEL::EL); - match el { - Some(CurrentEL::EL::Value::EL3) => (PrivilegeLevel::Unknown, "EL3"), - Some(CurrentEL::EL::Value::EL2) => (PrivilegeLevel::Hypervisor, "EL2"), - Some(CurrentEL::EL::Value::EL1) => (PrivilegeLevel::Kernel, "EL1"), - Some(CurrentEL::EL::Value::EL0) => (PrivilegeLevel::User, "EL0"), - _ => (PrivilegeLevel::Unknown, "Unknown"), - } -} - -/// Init exception handling by setting the exception vector base address register. -/// -/// - Changes the HW state of the executing core. -/// - The vector table and the symbol `__EXCEPTION_VECTORS_START` from the linker script must -/// adhere to the alignment and size constraints demanded by the ARMv8-A Architecture Reference -/// Manual. -pub fn handling_init() { - unsafe extern "Rust" { - static __EXCEPTION_VECTORS_START: UnsafeCell<()>; - } - - // Safety: __EXCEPTION_VECTORS_START is a valid pointer to exception vector table - // that is properly aligned and accessible during system initialization - unsafe { - set_vbar_el1_checked(__EXCEPTION_VECTORS_START.get() as u64) - .expect("Vector table properly aligned!"); - } - info!("[!] Exception traps set up"); -} - -/// Errors possibly returned from the traps module. -/// @todo a big over-engineered here. -#[derive(Debug, Snafu)] -enum Error { - /// IVT address is unaligned. - #[snafu(display("Unaligned base address for interrupt vector table"))] - Unaligned, -} - -/// Configure base address of interrupt vectors table. -/// Checks that address is properly 2KiB aligned. -/// -/// # Safety -/// -/// Totally unsafe in the land of the hardware. -unsafe fn set_vbar_el1_checked(vec_base_addr: u64) -> Result<(), Error> { - if vec_base_addr.trailing_zeros() < 11 { - return Err(Error::Unaligned); - } - - VBAR_EL1.set(vec_base_addr); - - // Force VBAR update to complete before next instruction. - barrier::isb(barrier::SY); - - Ok(()) -} diff --git a/libs/exception/src/arch/aarch64/exception_context.rs b/libs/exception/src/arch/aarch64/exception_context.rs new file mode 100644 index 000000000..9a09c0cd3 --- /dev/null +++ b/libs/exception/src/arch/aarch64/exception_context.rs @@ -0,0 +1,83 @@ +use { + aarch64_cpu::registers::{ESR_EL1, FAR_EL1, Readable}, + core::fmt, + tock_registers::LocalRegisterCopy, +}; + +/// The exception context as it is stored on the stack on exception entry. +/// Keep in sync with exception setup code in vectors.S! +#[repr(C)] +pub struct ExceptionContext { + /// General Purpose Registers, x0-x29 + pub gpr: [u64; 30], + /// The link register, aka x30. + pub lr: u64, + /// Saved program status. + pub spsr_el1: super::spsr_el1::SpsrEL1, + /// Exception link register. The program counter at the time the exception happened. + pub elr_el1: u64, +} + +impl ExceptionContext { + // #[inline(always)] + // fn exception_class(&self) -> Option { + // self.esr_el1.exception_class() + // } + + #[inline(always)] + fn fault_address_valid() -> bool { + use ESR_EL1::EC::Value::{ + DataAbortCurrentEL, DataAbortLowerEL, InstrAbortCurrentEL, InstrAbortLowerEL, + PCAlignmentFault, WatchpointCurrentEL, WatchpointLowerEL, + }; + + let esr_el1 = super::esr_el1::EsrEL1(LocalRegisterCopy::new(ESR_EL1.get())); + + match esr_el1.exception_class() { + None => false, + Some(ec) => matches!( + ec, + InstrAbortLowerEL + | InstrAbortCurrentEL + | PCAlignmentFault + | DataAbortLowerEL + | DataAbortCurrentEL + | WatchpointLowerEL + | WatchpointCurrentEL + ), + } + } + + pub fn write_gprs(&self, f: &mut fmt::Formatter) -> fmt::Result { + writeln!(f, "General purpose registers:")?; + + let alternating = |x| -> _ { if x % 2 == 0 { " " } else { "\n" } }; + + // Print two registers per line. + for (i, reg) in self.gpr.iter().enumerate() { + write!(f, " x{: <2}: {: >#018x}{}", i, reg, alternating(i))?; + } + Ok(()) + } +} + +/// Human readable print of the exception context. +impl fmt::Display for ExceptionContext { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + // writeln!(f, "{}", self.esr_el1)?; + + if Self::fault_address_valid() { + writeln!( + f, + "FAR_EL1: {:#018x}", + usize::try_from(FAR_EL1.get()).unwrap_or(0) + )?; + } + + writeln!(f, "{}", self.spsr_el1)?; + writeln!(f, "ELR_EL1: {:#018x} (return to)", self.elr_el1)?; + writeln!(f)?; + self.write_gprs(f)?; + write!(f, " lr : {:#018x}", self.lr) + } +} diff --git a/libs/exception/src/arch/aarch64/mod.rs b/libs/exception/src/arch/aarch64/mod.rs index aa4b15a44..1f006775e 100644 --- a/libs/exception/src/arch/aarch64/mod.rs +++ b/libs/exception/src/arch/aarch64/mod.rs @@ -1 +1,83 @@ -pub mod exception; +/* + * SPDX-License-Identifier: BlueOak-1.0.0 + * Copyright (c) Berkus Decker + */ + +//! Interrupt handling +//! +//! The base address is given by `VBAR_ELn` and each entry has a defined offset from this +//! base address. Each table has 16 entries, with each entry being 128 bytes (32 instructions) +//! in size. The table effectively consists of 4 sets of 4 entries. +//! +//! Minimal implementation to help catch MMU traps. +//! Reads `ESR_ELx` to understand why trap was taken. +//! +//! `VBAR_EL1`, `VBAR_EL2`, `VBAR_EL3` +//! +//! `CurrentEL` with `SP0`: +0x0 +//! +//! * Synchronous +//! * IRQ/vIRQ +//! * FIQ +//! * SError/vSError +//! +//! `CurrentEL` with `SPx`: +0x200 +//! +//! * Synchronous +//! * IRQ/vIRQ +//! * FIQ +//! * SError/vSError +//! +//! Lower EL using `AArch64`: +0x400 +//! +//! * Synchronous +//! * IRQ/vIRQ +//! * FIQ +//! * SError/vSError +//! +//! Lower EL using `AArch32`: +0x600 +//! +//! * Synchronous +//! * IRQ/vIRQ +//! * FIQ +//! * SError/vSError +//! +//! When the processor takes an exception to `AArch64` execution state, +//! all of the PSTATE interrupt masks is set automatically. This means +//! that further exceptions are disabled. If software is to support +//! nested exceptions, for example, to allow a higher priority interrupt +//! to interrupt the handling of a lower priority source, then software needs +//! to explicitly re-enable interrupts + +use { + crate::PrivilegeLevel, aarch64_cpu::registers::CurrentEL, tock_registers::interfaces::Readable, +}; + +mod debug; +pub mod esr_el1; +mod exception_context; +mod spsr_el1; + +pub use exception_context::ExceptionContext; + +// Helpers for exception debugging +pub use debug::{IssForDataAbort, cause_to_string, exception_dump, iss_dfsc_to_string}; + +// Exception vectors for syscall handing and general IRQ routing +core::arch::global_asm!(include_str!("vectors.S")); + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- + +/// The processor's current privilege level. +pub fn current_privilege_level() -> (PrivilegeLevel, &'static str) { + let el = CurrentEL.read_as_enum(CurrentEL::EL); + match el { + Some(CurrentEL::EL::Value::EL3) => (PrivilegeLevel::Unknown, "EL3"), + Some(CurrentEL::EL::Value::EL2) => (PrivilegeLevel::Hypervisor, "EL2"), + Some(CurrentEL::EL::Value::EL1) => (PrivilegeLevel::Kernel, "EL1"), + Some(CurrentEL::EL::Value::EL0) => (PrivilegeLevel::User, "EL0"), + _ => (PrivilegeLevel::Unknown, "Unknown"), + } +} diff --git a/libs/exception/src/arch/aarch64/spsr_el1.rs b/libs/exception/src/arch/aarch64/spsr_el1.rs new file mode 100644 index 000000000..5ce31a31c --- /dev/null +++ b/libs/exception/src/arch/aarch64/spsr_el1.rs @@ -0,0 +1,37 @@ +use {aarch64_cpu::registers::SPSR_EL1, core::fmt, tock_registers::LocalRegisterCopy}; + +#[repr(transparent)] +pub struct SpsrEL1(pub LocalRegisterCopy); + +/// Human readable `SPSR_EL1`. +#[rustfmt::skip] +impl fmt::Display for SpsrEL1 { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + // Raw value. + writeln!(f, "SPSR_EL1: {:#010x}", self.0.get())?; + + let to_flag_str = |x| -> _ { + if x { "Set" } else { "Not set" } + }; + + writeln!(f, " Flags:")?; + writeln!(f, " Negative (N): {}", to_flag_str(self.0.is_set(SPSR_EL1::N)))?; + writeln!(f, " Zero (Z): {}", to_flag_str(self.0.is_set(SPSR_EL1::Z)))?; + writeln!(f, " Carry (C): {}", to_flag_str(self.0.is_set(SPSR_EL1::C)))?; + writeln!(f, " Overflow (V): {}", to_flag_str(self.0.is_set(SPSR_EL1::V)))?; + + let to_mask_str = |x| -> _ { + if x { "Masked" } else { "Unmasked" } + }; + + writeln!(f, " Exception handling state:")?; + writeln!(f, " Debug (D): {}", to_mask_str(self.0.is_set(SPSR_EL1::D)))?; + writeln!(f, " SError (A): {}", to_mask_str(self.0.is_set(SPSR_EL1::A)))?; + writeln!(f, " IRQ (I): {}", to_mask_str(self.0.is_set(SPSR_EL1::I)))?; + writeln!(f, " FIQ (F): {}", to_mask_str(self.0.is_set(SPSR_EL1::F)))?; + + write!(f, " Illegal Execution State (IL): {}", + to_flag_str(self.0.is_set(SPSR_EL1::IL)) + ) + } +} diff --git a/libs/exception/src/arch/aarch64/exception/vectors.S b/libs/exception/src/arch/aarch64/vectors.S similarity index 58% rename from libs/exception/src/arch/aarch64/exception/vectors.S rename to libs/exception/src/arch/aarch64/vectors.S index 4d8959559..22aa709b6 100644 --- a/libs/exception/src/arch/aarch64/exception/vectors.S +++ b/libs/exception/src/arch/aarch64/vectors.S @@ -43,25 +43,45 @@ b 1b .endm -// The vector definitions +// Exception vector table must be 2KB aligned +// 16 entries × 128 bytes each = 2048 bytes total +// +// The table is organized as: +// - 4 entries for exceptions from current EL with SP_EL0 +// - 4 entries for exceptions from current EL with SP_ELx +// - 4 entries for exceptions from lower EL (AArch64) +// - 4 entries for exceptions from lower EL (AArch32) .section .vectors, "ax" +.balign 2048 .global __exception_vectors_start __exception_vectors_start: + // ═══════════════════════════════════════════════════════════════ + // Current EL with SP_EL0 (not used, we use SP_EL1) + // ═══════════════════════════════════════════════════════════════ SAVE_CONTEXT_CALL_HANDLER_AND_RESTORE current_el0_synchronous // 0x000 SAVE_CONTEXT_CALL_HANDLER_AND_RESTORE current_el0_irq // 0x080 FIQ_DUMMY // 0x100 SAVE_CONTEXT_CALL_HANDLER_AND_RESTORE current_el0_serror // 0x180 - SAVE_CONTEXT_CALL_HANDLER_AND_RESTORE current_elx_synchronous // 0x200 + // ═══════════════════════════════════════════════════════════════ + // Current EL with SP_ELx (kernel exceptions) + // ═══════════════════════════════════════════════════════════════ + SAVE_CONTEXT_CALL_HANDLER_AND_RESTORE current_elx_synchronous // 0x200 - SVC syscall in EL1/EL2 SAVE_CONTEXT_CALL_HANDLER_AND_RESTORE current_elx_irq // 0x280 FIQ_DUMMY // 0x300 SAVE_CONTEXT_CALL_HANDLER_AND_RESTORE current_elx_serror // 0x380 - SAVE_CONTEXT_CALL_HANDLER_AND_RESTORE lower_aarch64_synchronous // 0x400 + // ═══════════════════════════════════════════════════════════════ + // Lower EL using AArch64 (user-space syscalls and exceptions) + // ═══════════════════════════════════════════════════════════════ + SAVE_CONTEXT_CALL_HANDLER_AND_RESTORE lower_aarch64_synchronous // 0x400 - SVC syscall from user space SAVE_CONTEXT_CALL_HANDLER_AND_RESTORE lower_aarch64_irq // 0x480 FIQ_DUMMY // 0x500 SAVE_CONTEXT_CALL_HANDLER_AND_RESTORE lower_aarch64_serror // 0x580 + // ═══════════════════════════════════════════════════════════════ + // Lower EL using AArch32 (not supported) + // ═══════════════════════════════════════════════════════════════ SAVE_CONTEXT_CALL_HANDLER_AND_RESTORE lower_aarch32_synchronous // 0x600 SAVE_CONTEXT_CALL_HANDLER_AND_RESTORE lower_aarch32_irq // 0x680 FIQ_DUMMY // 0x700 diff --git a/libs/exception/src/arch/mod.rs b/libs/exception/src/arch/mod.rs index de0a2a6ad..1f57abe47 100644 --- a/libs/exception/src/arch/mod.rs +++ b/libs/exception/src/arch/mod.rs @@ -1 +1,2 @@ pub mod aarch64; +pub use aarch64::*; diff --git a/libs/exception/src/exception/asynchronous/mod.rs b/libs/exception/src/asynchronous/mod.rs similarity index 100% rename from libs/exception/src/exception/asynchronous/mod.rs rename to libs/exception/src/asynchronous/mod.rs diff --git a/libs/exception/src/exception/mod.rs b/libs/exception/src/exception/mod.rs deleted file mode 100644 index be67d6834..000000000 --- a/libs/exception/src/exception/mod.rs +++ /dev/null @@ -1,29 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 -// -// Copyright (c) 2020-2022 Andre Richter - -//! Synchronous and asynchronous exception handling. - -#[cfg(target_arch = "aarch64")] -use crate::arch::aarch64::exception as arch_exception; - -pub mod asynchronous; - -//-------------------------------------------------------------------------------------------------- -// Architectural Public Reexports -//-------------------------------------------------------------------------------------------------- -pub use arch_exception::{current_privilege_level, handling_init}; - -//-------------------------------------------------------------------------------------------------- -// Public Definitions -//-------------------------------------------------------------------------------------------------- - -/// Kernel privilege levels. -#[allow(missing_docs)] -#[derive(Eq, PartialEq)] -pub enum PrivilegeLevel { - User, - Kernel, - Hypervisor, - Unknown, -} diff --git a/libs/exception/src/lib.rs b/libs/exception/src/lib.rs index 7e1046f03..9672e6045 100644 --- a/libs/exception/src/lib.rs +++ b/libs/exception/src/lib.rs @@ -1,7 +1,35 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2020-2022 Andre Richter + +//! Synchronous and asynchronous exception handling. + #![no_std] #![no_main] #![feature(format_args_nl)] #![feature(stmt_expr_attributes)] pub mod arch; -pub mod exception; +pub mod asynchronous; + +#[cfg(target_arch = "aarch64")] +use crate::arch::aarch64 as arch_exception; + +//-------------------------------------------------------------------------------------------------- +// Architectural Public Reexports +//-------------------------------------------------------------------------------------------------- +pub use arch_exception::current_privilege_level; + +//-------------------------------------------------------------------------------------------------- +// Public Definitions +//-------------------------------------------------------------------------------------------------- + +/// Kernel privilege levels. +#[allow(missing_docs)] +#[derive(Eq, PartialEq)] +pub enum PrivilegeLevel { + User, + Kernel, + Hypervisor, + Unknown, +} diff --git a/nucleus/tests/exception.rs b/libs/exception/tests/exception.rs similarity index 80% rename from nucleus/tests/exception.rs rename to libs/exception/tests/exception.rs index 5a81f952a..f92d40cb6 100644 --- a/nucleus/tests/exception.rs +++ b/libs/exception/tests/exception.rs @@ -4,9 +4,9 @@ #![test_runner(libtest::test_runner)] #![reexport_test_harness_main = "test_main"] -use libexception::exception::{PrivilegeLevel, current_privilege_level}; +use libexception::exception::{current_privilege_level, PrivilegeLevel}; -mod common; +// mod common; /// libmachine unit tests must execute in kernel mode. #[test_case] diff --git a/libs/kernel-state/Cargo.toml b/libs/kernel-state/Cargo.toml index 3610e572d..b100d456c 100644 --- a/libs/kernel-state/Cargo.toml +++ b/libs/kernel-state/Cargo.toml @@ -1,7 +1,7 @@ [package] -name = "libkernel-state" +name = "vesper-kernel-state" -description = "Kernel boot state manager." +description = "Vesper nanokernel boot state manager." authors = { workspace = true } categories = { workspace = true } @@ -18,7 +18,12 @@ publish = false [badges] maintenance = { status = "experimental" } -[dependencies] +# [dev-dependencies] +# libtest = { workspace = true } + +[lib] +test = false +harness = false [lints] workspace = true diff --git a/libs/kernel-state/src/lib.rs b/libs/kernel-state/src/lib.rs index adb8d4c7c..20cabcad5 100644 --- a/libs/kernel-state/src/lib.rs +++ b/libs/kernel-state/src/lib.rs @@ -4,9 +4,9 @@ #![no_std] #![no_main] -#![feature(custom_test_frameworks)] -#![test_runner(libtest::test_runner)] -#![reexport_test_harness_main = "test_main"] +// #![feature(custom_test_frameworks)] +// #![test_runner(libtest::test_runner)] +// #![reexport_test_harness_main = "test_main"] //! State information about the kernel itself. diff --git a/libs/local-irq/Cargo.toml b/libs/local-irq/Cargo.toml index d1e7f8ef5..789726634 100644 --- a/libs/local-irq/Cargo.toml +++ b/libs/local-irq/Cargo.toml @@ -1,7 +1,7 @@ [package] -name = "liblocal-irq" +name = "vesper-local-irq" -description = "Local IRQ management." +description = "Vesper nanokernel local IRQ management." authors = { workspace = true } categories = { workspace = true } @@ -23,5 +23,12 @@ aarch64-cpu = { workspace = true } liblog = { workspace = true } tock-registers = { workspace = true } +# [dev-dependencies] +# libtest = { workspace = true } + +[lib] +test = false +harness = false + [lints] workspace = true diff --git a/libs/local-irq/src/lib.rs b/libs/local-irq/src/lib.rs index 1bd15fa04..caaf1fb9f 100644 --- a/libs/local-irq/src/lib.rs +++ b/libs/local-irq/src/lib.rs @@ -1,8 +1,8 @@ #![no_std] #![no_main] -#![feature(custom_test_frameworks)] -#![test_runner(libtest::test_runner)] -#![reexport_test_harness_main = "test_main"] +// #![feature(custom_test_frameworks)] +// #![test_runner(libtest::test_runner)] +// #![reexport_test_harness_main = "test_main"] pub mod arch; pub use arch::*; diff --git a/libs/locking/Cargo.toml b/libs/locking/Cargo.toml index 1bbb91af1..16a447574 100644 --- a/libs/locking/Cargo.toml +++ b/libs/locking/Cargo.toml @@ -1,7 +1,7 @@ [package] -name = "liblocking" +name = "vesper-locking" -description = "Synchronization primitives." +description = "Vesper nanokernel synchronization primitives." authors = { workspace = true } categories = { workspace = true } @@ -22,5 +22,12 @@ maintenance = { status = "experimental" } libkernel-state = { workspace = true } liblocal-irq = { workspace = true } +# [dev-dependencies] +# libtest = { workspace = true } + +[lib] +test = false +harness = false + [lints] workspace = true diff --git a/libs/locking/src/lib.rs b/libs/locking/src/lib.rs index d1795bff2..f6ba81be0 100644 --- a/libs/locking/src/lib.rs +++ b/libs/locking/src/lib.rs @@ -6,9 +6,9 @@ #![no_std] #![no_main] -#![feature(custom_test_frameworks)] -#![test_runner(libtest::test_runner)] -#![reexport_test_harness_main = "test_main"] +// #![feature(custom_test_frameworks)] +// #![test_runner(libtest::test_runner)] +// #![reexport_test_harness_main = "test_main"] use core::cell::UnsafeCell; diff --git a/nucleus/tests/locking.rs b/libs/locking/tests/locking.rs similarity index 96% rename from nucleus/tests/locking.rs rename to libs/locking/tests/locking.rs index e42a6648c..3450dc1a2 100644 --- a/nucleus/tests/locking.rs +++ b/libs/locking/tests/locking.rs @@ -4,7 +4,7 @@ #![test_runner(libtest::test_runner)] #![reexport_test_harness_main = "test_main"] -mod common; +// mod common; /// InitStateLock must be transparent. #[test_case] diff --git a/libs/log/Cargo.toml b/libs/log/Cargo.toml index 331de2cf8..dc372c77b 100644 --- a/libs/log/Cargo.toml +++ b/libs/log/Cargo.toml @@ -1,7 +1,7 @@ [package] -name = "liblog" +name = "vesper-log" -description = "Kernel logger modeled after log crate." +description = "Vesper nanokernel logger modeled after log crate." authors = { workspace = true } categories = { workspace = true } @@ -18,7 +18,13 @@ publish = false [badges] maintenance = { status = "experimental" } -[dependencies] +# [dependencies] +# libtest = { workspace = true } cyclic dep + +# Tests are offloaded to kernel integration tests binary. +[lib] +test = false +harness = false [lints] workspace = true diff --git a/libs/log/src/lib.rs b/libs/log/src/lib.rs index f5dc21d56..e4a88e444 100644 --- a/libs/log/src/lib.rs +++ b/libs/log/src/lib.rs @@ -52,7 +52,7 @@ static LEVEL_PARSE_ERROR: &str = /// An enum representing the available verbosity levels of the logger. /// #[repr(isize)] -#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug, Hash)] +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] pub enum Level { Print = -2, PrintLn = -1, @@ -95,13 +95,21 @@ impl Level { /// /// # Examples /// - /// ``` - /// use log::Level; + /// ```no_run + /// # #![no_std] + /// # #![no_main] + /// # #[panic_handler] fn panic(_: &core::panic::PanicInfo) -> ! { loop {} } + /// # #[unsafe(no_mangle)] pub extern "C" fn main() { + /// # use core::assert_eq; + /// # use core::iter::Iterator; + /// # use core::option::Option::Some; + /// use vesper_log::Level; /// /// let mut levels = Level::iter(); /// /// assert_eq!(Some(Level::Error), levels.next()); /// assert_eq!(Some(Level::Trace), levels.last()); + /// # } /// ``` pub fn iter() -> impl Iterator { (1..6).map(|i| Self::from_usize(i).unwrap()) @@ -114,14 +122,20 @@ impl Level { /// /// # Examples /// - /// ``` - /// use log::Level; + /// ```no_run + /// # #![no_std] + /// # #![no_main] + /// # #[panic_handler] fn panic(_: &core::panic::PanicInfo) -> ! { loop {} } + /// # #[unsafe(no_mangle)] pub extern "C" fn main() { + /// # use core::assert_eq; + /// use vesper_log::Level; /// /// let level = Level::Info; /// /// assert_eq!(Level::Debug, level.increment_severity()); /// assert_eq!(Level::Trace, level.increment_severity().increment_severity()); /// assert_eq!(Level::Trace, level.increment_severity().increment_severity().increment_severity()); // max level + /// # } /// ``` #[must_use] pub fn increment_severity(&self) -> Self { @@ -136,14 +150,20 @@ impl Level { /// /// # Examples /// - /// ``` - /// use log::Level; + /// ```no_run + /// # #![no_std] + /// # #![no_main] + /// # #[panic_handler] fn panic(_: &core::panic::PanicInfo) -> ! { loop {} } + /// # #[unsafe(no_mangle)] pub extern "C" fn main() { + /// # use core::assert_eq; + /// use vesper_log::Level; /// /// let level = Level::Info; /// /// assert_eq!(Level::Warn, level.decrement_severity()); /// assert_eq!(Level::Error, level.decrement_severity().decrement_severity()); /// assert_eq!(Level::Error, level.decrement_severity().decrement_severity().decrement_severity()); // min level + /// # } /// ``` #[must_use] pub fn decrement_severity(&self) -> Self { @@ -152,7 +172,7 @@ impl Level { } } -#[derive(Clone, Debug)] +#[derive(Clone)] pub struct Record<'a> { level: Level, args: fmt::Arguments<'a>, @@ -178,7 +198,6 @@ impl<'a> Record<'a> { } } -#[derive(Debug)] pub struct RecordBuilder<'a> { record: Record<'a>, } @@ -375,29 +394,28 @@ pub fn max_level() -> Level { /// /// # Examples /// -/// ``` -/// use log::{error, info, warn, Record, Level, Metadata, LevelFilter}; +/// ```no_run +/// # #![no_std] +/// # #![no_main] +/// # #![feature(format_args_nl)] +/// # #[panic_handler] fn panic(_: &core::panic::PanicInfo) -> ! { loop {} } +/// use vesper_log::{error, info, warn, Record, Level}; /// /// static MY_LOGGER: MyLogger = MyLogger; /// /// struct MyLogger; /// -/// impl log::Log for MyLogger { -/// fn enabled(&self, metadata: &Metadata) -> bool { -/// metadata.level() <= Level::Info -/// } -/// +/// impl vesper_log::Log for MyLogger { +/// fn enabled(&self, level: Level) -> bool { true } /// fn log(&self, record: &Record) { -/// if self.enabled(record.metadata()) { -/// println!("{} - {}", record.level(), record.args()); -/// } +/// vesper_log::println!("{:?} - {}", record.level(), record.args()); /// } /// fn flush(&self) {} /// } /// -/// # fn main(){ -/// log::set_logger(&MY_LOGGER).unwrap(); -/// log::set_max_level(LevelFilter::Info); +/// # #[unsafe(no_mangle)] fn main(){ +/// vesper_log::set_logger(&MY_LOGGER).unwrap(); +/// vesper_log::set_max_level(Level::Info); /// /// info!("hello log"); /// warn!("warning"); @@ -527,3 +545,21 @@ pub fn logger() -> &'static dyn Log { &NOP } } + +/// Convert a size into human readable format. +/// FIXME: A candidate for libdebug? +pub const fn size_human_readable_ceil(size: usize) -> (usize, &'static str) { + const KIB: usize = 1024; + const MIB: usize = 1024 * 1024; + const GIB: usize = 1024 * 1024 * 1024; + + if (size / GIB) > 0 { + (size.div_ceil(GIB), "GiB") + } else if (size / MIB) > 0 { + (size.div_ceil(MIB), "MiB") + } else if (size / KIB) > 0 { + (size.div_ceil(KIB), "KiB") + } else { + (size, "B") + } +} diff --git a/libs/machine/Cargo.toml b/libs/machine/Cargo.toml index 2c06df8cd..8a751146c 100644 --- a/libs/machine/Cargo.toml +++ b/libs/machine/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "libmachine" +name = "vesper-machines" description = "Vesper nanokernel shared code library, useful also for the chainboot loader." @@ -26,31 +26,15 @@ jtag = [] qemu = [] [dependencies] -aarch64-cpu = { workspace = true } -bit_field = { workspace = true } -bitflags = { workspace = true } -buddy-alloc = { workspace = true } -cfg-if = { workspace = true } -libconsole = { workspace = true } libcpu = { workspace = true } libexception = { workspace = true } -libkernel-state = { workspace = true } liblocal-irq = { workspace = true } -liblocking = { workspace = true } liblog = { workspace = true } -libmemory = { workspace = true } -libplatform = { workspace = true } -libprimitives = { workspace = true } libqemu = { workspace = true } -libtime = { workspace = true } -once_cell = { workspace = true } -snafu = { workspace = true } -tock-registers = { workspace = true } -usize_conversions = { workspace = true } -ux = { workspace = true } [lib] test = false +harness = false [lints] workspace = true diff --git a/libs/mapping/Cargo.toml b/libs/mapping/Cargo.toml new file mode 100644 index 000000000..06ee6b770 --- /dev/null +++ b/libs/mapping/Cargo.toml @@ -0,0 +1,32 @@ +[package] +name = "vesper-mapping" + +description = "Vesper nanokernel memory mapping abstractions." + +authors = { workspace = true } +categories = { workspace = true } +documentation = { workspace = true } +edition = { workspace = true } +homepage = { workspace = true } +license = { workspace = true } +readme = { workspace = true } +repository = { workspace = true } +version = { workspace = true } +publish = false + +[badges] +maintenance = { status = "experimental" } + +[dependencies] +libaddress = { workspace = true } +# liblocking = { workspace = true } +liblog = { workspace = true } +libmmio = { workspace = true } + +# Tests are offloaded to kernel integration tests binary. +[lib] +test = false +harness = false + +[lints] +workspace = true diff --git a/libs/mapping/README.md b/libs/mapping/README.md new file mode 100644 index 000000000..1e221b251 --- /dev/null +++ b/libs/mapping/README.md @@ -0,0 +1,2 @@ +This lib should actually be called libmapping and should handle mapping record +tracking for maps/unmaps done from privileged processes. diff --git a/libs/mapping/src/lib.rs b/libs/mapping/src/lib.rs new file mode 100644 index 000000000..f32f3a363 --- /dev/null +++ b/libs/mapping/src/lib.rs @@ -0,0 +1,118 @@ +#![no_std] +#![feature(const_trait_impl)] +#![feature(step_trait)] +#![feature(new_range_api)] + +use { + libaddress::{Address, Virtual}, + libmmio::MMIODescriptor, +}; + +mod mapping_attributes; +mod mapping_record; +mod memory_region; +mod page_address; + +pub use {mapping_attributes::*, memory_region::*, page_address::*}; + +//-------------------------------------------------------------------------------------------------- +// Public Code +// FIXME: this code should move to kickstart and be only used during boot-up! (see paging.rs) +//-------------------------------------------------------------------------------------------------- + +// Raw mapping of a virtual to physical region in the kernel translation tables. +// +// Prevents mapping into the MMIO range of the tables. +// +// Safety +// +// - See `kernel_map_at_unchecked()`. +// - Does not prevent aliasing. Currently, the callers must be trusted. +// pub unsafe fn kernel_map_at( +// name: &'static str, +// virt_region: &MemoryRegion, +// phys_region: &MemoryRegion, +// attr: AttributeFields, +// ) -> Result<(), &'static str> { +// // if platform::memory::mmu::virt_mmio_remap_region().overlaps(virt_region) { +// // return Err("Attempt to manually map into MMIO region"); +// // } +// unsafe { +// kernel_map_at_unchecked(name, virt_region, phys_region, attr)?; +// } +// Ok(()) +// } + +/// MMIO remapping in the kernel translation tables. +/// +/// Typically used by device drivers. +/// +/// # Safety +/// +/// - Same as `kernel_map_at_unchecked()`, minus the aliasing part. +pub unsafe fn kernel_map_mmio( + _name: &'static str, + _mmio_descriptor: &MMIODescriptor, +) -> Result, &'static str> { + // let phys_region = MemoryRegion::from(*mmio_descriptor); + // let offset_into_start_page = mmio_descriptor.start_addr().offset_into_page(&4096); // FIXME: hardcoded page size + + // // Check if an identical region has been mapped for another driver. If so, reuse it. + // let virt_addr = if let Some(addr) = + // mapping_record::kernel_find_and_insert_mmio_duplicate(mmio_descriptor, name) + // { + // addr + // // Otherwise, allocate a new region and map it. + // } else { + // let Some(num_pages) = NonZeroUsize::new(phys_region.num_pages()) else { + // return Err("Requested 0 pages"); + // }; + + // let virt_region = + // page_alloc::kernel_mmio_va_allocator().lock(|allocator| allocator.alloc(num_pages))?; + + // unsafe { + // kernel_map_at_unchecked( + // name, + // &virt_region, + // &phys_region, + // AttributeFields { + // mem_attributes: MemAttributes::Device, + // acc_perms: AccessPermissions::ReadWrite, + // execute_never: true, + // }, + // )?; + // } + + // virt_region.start_addr() + // }; + + // Ok(virt_addr + offset_into_start_page) + Ok(Address::zero()) +} + +// Map a region in the kernel's translation tables. +// +// No input checks done, input is passed through to the architectural implementation (syscall?). +// +// # Safety +// +// - See `map_at()`. +// - Does not prevent aliasing. +// unsafe fn kernel_map_at_unchecked( +// name: &'static str, +// virt_region: &MemoryRegion, +// phys_region: &MemoryRegion, +// attr: AttributeFields, +// ) -> Result<(), &'static str> { +// crate::platform::memory::mmu::kernel_translation_tables().write(|tables| +// // SAFETY: Make a mistake and you're dead, gaijin! +// unsafe { tables.map_at(virt_region, phys_region, attr) })?; + +// if let Err(x) = mapping_record::kernel_add(name, virt_region, phys_region, attr) { +// // warn!("{x}"); +// return Err(x); +// } + +// Ok(()) +// } diff --git a/libs/mapping/src/mapping_attributes.rs b/libs/mapping/src/mapping_attributes.rs new file mode 100644 index 000000000..674b13481 --- /dev/null +++ b/libs/mapping/src/mapping_attributes.rs @@ -0,0 +1,84 @@ +use core::fmt::{self, Formatter}; + +/// Architecture agnostic memory attributes. +#[derive(Copy, Clone, Debug, Eq, PartialOrd, PartialEq)] +pub enum MemAttributes { + /// Regular memory + CacheableDRAM, + /// Memory without caching + NonCacheableDRAM, + /// Device memory + Device, +} + +/// Architecture agnostic memory region access permissions. +#[derive(Copy, Clone, Debug, Eq, PartialOrd, PartialEq)] +pub enum AccessPermissions { + /// Read-write access + ReadWrite, + /// Read-only access + ReadOnly, +} + +/// Summary structure of memory region properties. +#[derive(Copy, Clone, Debug, Eq, PartialOrd, PartialEq)] +pub struct AttributeFields { + /// Attributes + pub mem_attributes: MemAttributes, + /// Permissions + pub acc_perms: AccessPermissions, + /// Disable executable code in this region + pub executable: bool, + /// Is the region occupied or free (use occupied for const init) + pub occupied: bool, + /// If this memory can be reclaimed into an Untyped after `kickstart` completes + pub droppable: bool, +} + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- + +impl AttributeFields { + pub const fn defaulted() -> AttributeFields { + AttributeFields { + mem_attributes: MemAttributes::CacheableDRAM, + acc_perms: AccessPermissions::ReadWrite, + executable: false, + occupied: false, + droppable: false, + } + } +} + +impl Default for AttributeFields { + fn default() -> Self { + Self::defaulted() + } +} + +/// Human-readable output of `AttributeFields` +impl fmt::Display for AttributeFields { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + let attr = match self.mem_attributes { + MemAttributes::CacheableDRAM => "C", + MemAttributes::NonCacheableDRAM => "NC", + MemAttributes::Device => "Dev", + }; + + let acc_p = match self.acc_perms { + AccessPermissions::ReadOnly => "RO", + AccessPermissions::ReadWrite => "RW", + }; + + let xn = if self.executable { "PX" } else { "PXN" }; + + let marker = if self.droppable { + "Drop" + } else { + if self.occupied { "Used" } else { "Free" } + }; + + write!(f, "({marker}) {attr: <3} {acc_p} {xn: <3}") + } +} diff --git a/libs/memory/src/mmu/mapping_record.rs b/libs/mapping/src/mapping_record.rs similarity index 64% rename from libs/memory/src/mmu/mapping_record.rs rename to libs/mapping/src/mapping_record.rs index cc180da36..0addb12cc 100644 --- a/libs/memory/src/mmu/mapping_record.rs +++ b/libs/mapping/src/mapping_record.rs @@ -5,13 +5,9 @@ //! A record of mapped pages. use { - super::{ - Address, Physical, Virtual, - types::{AccessPermissions, AttributeFields, MMIODescriptor, MemAttributes, MemoryRegion}, - }, - crate::{mm, platform}, - liblocking::{self, InitStateLock}, - liblog::{info, warn}, + super::{AccessPermissions, AttributeFields, MMIODescriptor, MemAttributes, MemoryRegion}, + libaddress::{Address, Physical, Virtual}, + liblog::info, }; //-------------------------------------------------------------------------------------------------- @@ -19,9 +15,9 @@ use { //-------------------------------------------------------------------------------------------------- /// Type describing a virtual memory mapping. -#[allow(missing_docs)] +#[allow(missing_docs, dead_code)] #[derive(Copy, Clone)] -struct MappingRecordEntry { +struct MappingRecordEntry { pub users: [Option<&'static str>; 5], pub phys_start_addr: Address, pub virt_start_addr: Address, @@ -29,26 +25,29 @@ struct MappingRecordEntry { pub attribute_fields: AttributeFields, } -struct MappingRecord { - inner: [Option; 12], +#[allow(missing_docs, dead_code)] +struct MappingRecord { + inner: [Option>; 12], } //-------------------------------------------------------------------------------------------------- // Global instances //-------------------------------------------------------------------------------------------------- -static KERNEL_MAPPING_RECORD: InitStateLock = - InitStateLock::new(MappingRecord::new()); +// FIXME: global state +// static KERNEL_MAPPING_RECORD: InitStateLock> = +// InitStateLock::new(MappingRecord::new()); //-------------------------------------------------------------------------------------------------- // Private Code //-------------------------------------------------------------------------------------------------- -impl MappingRecordEntry { +impl MappingRecordEntry { + #[allow(missing_docs, dead_code)] pub fn new( name: &'static str, - virt_region: &MemoryRegion, - phys_region: &MemoryRegion, + virt_region: &MemoryRegion, + phys_region: &MemoryRegion, attr: AttributeFields, ) -> Self { Self { @@ -60,6 +59,7 @@ impl MappingRecordEntry { } } + #[allow(missing_docs, dead_code)] fn find_next_free_user(&mut self) -> Result<&mut Option<&'static str>, &'static str> { if let Some(x) = self.users.iter_mut().find(|x| x.is_none()) { return Ok(x); @@ -68,6 +68,7 @@ impl MappingRecordEntry { Err("Storage for user info exhausted") } + #[allow(missing_docs, dead_code)] pub fn add_user(&mut self, user: &'static str) -> Result<(), &'static str> { let x = self.find_next_free_user()?; *x = Some(user); @@ -75,15 +76,18 @@ impl MappingRecordEntry { } } -impl MappingRecord { +impl MappingRecord { + #[allow(missing_docs, dead_code)] pub const fn new() -> Self { Self { inner: [None; 12] } } + #[allow(missing_docs, dead_code)] fn size(&self) -> usize { self.inner.iter().filter(|x| x.is_some()).count() } + #[allow(missing_docs, dead_code)] fn sort(&mut self) { let upper_bound_exclusive = self.size(); let entries = &mut self.inner.get_mut(0..upper_bound_exclusive).unwrap(); @@ -93,7 +97,10 @@ impl MappingRecord { } } - fn find_next_free(&mut self) -> Result<&mut Option, &'static str> { + #[allow(missing_docs, dead_code)] + fn find_next_free( + &mut self, + ) -> Result<&mut Option>, &'static str> { if let Some(x) = self.inner.iter_mut().find(|x| x.is_none()) { return Ok(x); } @@ -101,32 +108,39 @@ impl MappingRecord { Err("Storage for mapping info exhausted") } + #[allow(missing_docs, dead_code)] fn find_duplicate( &mut self, - phys_region: &MemoryRegion, - ) -> Option<&mut MappingRecordEntry> { + phys_region: &MemoryRegion, + ) -> Option<&mut MappingRecordEntry> { self.inner .iter_mut() .filter_map(|x| x.as_mut()) .filter(|x| x.attribute_fields.mem_attributes == MemAttributes::Device) .find(|x| { - if x.phys_start_addr != phys_region.start_addr() { - return false; - } - - if x.num_pages != phys_region.num_pages() { - return false; - } - - true + x.phys_start_addr == phys_region.start_addr() + && x.num_pages == phys_region.num_pages() }) } + /// Adds a new mapping to the mapping record. + /// + /// # Arguments + /// + /// * `name` - The name of the entity that owns the mapping. + /// * `virt_region` - The virtual memory region being mapped. + /// * `phys_region` - The physical memory region being mapped. + /// * `attr` - The memory attributes of the mapping. + /// + /// # Returns + /// + /// Returns `Ok(())` on success, or a string error message on failure. + #[allow(missing_docs, dead_code)] pub fn add( &mut self, name: &'static str, - virt_region: &MemoryRegion, - phys_region: &MemoryRegion, + virt_region: &MemoryRegion, + phys_region: &MemoryRegion, attr: AttributeFields, ) -> Result<(), &'static str> { let x = self.find_next_free()?; @@ -143,6 +157,7 @@ impl MappingRecord { Ok(()) } + #[allow(missing_docs, dead_code)] pub fn print(&self) { info!( " -------------------------------------------------------------------------------------------------------------------------------------------" @@ -156,13 +171,13 @@ impl MappingRecord { ); for i in self.inner.iter().flatten() { - let size = i.num_pages * platform::KernelGranule::SIZE; + let size = i.num_pages * PAGE_SIZE; let virt_start = i.virt_start_addr; let virt_end_inclusive = virt_start + (size - 1); let phys_start = i.phys_start_addr; let phys_end_inclusive = phys_start + (size - 1); - let (size, unit) = mm::size_human_readable_ceil(size); + let (size, unit) = liblog::size_human_readable_ceil(size); let attr = match i.attribute_fields.mem_attributes { MemAttributes::CacheableDRAM => "C", @@ -175,10 +190,10 @@ impl MappingRecord { AccessPermissions::ReadWrite => "RW", }; - let xn = if i.attribute_fields.execute_never { - "XN" - } else { + let xn = if i.attribute_fields.executable { "X" + } else { + "XN" }; info!( @@ -213,36 +228,40 @@ impl MappingRecord { //-------------------------------------------------------------------------------------------------- // Public Code //-------------------------------------------------------------------------------------------------- -use liblocking::interface::ReadWriteEx; /// Add an entry to the mapping info record. -pub fn kernel_add( - name: &'static str, - virt_region: &MemoryRegion, - phys_region: &MemoryRegion, - attr: AttributeFields, +#[expect(dead_code, clippy::unnecessary_wraps)] +pub fn kernel_add( + _name: &'static str, + _virt_region: &MemoryRegion, + _phys_region: &MemoryRegion, + _attr: AttributeFields, ) -> Result<(), &'static str> { - KERNEL_MAPPING_RECORD.write(|mr| mr.add(name, virt_region, phys_region, attr)) + // KERNEL_MAPPING_RECORD.write(|mr| mr.add(name, virt_region, phys_region, attr)) + Ok(()) } -pub fn kernel_find_and_insert_mmio_duplicate( - mmio_descriptor: &MMIODescriptor, - new_user: &'static str, +#[expect(dead_code, clippy::unnecessary_wraps)] +pub fn kernel_find_and_insert_mmio_duplicate( + _mmio_descriptor: &MMIODescriptor, + _new_user: &'static str, ) -> Option> { - let phys_region: MemoryRegion = (*mmio_descriptor).into(); + // let phys_region: MemoryRegion = (*mmio_descriptor).into(); - KERNEL_MAPPING_RECORD.write(|mr| { - let dup = mr.find_duplicate(&phys_region)?; + // KERNEL_MAPPING_RECORD.write(|mr| { + // let dup = mr.find_duplicate(&phys_region)?; - if let Err(x) = dup.add_user(new_user) { - warn!("{x}"); - } + // if let Err(x) = dup.add_user(new_user) { + // warn!("{x}"); + // } - Some(dup.virt_start_addr) - }) + // Some(dup.virt_start_addr) + // }) + Some(Address::zero()) } /// Human-readable print of all recorded kernel mappings. +#[allow(missing_docs, dead_code)] pub fn kernel_print() { - KERNEL_MAPPING_RECORD.read(MappingRecord::print); + // KERNEL_MAPPING_RECORD.read(MappingRecord::print); } diff --git a/libs/mapping/src/memory_region.rs b/libs/mapping/src/memory_region.rs new file mode 100644 index 000000000..6df2b6f60 --- /dev/null +++ b/libs/mapping/src/memory_region.rs @@ -0,0 +1,140 @@ +use { + crate::{MMIODescriptor, PageAddress}, + core::{iter::Step, num::NonZeroUsize, ops::Range}, + libaddress::{Address, AddressType, Physical}, +}; + +/// A type that describes a region of memory in quantities of pages. +#[derive(Copy, Clone, Debug, Eq, PartialOrd, PartialEq)] +pub struct MemoryRegion { + start: PageAddress, + end_exclusive: PageAddress, +} + +impl MemoryRegion { + /// Create an instance. + pub fn new( + start: PageAddress, + end_exclusive: PageAddress, + ) -> Self { + assert!(start <= end_exclusive); + + Self { + start, + end_exclusive, + } + } + + fn as_range(&self) -> Range> { + self.into_iter() + } + + /// Returns the start page address. + pub fn start_page_addr(&self) -> PageAddress { + self.start + } + + /// Returns the start address. + pub fn start_addr(&self) -> Address { + self.start.into_inner() + } + + /// Returns the exclusive end page address. + pub fn end_exclusive_page_addr(&self) -> PageAddress { + self.end_exclusive + } + + /// Returns the exclusive end page address. + pub fn end_inclusive_page_addr(&self) -> PageAddress { + self.end_exclusive.checked_page_offset(-1).unwrap() + } + + /// Checks if self contains an address. + pub fn contains(&self, addr: Address) -> bool { + let page_addr = PageAddress::::from(addr.align_down_page(&PAGE_SIZE)); + self.as_range().contains(&page_addr) + } + + /// Checks if there is an overlap with another memory region. + pub fn overlaps(&self, other_region: &Self) -> bool { + let self_range = self.as_range(); + + self_range.contains(&other_region.start_page_addr()) + || self_range.contains(&other_region.end_inclusive_page_addr()) + } + + /// Returns the number of pages contained in this region. + pub fn num_pages(&self) -> usize { + PageAddress::steps_between(&self.start, &self.end_exclusive).0 + } + + /// Returns the size in bytes of this region. + pub fn size(&self) -> usize { + // Invariant: start <= end_exclusive, so do unchecked arithmetic. + let end_exclusive = self.end_exclusive.into_inner().as_usize(); + let start = self.start.into_inner().as_usize(); + + end_exclusive - start + } + + /// Splits the `MemoryRegion` like in the following diagram. + /// Left region is returned to the caller. Right region is the new region for this struct. + /// + /// -------------------------------------------------------------------------------- + /// | | | | | | | | | | | | | | | | | | | + /// -------------------------------------------------------------------------------- + /// ^ ^ ^ + /// | | | + /// `left_start` `left_end_exclusive` | + /// | + /// ^ | + /// | | + /// `right_start` `right_end_exclusive` + /// + pub fn take_first_n_pages(&mut self, num_pages: NonZeroUsize) -> Result { + let count: usize = num_pages.into(); + + let left_end_exclusive = self.start.checked_page_offset(count.cast_signed()); + let Some(left_end_exclusive) = left_end_exclusive else { + return Err("Overflow while calculating left_end_exclusive"); + }; + + if left_end_exclusive > self.end_exclusive { + return Err("Not enough free pages"); + } + + let allocation = Self { + start: self.start, + end_exclusive: left_end_exclusive, + }; + self.start = left_end_exclusive; + + Ok(allocation) + } +} + +impl IntoIterator + for MemoryRegion +{ + type Item = PageAddress; + type IntoIter = Range; + + fn into_iter(self) -> Self::IntoIter { + Range { + start: self.start, + end: self.end_exclusive, + } + } +} + +impl From for MemoryRegion { + fn from(desc: MMIODescriptor) -> Self { + let start = PageAddress::from(desc.start_addr().align_down_page(&PAGE_SIZE)); + let end_exclusive = PageAddress::from(desc.end_addr_exclusive().align_up_page(&PAGE_SIZE)); + + Self { + start, + end_exclusive, + } + } +} diff --git a/libs/mapping/src/page_address.rs b/libs/mapping/src/page_address.rs new file mode 100644 index 000000000..c605d5b3c --- /dev/null +++ b/libs/mapping/src/page_address.rs @@ -0,0 +1,86 @@ +use { + core::iter::Step, + libaddress::{Address, AddressType}, +}; + +/// A wrapper type around [Address] that ensures page alignment. +#[derive(Copy, Clone, Debug, Eq, PartialOrd, PartialEq)] +pub struct PageAddress { + inner: Address, +} + +impl PageAddress { + /// Unwraps the value. + pub fn into_inner(self) -> Address { + self.inner + } + + /// Calculates the offset from the page address. + /// + /// `count` is in units of [`PageAddress`]. For example, a count of 2 means `result = self + 2 * + /// page_size`. + pub fn checked_page_offset(self, count: isize) -> Option { + if count == 0 { + return Some(self); + } + + let delta = count.unsigned_abs().checked_mul(PAGE_SIZE)? as u64; + let result = if count.is_positive() { + self.inner.as_u64().checked_add(delta)? + } else { + self.inner.as_u64().checked_sub(delta)? + }; + + Some(Self { + inner: Address::::new(result), + }) + } +} + +impl From + for PageAddress +{ + fn from(addr: usize) -> Self { + assert!( + libaddress::align::is_aligned(addr as u64, PAGE_SIZE as u64), + "Input usize not page aligned" + ); + + Self { + inner: Address::::new(addr as u64), + } + } +} + +impl From> + for PageAddress +{ + fn from(addr: Address) -> Self { + assert!( + addr.is_page_aligned(&PAGE_SIZE), + "Input Address not page aligned" + ); + + Self { inner: addr } + } +} + +impl Step for PageAddress { + fn steps_between(start: &Self, end: &Self) -> (usize, Option) { + if start > end { + return (0, None); + } + + // Since start <= end, do unchecked arithmetic. + let steps = (end.inner.as_usize() - start.inner.as_usize()) / PAGE_SIZE; + (steps, Some(steps)) + } + + fn forward_checked(start: Self, count: usize) -> Option { + start.checked_page_offset(count.cast_signed()) + } + + fn backward_checked(start: Self, count: usize) -> Option { + start.checked_page_offset(-(count.cast_signed())) + } +} diff --git a/libs/memory/Cargo.toml b/libs/memory/Cargo.toml index 616a246e6..1371c7894 100644 --- a/libs/memory/Cargo.toml +++ b/libs/memory/Cargo.toml @@ -1,7 +1,7 @@ [package] -name = "libmemory" +name = "vesper-memory" -description = "Vesper nanokernel Memory Descriptors shared code library, useful also for the ttt tool." +description = "Vesper nanokernel memory descriptors shared code library." authors = { workspace = true } categories = { workspace = true } @@ -20,20 +20,15 @@ maintenance = { status = "experimental" } [dependencies] aarch64-cpu = { workspace = true } -bit_field = { workspace = true } -bitflags = { workspace = true } -buddy-alloc = { workspace = true } -cfg-if = { workspace = true } -liblocking = { workspace = true } +libaddress = { workspace = true } liblog = { workspace = true } -once_cell = { workspace = true } +libmapping = { workspace = true } snafu = { workspace = true } tock-registers = { workspace = true } -usize_conversions = { workspace = true } -ux = { workspace = true } [lib] test = false +harness = false [lints] workspace = true diff --git a/libs/memory/README.md b/libs/memory/README.md new file mode 100644 index 000000000..a2a036965 --- /dev/null +++ b/libs/memory/README.md @@ -0,0 +1,24 @@ +# libmemory + +libmemory contains only types (structs and traits) and abstractions useful for memory translation. +It includes memory mapping tables layout and MMU interface. + +## Exports + +Page table types must represent pages of differing sizes. +For every entry in the MMU page table we should be able to receive a proper page type - e.g. Invalid, further page table, or a specific-size page. + +This library should export the following: + +- MMU control interface under `interface::MMU` +- Page-table hierarchy representation. This needs to be platform-independent. + Abstract translation stages and page size granularity. + + +---------------- work with MMU structures and kernel knowledge of mappings (THIS LIBRARY) + | +----- work with higher level mapping abstractions, without MMU details? + v v +libmmu -> libmapping + +--- + +For more information please re-read. diff --git a/libs/memory/TODO.md b/libs/memory/TODO.md new file mode 100644 index 000000000..45ba1473c --- /dev/null +++ b/libs/memory/TODO.md @@ -0,0 +1,2 @@ +- [x] Expand to include PowerPC +- [ ] Expand to include other granule sizes (16K and 64K for arm, to test how it scales) diff --git a/libs/memory/docs/970MP_um.2008MAR07_pub.pdf b/libs/memory/docs/970MP_um.2008MAR07_pub.pdf new file mode 100644 index 000000000..b6bc155d0 Binary files /dev/null and b/libs/memory/docs/970MP_um.2008MAR07_pub.pdf differ diff --git a/libs/memory/docs/paging.puml b/libs/memory/docs/paging.puml new file mode 100644 index 000000000..de1ca8e72 --- /dev/null +++ b/libs/memory/docs/paging.puml @@ -0,0 +1,40 @@ +@startuml +'https://plantuml.com/object-diagram + +object GiantPage +object GiantPage_2 +object LargePage +object Page +object Page_2 +object Unmapped + +map L1PageUpperDirectory_2 { + entry0 *--> GiantPage_2 + entry1 *--> Unmapped +} + +map L3PageTable { + entry0 *--> Page + entry1 *--> Page_2 +} + +map L2PageDirectory { + entry0 *-> L3PageTable + entry1 *--> LargePage +} + +map L1PageUpperDirectory { + entry0 *-> L2PageDirectory + entry1 *--> GiantPage +} + +map L0PageGlobalDirectory { + entry0 *-> L1PageUpperDirectory + entry1 *--> L1PageUpperDirectory_2 +} + +map VirtSpace { + root *-> L0PageGlobalDirectory +} + +@enduml diff --git a/libs/memory/src/arch/aarch64/README.md b/libs/memory/src/arch/aarch64/README.md index 1e4943b11..b2c1d266c 100644 --- a/libs/memory/src/arch/aarch64/README.md +++ b/libs/memory/src/arch/aarch64/README.md @@ -8,4 +8,143 @@ a specific-size page. ---- +## Plan + +- 1. MMU tables - because we need separate memspaces for kernel and userspace + - 1a. Allocate initial page tables + - 1b. Map over available RAM sensibly + - 1c. Create kernel's own mapping (TTBR_EL1) + +## What does the kernel MMU code support? + +* mapping +* unmapping +* switching per-process mappings (virtspaces) +* virt2phys resolution +* direct phys access for kernel (TTBR_EL1 mapping to physmem) +* initial kernel memory allocation: for mapping tables and capnodes, for initial thread TCB and stacks + +## public api: + + ARM_MMU invocations: + on page directory cap + cache maintenance (clean/invalidate/unify) + on page table cap + map + unmap + on small frame/frame caps + map + remap + unmap + cache maintenance (clean/invalidate/unify) + get address + on asid control cap + on asid pool cap + + +## Minimum Required Functionality (build from this) + +* resolve VA to PA - resolving lets kernel access mapped process memory. + (start from the process' virtspace root - Page Directory) +* flush page, pd, pt, virtspace - will be important for thread switching +* map a page table to appropriate location +* unmap entire mapped page table +* map a phys frame to virt location +* unmap a mapped frame + + +## Requirements + +``` +GIVEN + A PageGlobalDirectory of process +FIND + A kernel-physical address of where it contains a certain leaf node. +``` + +## sel4 + +> seL4 does not provide virtual memory management, beyond kernel primitives for manipulating hardware paging structures. User-level must provide services for creating intermediate paging structures, mapping and unmapping pages. +> Users are free to define their own address space layout with one restriction: the seL4 kernel claims the high part of the virtual memory range. On most 32-bit platforms, this is 0xe0000000 and above. This variable is set per platform, and can be found by finding the kernelBase variable in the seL4 source. +(from https://docs.sel4.systems/Tutorials/mapping.html) + +> Note that to map a frame multiple times, one must make copies of the frame capability: each frame capability can only track one mapping. + +## howto steps + +initial mapping: +* for kernel space - + 1. obtain memory map + 2. build a list of regions available as RAM + 3. find largest covering page sizes + 4. allocate several memory pages and fill them with table info + (need page table creation functions here) + 5. now kernel is able to address physical memory through it's (negative) kernel mapping. + 6. prepare init thread VSpace + - this is more complicated wrt mapping + +Memory areas: +- map entire RAM through TTBR1 +- need device data map to skip MMIO areas + - read and parse device tree data then.. +- map only kickstart and boot_info structures through TTBR0 + +``` +// The region of the initial thread is the user image + ipcbuf and boot info. + +/* setup virtual memory for the kernel */ ++map_kernel_window(); + +/* Construct an initial address space with enough virtual addresses + * to cover the user image + ipc buffer and bootinfo frames */ +it_pd_cap = create_it_address_space(root_cnode_cap, it_v_reg); + +/* Create and map bootinfo frame cap */ +create_bi_frame_cap( +root_cnode_cap, +it_pd_cap, +bi_frame_pptr, +bi_frame_vptr +); + +/* create the initial thread's IPC buffer */ +ipcbuf_cap = create_ipcbuf_frame(root_cnode_cap, it_pd_cap, ipcbuf_vptr); + +/* create all userland image frames */ +create_frames_ret = + create_frames_of_region( + root_cnode_cap, + it_pd_cap, + ui_reg, + true, + pv_offset + ); +ndks_boot.bi_frame->userImageFrames = create_frames_ret.region; + +... later ... + +/* create the initial thread */ +if (!create_initial_thread( + root_cnode_cap, + it_pd_cap, + v_entry, + bi_frame_vptr, + ipcbuf_vptr, + ipcbuf_cap + )) { + + +/* create all of the untypeds. Both devices and kernel window memory */ +if (!create_untypeds( + root_cnode_cap, +(region_t) { +0xf0000000, (pptr_t)ki_boot_end +} /* reusable boot code/data */ + )) { + return false; +} +``` + +---- + For more information please re-read. diff --git a/libs/memory/src/arch/aarch64/addr/asid.rs b/libs/memory/src/arch/aarch64/addr/asid.rs deleted file mode 100644 index 5855abcfd..000000000 --- a/libs/memory/src/arch/aarch64/addr/asid.rs +++ /dev/null @@ -1,7 +0,0 @@ -/* - * SPDX-License-Identifier: BlueOak-1.0.0 - * Copyright (c) Berkus Decker - */ - -#[allow(dead_code)] -pub type Asid = u16; diff --git a/libs/memory/src/arch/aarch64/addr/mod.rs b/libs/memory/src/arch/aarch64/addr/mod.rs deleted file mode 100644 index 9a5d580f2..000000000 --- a/libs/memory/src/arch/aarch64/addr/mod.rs +++ /dev/null @@ -1,7 +0,0 @@ -/* - * SPDX-License-Identifier: BlueOak-1.0.0 - * Copyright (c) Berkus Decker - */ - -pub mod asid; -// pub use asid::*; diff --git a/libs/memory/src/arch/aarch64/common.rs b/libs/memory/src/arch/aarch64/common.rs new file mode 100644 index 000000000..1378daf0f --- /dev/null +++ b/libs/memory/src/arch/aarch64/common.rs @@ -0,0 +1,78 @@ +use libmapping::{AccessPermissions, AttributeFields, MemAttributes}; + +// --------------------------------------------------------------------------- +// AArch64 descriptor bit layout (shared across all granule sizes) +// --------------------------------------------------------------------------- +// +// The descriptor format is identical for 4K, 16K, and 64K granules. +// Only the address masks and index extraction differ per granule. + +// Descriptor bit positions +pub(super) const VALID_BIT: u64 = 1 << 0; +pub(super) const TYPE_BIT: u64 = 1 << 1; + +// Lower attribute bits +pub(super) const ATTR_INDX_SHIFT: u64 = 2; +pub(super) const AP_SHIFT: u64 = 6; +pub(super) const SH_SHIFT: u64 = 8; +pub(super) const AF_BIT: u64 = 1 << 10; + +// Upper attribute bits +pub(super) const PXN_BIT: u64 = 1 << 53; +pub(super) const UXN_BIT: u64 = 1 << 54; + +// Table descriptor address mask (bits [47:12], common to all granules) +pub(super) const TABLE_ADDR_MASK: u64 = 0x0000_FFFF_FFFF_F000; + +// AP field values +pub(super) const AP_RW_EL1: u64 = 0b00 << AP_SHIFT; +pub(super) const AP_RO_EL1: u64 = 0b10 << AP_SHIFT; + +// SH field values +pub(super) const SH_INNER: u64 = 0b11 << SH_SHIFT; +pub(super) const SH_OUTER: u64 = 0b10 << SH_SHIFT; + +/// `MAIR_EL1` attribute indices, matching the MAIR setup in mmu.rs. +pub mod mair { + pub const NORMAL: u64 = 0; + pub const NORMAL_NON_CACHEABLE: u64 = 1; + pub const DEVICE_NGNRE: u64 = 2; +} + +/// Encode `AttributeFields` into the lower+upper attribute bits of a +/// block/page descriptor. Shared by all `AArch64` granule implementations. +pub(super) fn encode_attributes(attr: AttributeFields) -> u64 { + let mut bits: u64 = 0; + + // Memory type -> MAIR index + shareability + match attr.mem_attributes { + MemAttributes::CacheableDRAM => { + bits |= mair::NORMAL << ATTR_INDX_SHIFT; + bits |= SH_INNER; + } + MemAttributes::NonCacheableDRAM => { + bits |= mair::NORMAL_NON_CACHEABLE << ATTR_INDX_SHIFT; + bits |= SH_INNER; + } + MemAttributes::Device => { + bits |= mair::DEVICE_NGNRE << ATTR_INDX_SHIFT; + bits |= SH_OUTER; + } + } + + // Access permissions + bits |= match attr.acc_perms { + AccessPermissions::ReadWrite => AP_RW_EL1, + AccessPermissions::ReadOnly => AP_RO_EL1, + }; + + // Execute-never when not executable + if !attr.executable { + bits |= PXN_BIT; + } + + // Always set UXN until userspace is implemented + bits |= UXN_BIT; + + bits +} diff --git a/libs/memory/src/arch/aarch64/features.rs b/libs/memory/src/arch/aarch64/features.rs new file mode 100644 index 000000000..7d6a94845 --- /dev/null +++ b/libs/memory/src/arch/aarch64/features.rs @@ -0,0 +1,127 @@ +//! Print MMU suported features for debugging. + +use { + aarch64_cpu::registers::{ID_AA64MMFR0_EL1, SCTLR_EL1, TCR_EL1}, + liblog::println, + tock_registers::interfaces::Readable, +}; + +/// Parse the `ID_AA64MMFR0_EL1` register for runtime information about supported MMU features. +/// Print the current state of TCR register. +pub fn print_features() { + let sctlr = SCTLR_EL1.extract(); + + if let Some(SCTLR_EL1::M::Value::Enable) = sctlr.read_as_enum(SCTLR_EL1::M) { + println!("[i] MMU currently enabled"); + } + + if let Some(SCTLR_EL1::I::Value::Cacheable) = sctlr.read_as_enum(SCTLR_EL1::I) { + println!("[i] MMU I-cache enabled"); + } + + if let Some(SCTLR_EL1::C::Value::Cacheable) = sctlr.read_as_enum(SCTLR_EL1::C) { + println!("[i] MMU D-cache enabled"); + } + + let mmfr = ID_AA64MMFR0_EL1.extract(); + + if let Some(ID_AA64MMFR0_EL1::TGran4::Value::Supported) = + mmfr.read_as_enum(ID_AA64MMFR0_EL1::TGran4) + { + println!("[i] MMU: 4 KiB granule supported!"); + } + + if let Some(ID_AA64MMFR0_EL1::TGran16::Value::Supported) = + mmfr.read_as_enum(ID_AA64MMFR0_EL1::TGran16) + { + println!("[i] MMU: 16 KiB granule supported!"); + } + + if let Some(ID_AA64MMFR0_EL1::TGran64::Value::Supported) = + mmfr.read_as_enum(ID_AA64MMFR0_EL1::TGran64) + { + println!("[i] MMU: 64 KiB granule supported!"); + } + + match mmfr.read_as_enum(ID_AA64MMFR0_EL1::ASIDBits) { + Some(ID_AA64MMFR0_EL1::ASIDBits::Value::Bits_16) => { + println!("[i] MMU: 16 bit ASIDs supported!"); + } + Some(ID_AA64MMFR0_EL1::ASIDBits::Value::Bits_8) => { + println!("[i] MMU: 8 bit ASIDs supported!"); + } + _ => println!("[i] MMU: Invalid ASID bits specified!"), + } + + match mmfr.read_as_enum(ID_AA64MMFR0_EL1::PARange) { + Some(ID_AA64MMFR0_EL1::PARange::Value::Bits_32) => { + println!("[i] MMU: Up to 32 Bit physical address range supported!"); + } + Some(ID_AA64MMFR0_EL1::PARange::Value::Bits_36) => { + println!("[i] MMU: Up to 36 Bit physical address range supported!"); + } + Some(ID_AA64MMFR0_EL1::PARange::Value::Bits_40) => { + println!("[i] MMU: Up to 40 Bit physical address range supported!"); + } + Some(ID_AA64MMFR0_EL1::PARange::Value::Bits_42) => { + println!("[i] MMU: Up to 42 Bit physical address range supported!"); + } + Some(ID_AA64MMFR0_EL1::PARange::Value::Bits_44) => { + println!("[i] MMU: Up to 44 Bit physical address range supported!"); + } + Some(ID_AA64MMFR0_EL1::PARange::Value::Bits_48) => { + println!("[i] MMU: Up to 48 Bit physical address range supported!"); + } + Some(ID_AA64MMFR0_EL1::PARange::Value::Bits_52) => { + println!("[i] MMU: Up to 52 Bit physical address range supported!"); + } + _ => println!("[i] MMU: Invalid PARange specified!"), + } + + let tcr = TCR_EL1.extract(); + + match tcr.read_as_enum(TCR_EL1::IPS) { + Some(TCR_EL1::IPS::Value::Bits_32) => { + println!("[i] MMU: 32 Bit intermediate physical address size supported!"); + } + Some(TCR_EL1::IPS::Value::Bits_36) => { + println!("[i] MMU: 36 Bit intermediate physical address size supported!"); + } + Some(TCR_EL1::IPS::Value::Bits_40) => { + println!("[i] MMU: 40 Bit intermediate physical address size supported!"); + } + Some(TCR_EL1::IPS::Value::Bits_42) => { + println!("[i] MMU: 42 Bit intermediate physical address size supported!"); + } + Some(TCR_EL1::IPS::Value::Bits_44) => { + println!("[i] MMU: 44 Bit intermediate physical address size supported!"); + } + Some(TCR_EL1::IPS::Value::Bits_48) => { + println!("[i] MMU: 48 Bit intermediate physical address size supported!"); + } + Some(TCR_EL1::IPS::Value::Bits_52) => { + println!("[i] MMU: 52 Bit intermediate physical address size supported!"); + } + _ => println!("[i] MMU: Invalid IPS specified!"), + } + + match tcr.read_as_enum(TCR_EL1::TG0) { + Some(TCR_EL1::TG0::Value::KiB_4) => println!("[i] MMU: TTBR0 4 KiB granule active!"), + Some(TCR_EL1::TG0::Value::KiB_16) => println!("[i] MMU: TTBR0 16 KiB granule active!"), + Some(TCR_EL1::TG0::Value::KiB_64) => println!("[i] MMU: TTBR0 64 KiB granule active!"), + _ => println!("[i] MMU: Invalid TTBR0 granule size specified!"), + } + + let t0sz = tcr.read(TCR_EL1::T0SZ); + println!("[i] MMU: T0sz = 64-{} = {} bits", t0sz, 64 - t0sz); + + match tcr.read_as_enum(TCR_EL1::TG1) { + Some(TCR_EL1::TG1::Value::KiB_4) => println!("[i] MMU: TTBR1 4 KiB granule active!"), + Some(TCR_EL1::TG1::Value::KiB_16) => println!("[i] MMU: TTBR1 16 KiB granule active!"), + Some(TCR_EL1::TG1::Value::KiB_64) => println!("[i] MMU: TTBR1 64 KiB granule active!"), + _ => println!("[i] MMU: Invalid TTBR1 granule size specified!"), + } + + let t1sz = tcr.read(TCR_EL1::T1SZ); + println!("[i] MMU: T1sz = 64-{} = {} bits", t1sz, 64 - t1sz); +} diff --git a/libs/memory/src/arch/aarch64/granule_16k.rs b/libs/memory/src/arch/aarch64/granule_16k.rs new file mode 100644 index 000000000..f9cc12cd4 --- /dev/null +++ b/libs/memory/src/arch/aarch64/granule_16k.rs @@ -0,0 +1,170 @@ +use { + super::common::*, + crate::arch_trait::{EntryKind, LevelCapabilities, TranslationArch}, + libaddress::{PhysAddr, VirtAddr}, + libmapping::AttributeFields, +}; + +// --------------------------------------------------------------------------- +// AArch64 Stage 1, 16KiB granule +// --------------------------------------------------------------------------- +// +// Virtual address split (48-bit VA): +// +// 63-48 47 46-36 35-25 24-14 13-0 +// signx L0 L1 L2 L3 off +// 1bit 11bits 11bits 11bits 14bits +// +// L0: 2 entries, table pointer only. Table size = 16 bytes. +// L1: 2048 entries, table pointer only. +// L2: 2048 entries, table pointer or 32MiB block. +// L3: 2048 entries, 16KiB page. +// +// All tables are 16KiB-aligned (including L0, even though it is only 16 bytes). + +// Address masks (16K granule specific) +const L2_BLOCK_ADDR_MASK: u64 = 0x0000_FFFF_FE00_0000; // [47:25] +const L3_PAGE_ADDR_MASK: u64 = 0x0000_FFFF_FFFF_C000; // [47:14] + +// Block sizes +const SIZE_16K: usize = 16 * 1024; +const SIZE_32M: usize = 32 * 1024 * 1024; + +/// `AArch64` Stage 1 translation with 16KiB granule. +/// +/// 4-level hierarchy: L0 (2 entries) -> L1 (2048) -> L2 (2048) -> L3 (2048). +/// L0 table is only 16 bytes but requires 16KiB alignment. +pub struct Aarch64_16K; + +impl TranslationArch for Aarch64_16K { + const NUM_LEVELS: usize = 4; + + fn entries_per_table(level: usize) -> usize { + match level { + 0 => 2, + 1..=3 => 2048, + _ => 0, + } + } + + fn table_alignment(_level: usize) -> usize { + SIZE_16K + } + + fn level_capabilities(level: usize) -> LevelCapabilities { + match level { + 0 | 1 => LevelCapabilities { + supports_table_pointer: true, + supports_block: false, + block_size: 0, + }, + 2 => LevelCapabilities { + supports_table_pointer: true, + supports_block: true, + block_size: SIZE_32M, + }, + 3 => LevelCapabilities { + supports_table_pointer: false, + supports_block: true, + block_size: SIZE_16K, + }, + _ => LevelCapabilities { + supports_table_pointer: false, + supports_block: false, + block_size: 0, + }, + } + } + + fn index_from_vaddr(vaddr: VirtAddr, level: usize) -> usize { + let (shift, mask) = match level { + 0 => (47, 0x1), + 1 => (36, 0x7FF), + 2 => (25, 0x7FF), + 3 => (14, 0x7FF), + _ => return 0, + }; + usize::try_from((vaddr.as_u64() >> shift) & mask).unwrap() + } + + fn decode_entry(raw: u64, level: usize) -> EntryKind { + if raw & VALID_BIT == 0 { + return EntryKind::Invalid; + } + + match level { + // L0, L1: TYPE=1 means table, TYPE=0 is reserved/invalid + 0 | 1 => { + if raw & TYPE_BIT != 0 { + EntryKind::Table(PhysAddr::new(raw & TABLE_ADDR_MASK)) + } else { + EntryKind::Invalid + } + } + // L2: TYPE=1 means table, TYPE=0 means 32MiB block + 2 => { + if raw & TYPE_BIT != 0 { + EntryKind::Table(PhysAddr::new(raw & TABLE_ADDR_MASK)) + } else { + EntryKind::Block(PhysAddr::new(raw & L2_BLOCK_ADDR_MASK)) + } + } + // L3: TYPE=1 means valid page, TYPE=0 is reserved/invalid + 3 => { + if raw & TYPE_BIT != 0 { + EntryKind::Block(PhysAddr::new(raw & L3_PAGE_ADDR_MASK)) + } else { + EntryKind::Invalid + } + } + _ => EntryKind::Invalid, + } + } + + fn encode_table_entry(next_table_phys: PhysAddr, _level: usize) -> u64 { + let addr = next_table_phys.as_u64(); + debug_assert!( + addr & !TABLE_ADDR_MASK == 0, + "Table address not properly aligned" + ); + (addr & TABLE_ADDR_MASK) | TYPE_BIT | VALID_BIT + } + + fn encode_block_entry(phys: PhysAddr, attr: AttributeFields, level: usize) -> u64 { + let addr = phys.as_u64(); + let addr_mask = match level { + 2 => L2_BLOCK_ADDR_MASK, + _ => panic!("Block entries only valid at L2 for 16K granule"), + }; + debug_assert!( + addr & !addr_mask == 0, + "Block address not aligned for level" + ); + (addr & addr_mask) | encode_attributes(attr) | AF_BIT | VALID_BIT + } + + fn encode_page_entry(phys: PhysAddr, attr: AttributeFields) -> u64 { + let addr = phys.as_u64(); + debug_assert!( + addr & !L3_PAGE_ADDR_MASK == 0, + "Page address not 16K aligned" + ); + (addr & L3_PAGE_ADDR_MASK) | encode_attributes(attr) | AF_BIT | TYPE_BIT | VALID_BIT + } + + fn output_address(raw: u64, level: usize) -> PhysAddr { + let mask = match level { + 0 | 1 => TABLE_ADDR_MASK, + 2 => { + if raw & TYPE_BIT != 0 { + TABLE_ADDR_MASK + } else { + L2_BLOCK_ADDR_MASK + } + } + 3 => L3_PAGE_ADDR_MASK, + _ => 0, + }; + PhysAddr::new(raw & mask) + } +} diff --git a/libs/memory/src/arch/aarch64/granule_4k.rs b/libs/memory/src/arch/aarch64/granule_4k.rs new file mode 100644 index 000000000..d0b8994be --- /dev/null +++ b/libs/memory/src/arch/aarch64/granule_4k.rs @@ -0,0 +1,181 @@ +use { + super::common::*, + crate::arch_trait::{EntryKind, LevelCapabilities, TranslationArch}, + libaddress::{PhysAddr, VirtAddr}, + libmapping::AttributeFields, +}; + +// --------------------------------------------------------------------------- +// AArch64 Stage 1, 4KiB granule +// --------------------------------------------------------------------------- +// +// Virtual address split (48-bit VA): +// +// 63-48 47-39 38-30 29-21 20-12 11-0 +// signx L0 L1 L2 L3 off +// +// Each level indexes 512 entries (9 bits). Table size = 512 * 8 = 4096 bytes. +// +// L0: table pointer only +// L1: table pointer or 1GiB block +// L2: table pointer or 2MiB block +// L3: 4KiB page only (TYPE bit 1 = 1 for valid page) + +// Address masks (4K granule specific) +const L1_BLOCK_ADDR_MASK: u64 = 0x0000_FFFF_C000_0000; // [47:30] +const L2_BLOCK_ADDR_MASK: u64 = 0x0000_FFFF_FFE0_0000; // [47:21] +const L3_PAGE_ADDR_MASK: u64 = 0x0000_FFFF_FFFF_F000; // [47:12] + +// Block sizes +const SIZE_4K: usize = 4096; +const SIZE_2M: usize = 2 * 1024 * 1024; +const SIZE_1G: usize = 1024 * 1024 * 1024; + +/// `AArch64` Stage 1 translation with 4KiB granule. +/// +/// 4-level hierarchy: L0 -> L1 -> L2 -> L3. +/// 512 entries per table, 4096-byte table size, 4096-byte alignment. +pub struct Aarch64_4K; + +impl TranslationArch for Aarch64_4K { + const NUM_LEVELS: usize = 4; + + fn entries_per_table(_level: usize) -> usize { + 512 + } + + fn table_alignment(_level: usize) -> usize { + SIZE_4K + } + + fn level_capabilities(level: usize) -> LevelCapabilities { + match level { + 0 => LevelCapabilities { + supports_table_pointer: true, + supports_block: false, + block_size: 0, + }, + 1 => LevelCapabilities { + supports_table_pointer: true, + supports_block: true, + block_size: SIZE_1G, + }, + 2 => LevelCapabilities { + supports_table_pointer: true, + supports_block: true, + block_size: SIZE_2M, + }, + 3 => LevelCapabilities { + supports_table_pointer: false, + supports_block: true, + block_size: SIZE_4K, + }, + _ => LevelCapabilities { + supports_table_pointer: false, + supports_block: false, + block_size: 0, + }, + } + } + + fn index_from_vaddr(vaddr: VirtAddr, level: usize) -> usize { + let shift = match level { + 0 => 39, + 1 => 30, + 2 => 21, + 3 => 12, + _ => return 0, + }; + ((vaddr.as_u64() >> shift) & 0x1FF) as usize + } + + fn decode_entry(raw: u64, level: usize) -> EntryKind { + if raw & VALID_BIT == 0 { + return EntryKind::Invalid; + } + + match level { + 0 => { + if raw & TYPE_BIT != 0 { + EntryKind::Table(PhysAddr::new(raw & TABLE_ADDR_MASK)) + } else { + EntryKind::Invalid + } + } + 1 => { + if raw & TYPE_BIT != 0 { + EntryKind::Table(PhysAddr::new(raw & TABLE_ADDR_MASK)) + } else { + EntryKind::Block(PhysAddr::new(raw & L1_BLOCK_ADDR_MASK)) + } + } + 2 => { + if raw & TYPE_BIT != 0 { + EntryKind::Table(PhysAddr::new(raw & TABLE_ADDR_MASK)) + } else { + EntryKind::Block(PhysAddr::new(raw & L2_BLOCK_ADDR_MASK)) + } + } + 3 => { + if raw & TYPE_BIT != 0 { + EntryKind::Block(PhysAddr::new(raw & L3_PAGE_ADDR_MASK)) + } else { + EntryKind::Invalid + } + } + _ => EntryKind::Invalid, + } + } + + fn encode_table_entry(next_table_phys: PhysAddr, _level: usize) -> u64 { + let addr = next_table_phys.as_u64(); + debug_assert!(addr & !TABLE_ADDR_MASK == 0, "Table address not 4K aligned"); + (addr & TABLE_ADDR_MASK) | TYPE_BIT | VALID_BIT + } + + fn encode_block_entry(phys: PhysAddr, attr: AttributeFields, level: usize) -> u64 { + let addr = phys.as_u64(); + let addr_mask = match level { + 1 => L1_BLOCK_ADDR_MASK, + 2 => L2_BLOCK_ADDR_MASK, + _ => panic!("Block entries only valid at L1 and L2"), + }; + debug_assert!( + addr & !addr_mask == 0, + "Block address not aligned for level" + ); + (addr & addr_mask) | encode_attributes(attr) | AF_BIT | VALID_BIT + } + + fn encode_page_entry(phys: PhysAddr, attr: AttributeFields) -> u64 { + let addr = phys.as_u64(); + debug_assert!( + addr & !L3_PAGE_ADDR_MASK == 0, + "Page address not 4K aligned" + ); + (addr & L3_PAGE_ADDR_MASK) | encode_attributes(attr) | AF_BIT | TYPE_BIT | VALID_BIT + } + + fn output_address(raw: u64, level: usize) -> PhysAddr { + let mask = match level { + 0 => TABLE_ADDR_MASK, + 1 => { + if raw & TYPE_BIT != 0 { + TABLE_ADDR_MASK + } else { + L1_BLOCK_ADDR_MASK + } + } + 2 => { + if raw & TYPE_BIT != 0 { + TABLE_ADDR_MASK + } else { + L2_BLOCK_ADDR_MASK + } + } + 3 => L3_PAGE_ADDR_MASK, + _ => 0, + }; + PhysAddr::new(raw & mask) + } +} diff --git a/libs/memory/src/arch/aarch64/granule_64k.rs b/libs/memory/src/arch/aarch64/granule_64k.rs new file mode 100644 index 000000000..d1af2fa23 --- /dev/null +++ b/libs/memory/src/arch/aarch64/granule_64k.rs @@ -0,0 +1,172 @@ +use { + super::common::*, + crate::arch_trait::{EntryKind, LevelCapabilities, TranslationArch}, + libaddress::{PhysAddr, VirtAddr}, + libmapping::AttributeFields, +}; + +// --------------------------------------------------------------------------- +// AArch64 Stage 1, 64KiB granule +// --------------------------------------------------------------------------- +// +// Virtual address split (48-bit VA, no LPA2): +// +// 63-48 47-42 41-29 28-16 15-0 +// signx L0 L1 L2 off +// 6bits 13bits 13bits 16bits +// +// Our level 0 corresponds to ARM's "Level 1" — we always number from +// the root. ARM skips Level 0 entirely for 64K granule. +// +// L0: 64 entries, table pointer only. Table size = 512 bytes. +// L1: 8192 entries, table pointer or 512MiB block. +// L2: 8192 entries, 64KiB page. +// +// All tables require 64KiB alignment (including L0, even though it +// is only 512 bytes). + +// Address masks (64K granule specific) +const L1_BLOCK_ADDR_MASK: u64 = 0x0000_FFFE_0000_0000; // [47:29] +const L2_PAGE_ADDR_MASK: u64 = 0x0000_FFFF_FFFF_0000; // [47:16] + +// Block sizes +const SIZE_64K: usize = 64 * 1024; +const SIZE_512M: usize = 512 * 1024 * 1024; + +/// `AArch64` Stage 1 translation with 64KiB granule. +/// +/// 3-level hierarchy: L0 (64 entries) -> L1 (8192) -> L2 (8192). +/// Our level numbering is 0-based from root; ARM calls these Level 1/2/3. +pub struct Aarch64_64K; + +impl TranslationArch for Aarch64_64K { + const NUM_LEVELS: usize = 3; + + fn entries_per_table(level: usize) -> usize { + match level { + 0 => 64, + 1 | 2 => 8192, + _ => 0, + } + } + + fn table_alignment(_level: usize) -> usize { + SIZE_64K + } + + fn level_capabilities(level: usize) -> LevelCapabilities { + match level { + 0 => LevelCapabilities { + supports_table_pointer: true, + supports_block: false, + block_size: 0, + }, + 1 => LevelCapabilities { + supports_table_pointer: true, + supports_block: true, + block_size: SIZE_512M, + }, + 2 => LevelCapabilities { + supports_table_pointer: false, + supports_block: true, + block_size: SIZE_64K, + }, + _ => LevelCapabilities { + supports_table_pointer: false, + supports_block: false, + block_size: 0, + }, + } + } + + fn index_from_vaddr(vaddr: VirtAddr, level: usize) -> usize { + let (shift, mask) = match level { + 0 => (42, 0x3F), + 1 => (29, 0x1FFF), + 2 => (16, 0x1FFF), + _ => return 0, + }; + usize::try_from((vaddr.as_u64() >> shift) & mask).unwrap() + } + + fn decode_entry(raw: u64, level: usize) -> EntryKind { + if raw & VALID_BIT == 0 { + return EntryKind::Invalid; + } + + match level { + // L0: TYPE=1 means table, TYPE=0 is reserved/invalid + 0 => { + if raw & TYPE_BIT != 0 { + EntryKind::Table(PhysAddr::new(raw & TABLE_ADDR_MASK)) + } else { + EntryKind::Invalid + } + } + // L1: TYPE=1 means table, TYPE=0 means 512MiB block + 1 => { + if raw & TYPE_BIT != 0 { + EntryKind::Table(PhysAddr::new(raw & TABLE_ADDR_MASK)) + } else { + EntryKind::Block(PhysAddr::new(raw & L1_BLOCK_ADDR_MASK)) + } + } + // L2: TYPE=1 means valid page, TYPE=0 is reserved/invalid + 2 => { + if raw & TYPE_BIT != 0 { + EntryKind::Block(PhysAddr::new(raw & L2_PAGE_ADDR_MASK)) + } else { + EntryKind::Invalid + } + } + _ => EntryKind::Invalid, + } + } + + fn encode_table_entry(next_table_phys: PhysAddr, _level: usize) -> u64 { + let addr = next_table_phys.as_u64(); + debug_assert!( + addr & !TABLE_ADDR_MASK == 0, + "Table address not properly aligned" + ); + (addr & TABLE_ADDR_MASK) | TYPE_BIT | VALID_BIT + } + + fn encode_block_entry(phys: PhysAddr, attr: AttributeFields, level: usize) -> u64 { + let addr = phys.as_u64(); + let addr_mask = match level { + 1 => L1_BLOCK_ADDR_MASK, + _ => panic!("Block entries only valid at L1 for 64K granule"), + }; + debug_assert!( + addr & !addr_mask == 0, + "Block address not aligned for level" + ); + (addr & addr_mask) | encode_attributes(attr) | AF_BIT | VALID_BIT + } + + fn encode_page_entry(phys: PhysAddr, attr: AttributeFields) -> u64 { + let addr = phys.as_u64(); + debug_assert!( + addr & !L2_PAGE_ADDR_MASK == 0, + "Page address not 64K aligned" + ); + (addr & L2_PAGE_ADDR_MASK) | encode_attributes(attr) | AF_BIT | TYPE_BIT | VALID_BIT + } + + fn output_address(raw: u64, level: usize) -> PhysAddr { + let mask = match level { + 0 => TABLE_ADDR_MASK, + 1 => { + if raw & TYPE_BIT != 0 { + TABLE_ADDR_MASK + } else { + L1_BLOCK_ADDR_MASK + } + } + 2 => L2_PAGE_ADDR_MASK, + _ => 0, + }; + PhysAddr::new(raw & mask) + } +} diff --git a/libs/memory/src/arch/aarch64/mmu.rs b/libs/memory/src/arch/aarch64/mmu.rs new file mode 100644 index 000000000..e02f912dd --- /dev/null +++ b/libs/memory/src/arch/aarch64/mmu.rs @@ -0,0 +1,188 @@ +/* + * SPDX-License-Identifier: BlueOak-1.0.0 + * Copyright (c) Berkus Decker + */ + +//! `AArch64` MMU hardware configuration and enablement. +//! +//! This module handles the actual hardware register setup (MAIR, TCR, SCTLR) +//! and MMU enable sequence. It does NOT manage translation table contents — +//! that is handled by the `Table` type and `Aarch64_4K` arch implementation. + +use { + aarch64_cpu::{ + asm::{self, barrier}, + registers::{ID_AA64MMFR0_EL1, SCTLR_EL1, TCR_EL1}, + }, + core::intrinsics::unlikely, + libaddress::{Address, Physical}, + liblog::println, + tock_registers::interfaces::{ReadWriteable, Readable, Writeable}, +}; + +/// MMU enable errors. +#[derive(Debug)] +pub enum MMUEnableError { + /// MMU is already enabled. + AlreadyEnabled, + /// Other error. + Other { err: &'static str }, +} + +impl core::fmt::Display for MMUEnableError { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + match self { + Self::AlreadyEnabled => write!(f, "MMU is already enabled"), + Self::Other { err } => write!(f, "{err}"), + } + } +} + +/// Memory Management Unit type. +pub struct MemoryManagementUnit; + +/// `MAIR_EL1` attribute indices. +/// +/// These must match the MAIR setup in `set_up_mair()` and the indices +/// used by `arch::aarch64::translation::encode_attributes()`. +pub mod mair { + pub mod attr { + pub const NORMAL: u64 = 0; + pub const NORMAL_NON_CACHEABLE: u64 = 1; + pub const DEVICE_NGNRE: u64 = 2; + } +} + +static MMU: MemoryManagementUnit = MemoryManagementUnit; + +impl MemoryManagementUnit { + /// Setup function for the `MAIR_EL1` register. + #[expect(clippy::unused_self)] + fn set_up_mair(&self) { + use aarch64_cpu::registers::MAIR_EL1; + MAIR_EL1.write( + // Attribute 2 -- Device Memory + MAIR_EL1::Attr2_Device::nonGathering_nonReordering_EarlyWriteAck + // Attribute 1 -- Non Cacheable DRAM + + MAIR_EL1::Attr1_Normal_Outer::NonCacheable + + MAIR_EL1::Attr1_Normal_Inner::NonCacheable + // Attribute 0 -- Regular Cacheable + + MAIR_EL1::Attr0_Normal_Outer::WriteBack_NonTransient_ReadWriteAlloc + + MAIR_EL1::Attr0_Normal_Inner::WriteBack_NonTransient_ReadWriteAlloc, + ); + } + + /// Configure various settings of stage 1 of the EL1 translation regime. + #[expect(clippy::unused_self)] + fn configure_translation_control(&self) { + let ips = ID_AA64MMFR0_EL1.read(ID_AA64MMFR0_EL1::PARange); + + // Maximum 8Gb user VA + let user_va_bits = 33; // ARMv8ARM Table D5-11 minimum TxSZ for starting table level 1 + + // Maximum 8Gb kernel VA + let kernel_va_bits = 33; + + TCR_EL1.write( + TCR_EL1::TBI0::Ignored + + TCR_EL1::IPS.val(ips) + // ttbr0 user memory addresses + + TCR_EL1::TG0::KiB_4 + + TCR_EL1::SH0::Inner + + TCR_EL1::ORGN0::WriteBack_ReadAlloc_WriteAlloc_Cacheable + + TCR_EL1::IRGN0::WriteBack_ReadAlloc_WriteAlloc_Cacheable + + TCR_EL1::EPD0::EnableTTBR0Walks + + TCR_EL1::T0SZ.val(64 - user_va_bits) + // ttbr1 kernel memory addresses + + TCR_EL1::TBI1::Ignored + + TCR_EL1::TG1::KiB_4 + + TCR_EL1::SH1::Inner + + TCR_EL1::ORGN1::WriteBack_ReadAlloc_WriteAlloc_Cacheable + + TCR_EL1::IRGN1::WriteBack_ReadAlloc_WriteAlloc_Cacheable + + TCR_EL1::EPD1::DisableTTBR1Walks // disabled for now + + TCR_EL1::T1SZ.val(64 - kernel_va_bits), + ); + } + + #[expect(clippy::unused_self)] + fn is_enabled(&self) -> bool { + SCTLR_EL1.matches_all(SCTLR_EL1::M::Enable) + } +} + +/// Return a reference to the MMU instance. +pub fn mmu() -> &'static MemoryManagementUnit { + &MMU +} + +impl MemoryManagementUnit { + /// Turns on the MMU and enables data and instruction caching. + /// + /// `phys_tables_base_addr` is the physical address of the root translation + /// table (L0 or L1 depending on T0SZ configuration). + /// + /// # Safety + /// + /// - Changes the hardware's global state. + /// - The caller must ensure translation tables are properly populated. + pub unsafe fn enable_mmu_and_caching( + &self, + _phys_tables_base_addr: Address, + ) -> Result<(), MMUEnableError> { + if unlikely(self.is_enabled()) { + return Err(MMUEnableError::AlreadyEnabled); + } + + // Fail early if translation granule is not supported. + if unlikely(!ID_AA64MMFR0_EL1.matches_all(ID_AA64MMFR0_EL1::TGran4::Supported)) { + return Err(MMUEnableError::Other { + err: "4KiB translation granule not supported by hardware", + }); + } + + // Prepare the memory attribute indirection register. + self.set_up_mair(); + + // TODO: Set TTBR0_EL1 and TTBR1_EL1 to point to the root table. + // TTBR0_EL1.set_baddr(phys_tables_base_addr.as_u64()); + // TTBR0_EL1.modify(TTBR0_EL1::CnP.val(1)); + + self.configure_translation_control(); + + // Switch the MMU on. + // First, force all previous changes to be seen before the MMU is enabled. + barrier::dsb(barrier::ISHST); + barrier::dsb(barrier::ISH); + barrier::isb(barrier::SY); + + // Enable the MMU and turn on data and instruction caching. + SCTLR_EL1.modify( + SCTLR_EL1::EE::LittleEndian + + SCTLR_EL1::E0E::LittleEndian + + SCTLR_EL1::WXN::Disable + + SCTLR_EL1::SA::Disable + + SCTLR_EL1::SA0::Disable + + SCTLR_EL1::A::Disable + + SCTLR_EL1::UCI::Trap + + SCTLR_EL1::UCT::Trap + + SCTLR_EL1::UMA::Trap + + SCTLR_EL1::NTWE::Trap + + SCTLR_EL1::NTWI::Trap + + SCTLR_EL1::DZE::Trap + + SCTLR_EL1::C::Cacheable + + SCTLR_EL1::I::Cacheable + + SCTLR_EL1::M::Enable, + ); + + // Let 2 CPU cycles pass then invalidate TLB. + asm::nop(); + asm::nop(); + + barrier::dsb(barrier::ISH); + barrier::isb(barrier::SY); + + println!("MMU activated"); + + Ok(()) + } +} diff --git a/libs/memory/src/arch/aarch64/mmu/mod.rs b/libs/memory/src/arch/aarch64/mmu/mod.rs deleted file mode 100644 index de7976ec2..000000000 --- a/libs/memory/src/arch/aarch64/mmu/mod.rs +++ /dev/null @@ -1,295 +0,0 @@ -/* - * SPDX-License-Identifier: MIT OR BlueOak-1.0.0 - * Copyright (c) 2018-2019 Andre Richter - * Copyright (c) Berkus Decker - * Original code distributed under MIT, additional changes are under BlueOak-1.0.0 - */ - -//! MMU initialisation. -//! -//! Paging is mostly based on [previous version](https://os.phil-opp.com/page-tables/) of -//! Phil Opp's [paging guide](https://os.phil-opp.com/paging-implementation/) and -//! [ARMv8 ARM memory addressing](https://static.docs.arm.com/100940/0100/armv8_a_address%20translation_100940_0100_en.pdf). - -use { - crate::{ - Address, Physical, - mmu::{AddressSpace, MMUEnableError, TranslationGranule, interface}, - platform, - }, - aarch64_cpu::{ - asm::barrier, - registers::{ID_AA64MMFR0_EL1, SCTLR_EL1, TCR_EL1, TTBR0_EL1}, - }, - core::intrinsics::unlikely, - liblog::println, - tock_registers::interfaces::{ReadWriteable, Readable, Writeable}, -}; - -pub mod translation_table; - -//-------------------------------------------------------------------------------------------------- -// Private Definitions -//-------------------------------------------------------------------------------------------------- - -/// Memory Management Unit type. -struct MemoryManagementUnit; - -//-------------------------------------------------------------------------------------------------- -// Public Definitions -//-------------------------------------------------------------------------------------------------- - -pub type Granule512MiB = TranslationGranule<{ 512 * 1024 * 1024 }>; -pub type Granule64KiB = TranslationGranule<{ 64 * 1024 }>; - -/// Constants for indexing the `MAIR_EL1`. -#[allow(dead_code)] -pub mod mair { - // Three descriptive consts for indexing into the correct MAIR_EL1 attributes. - pub mod attr { - pub const NORMAL: u64 = 0; - pub const NORMAL_NON_CACHEABLE: u64 = 1; - pub const DEVICE_NGNRE: u64 = 2; - } -} - -//-------------------------------------------------------------------------------------------------- -// Global instances -//-------------------------------------------------------------------------------------------------- - -static MMU: MemoryManagementUnit = MemoryManagementUnit; - -//-------------------------------------------------------------------------------------------------- -// Private Implementations -//-------------------------------------------------------------------------------------------------- - -impl AddressSpace { - /// Checks for architectural restrictions. - pub const fn arch_address_space_size_sanity_checks() { - // Size must be at least one full 512 MiB table. - assert!(AS_SIZE.is_multiple_of(Granule512MiB::SIZE)); // assert!() is const-friendly - - // Check for 48 bit virtual address size as maximum, which is supported by any ARMv8 - // version. - assert!(AS_SIZE <= (1 << 48)); - } -} - -impl MemoryManagementUnit { - /// Setup function for the `MAIR_EL1` register. - fn set_up_mair() { - use aarch64_cpu::registers::MAIR_EL1; - // Define the three memory types that we will map: Normal DRAM, Uncached and device. - MAIR_EL1.write( - // Attribute 2 -- Device Memory - MAIR_EL1::Attr2_Device::nonGathering_nonReordering_EarlyWriteAck - // Attribute 1 -- Non Cacheable DRAM - + MAIR_EL1::Attr1_Normal_Outer::NonCacheable - + MAIR_EL1::Attr1_Normal_Inner::NonCacheable - // Attribute 0 -- Regular Cacheable - + MAIR_EL1::Attr0_Normal_Outer::WriteBack_NonTransient_ReadWriteAlloc - + MAIR_EL1::Attr0_Normal_Inner::WriteBack_NonTransient_ReadWriteAlloc, - ); - } - - /// Configure various settings of stage 1 of the EL1 translation regime. - fn configure_translation_control() { - let t0sz = (64 - platform::memory::mmu::KernelVirtAddrSpace::SIZE_SHIFT) as u64; - - TCR_EL1.write( - TCR_EL1::TBI0::Used - + TCR_EL1::IPS::Bits_40 - + TCR_EL1::TG0::KiB_64 - + TCR_EL1::SH0::Inner - + TCR_EL1::ORGN0::WriteBack_ReadAlloc_WriteAlloc_Cacheable - + TCR_EL1::IRGN0::WriteBack_ReadAlloc_WriteAlloc_Cacheable - + TCR_EL1::EPD0::EnableTTBR0Walks - + TCR_EL1::A1::TTBR0 // TTBR0 defines the ASID - + TCR_EL1::T0SZ.val(t0sz) - + TCR_EL1::EPD1::DisableTTBR1Walks, - ); - } -} - -//-------------------------------------------------------------------------------------------------- -// Public Implementations -//-------------------------------------------------------------------------------------------------- - -/// Return a reference to the MMU instance. -pub fn mmu() -> &'static impl interface::MMU { - &MMU -} - -//------------------------------------------------------------------------------ -// OS Interface Code -//------------------------------------------------------------------------------ - -impl interface::MMU for MemoryManagementUnit { - unsafe fn enable_mmu_and_caching( - &self, - phys_tables_base_addr: Address, - ) -> Result<(), MMUEnableError> { - if unlikely(self.is_enabled()) { - return Err(MMUEnableError::AlreadyEnabled); - } - - // Fail early if translation granule is not supported. - if unlikely(!ID_AA64MMFR0_EL1.matches_all(ID_AA64MMFR0_EL1::TGran64::Supported)) { - return Err(MMUEnableError::Other { - err: "Translation granule not supported by hardware", - }); - } - - // Prepare the memory attribute indirection register. - Self::set_up_mair(); - - // // Populate translation tables. - // KERNEL_TABLES - // .populate_translation_table_entries() - // .map_err(|err| MMUEnableError::Other { err })?; - - // Set the "Translation Table Base Register". - TTBR0_EL1.set_baddr(phys_tables_base_addr.as_usize() as u64); - - Self::configure_translation_control(); - - // Switch the MMU on. - // - // First, force all previous changes to be seen before the MMU is enabled. - barrier::isb(barrier::SY); - - // Enable the MMU and turn on data and instruction caching. - SCTLR_EL1.modify(SCTLR_EL1::M::Enable + SCTLR_EL1::C::Cacheable + SCTLR_EL1::I::Cacheable); - - // Force MMU init to complete before next instruction. - barrier::isb(barrier::SY); - - Ok(()) - } - - #[inline(always)] - fn is_enabled(&self) -> bool { - SCTLR_EL1.matches_all(SCTLR_EL1::M::Enable) - } - - /// Parse the `ID_AA64MMFR0_EL1` register for runtime information about supported MMU features. - /// Print the current state of TCR register. - fn print_features(&self) { - // use crate::cortex_a::regs::RegisterReadWrite; - let sctlr = SCTLR_EL1.extract(); - - if let Some(SCTLR_EL1::M::Value::Enable) = sctlr.read_as_enum(SCTLR_EL1::M) { - println!("[i] MMU currently enabled"); - } - - if let Some(SCTLR_EL1::I::Value::Cacheable) = sctlr.read_as_enum(SCTLR_EL1::I) { - println!("[i] MMU I-cache enabled"); - } - - if let Some(SCTLR_EL1::C::Value::Cacheable) = sctlr.read_as_enum(SCTLR_EL1::C) { - println!("[i] MMU D-cache enabled"); - } - - let mmfr = ID_AA64MMFR0_EL1.extract(); - - if let Some(ID_AA64MMFR0_EL1::TGran4::Value::Supported) = - mmfr.read_as_enum(ID_AA64MMFR0_EL1::TGran4) - { - println!("[i] MMU: 4 KiB granule supported!"); - } - - if let Some(ID_AA64MMFR0_EL1::TGran16::Value::Supported) = - mmfr.read_as_enum(ID_AA64MMFR0_EL1::TGran16) - { - println!("[i] MMU: 16 KiB granule supported!"); - } - - if let Some(ID_AA64MMFR0_EL1::TGran64::Value::Supported) = - mmfr.read_as_enum(ID_AA64MMFR0_EL1::TGran64) - { - println!("[i] MMU: 64 KiB granule supported!"); - } - - match mmfr.read_as_enum(ID_AA64MMFR0_EL1::ASIDBits) { - Some(ID_AA64MMFR0_EL1::ASIDBits::Value::Bits_16) => { - println!("[i] MMU: 16 bit ASIDs supported!"); - } - Some(ID_AA64MMFR0_EL1::ASIDBits::Value::Bits_8) => { - println!("[i] MMU: 8 bit ASIDs supported!"); - } - _ => println!("[i] MMU: Invalid ASID bits specified!"), - } - - match mmfr.read_as_enum(ID_AA64MMFR0_EL1::PARange) { - Some(ID_AA64MMFR0_EL1::PARange::Value::Bits_32) => { - println!("[i] MMU: Up to 32 Bit physical address range supported!"); - } - Some(ID_AA64MMFR0_EL1::PARange::Value::Bits_36) => { - println!("[i] MMU: Up to 36 Bit physical address range supported!"); - } - Some(ID_AA64MMFR0_EL1::PARange::Value::Bits_40) => { - println!("[i] MMU: Up to 40 Bit physical address range supported!"); - } - Some(ID_AA64MMFR0_EL1::PARange::Value::Bits_42) => { - println!("[i] MMU: Up to 42 Bit physical address range supported!"); - } - Some(ID_AA64MMFR0_EL1::PARange::Value::Bits_44) => { - println!("[i] MMU: Up to 44 Bit physical address range supported!"); - } - Some(ID_AA64MMFR0_EL1::PARange::Value::Bits_48) => { - println!("[i] MMU: Up to 48 Bit physical address range supported!"); - } - Some(ID_AA64MMFR0_EL1::PARange::Value::Bits_52) => { - println!("[i] MMU: Up to 52 Bit physical address range supported!"); - } - _ => println!("[i] MMU: Invalid PARange specified!"), - } - - let tcr = TCR_EL1.extract(); - - match tcr.read_as_enum(TCR_EL1::IPS) { - Some(TCR_EL1::IPS::Value::Bits_32) => { - println!("[i] MMU: 32 Bit intermediate physical address size supported!"); - } - Some(TCR_EL1::IPS::Value::Bits_36) => { - println!("[i] MMU: 36 Bit intermediate physical address size supported!"); - } - Some(TCR_EL1::IPS::Value::Bits_40) => { - println!("[i] MMU: 40 Bit intermediate physical address size supported!"); - } - Some(TCR_EL1::IPS::Value::Bits_42) => { - println!("[i] MMU: 42 Bit intermediate physical address size supported!"); - } - Some(TCR_EL1::IPS::Value::Bits_44) => { - println!("[i] MMU: 44 Bit intermediate physical address size supported!"); - } - Some(TCR_EL1::IPS::Value::Bits_48) => { - println!("[i] MMU: 48 Bit intermediate physical address size supported!"); - } - Some(TCR_EL1::IPS::Value::Bits_52) => { - println!("[i] MMU: 52 Bit intermediate physical address size supported!"); - } - _ => println!("[i] MMU: Invalid IPS specified!"), - } - - match tcr.read_as_enum(TCR_EL1::TG0) { - Some(TCR_EL1::TG0::Value::KiB_4) => println!("[i] MMU: TTBR0 4 KiB granule active!"), - Some(TCR_EL1::TG0::Value::KiB_16) => println!("[i] MMU: TTBR0 16 KiB granule active!"), - Some(TCR_EL1::TG0::Value::KiB_64) => println!("[i] MMU: TTBR0 64 KiB granule active!"), - _ => println!("[i] MMU: Invalid TTBR0 granule size specified!"), - } - - let t0sz = tcr.read(TCR_EL1::T0SZ); - println!("[i] MMU: T0sz = 64-{} = {} bits", t0sz, 64 - t0sz); - - match tcr.read_as_enum(TCR_EL1::TG1) { - Some(TCR_EL1::TG1::Value::KiB_4) => println!("[i] MMU: TTBR1 4 KiB granule active!"), - Some(TCR_EL1::TG1::Value::KiB_16) => println!("[i] MMU: TTBR1 16 KiB granule active!"), - Some(TCR_EL1::TG1::Value::KiB_64) => println!("[i] MMU: TTBR1 64 KiB granule active!"), - _ => println!("[i] MMU: Invalid TTBR1 granule size specified!"), - } - - let t1sz = tcr.read(TCR_EL1::T1SZ); - println!("[i] MMU: T1sz = 64-{} = {} bits", t1sz, 64 - t1sz); - } -} diff --git a/libs/memory/src/arch/aarch64/mmu/translation_table.rs b/libs/memory/src/arch/aarch64/mmu/translation_table.rs deleted file mode 100644 index 2f5562da2..000000000 --- a/libs/memory/src/arch/aarch64/mmu/translation_table.rs +++ /dev/null @@ -1,423 +0,0 @@ -use { - super::{Granule64KiB, Granule512MiB, mair}, - crate::{ - Address, Physical, Virtual, - mmu::{AccessPermissions, AttributeFields, MemAttributes, MemoryRegion, PageAddress}, - platform, - }, - core::convert, - tock_registers::{ - interfaces::{Readable, Writeable}, - register_bitfields, - registers::InMemoryRegister, - }, -}; - -//-------------------------------------------------------------------------------------------------- -// Private Definitions -//-------------------------------------------------------------------------------------------------- - -register_bitfields! { - u64, - - /// A table descriptor, as per ARMv8-A Architecture Reference Manual Figure D5-15. - /// AArch64 Reference Manual page 2150, D5-2445 - STAGE1_TABLE_DESCRIPTOR [ - /// Physical address of the next descriptor. - NEXT_LEVEL_TABLE_ADDR_64KiB OFFSET(16) NUMBITS(32) [], // [47:16] - NEXT_LEVEL_TABLE_ADDR_4KiB OFFSET(12) NUMBITS(36) [], // [47:12] - - TYPE OFFSET(1) NUMBITS(1) [ - Block = 0, - Table = 1 - ], - - VALID OFFSET(0) NUMBITS(1) [ - False = 0, - True = 1 - ] - ] -} - -register_bitfields! { - u64, - - /// A level 3 page descriptor, as per ARMv8-A Architecture Reference Manual Figure D5-17. - /// AArch64 Reference Manual page 2150, D5-2445 - STAGE1_PAGE_DESCRIPTOR [ - /// Unprivileged execute-never. - UXN OFFSET(54) NUMBITS(1) [ - Execute = 0, - NeverExecute = 1 - ], - - /// Privileged execute-never - PXN OFFSET(53) NUMBITS(1) [ - Execute = 0, - NeverExecute = 1 - ], - - /// Physical address of the next table descriptor (lvl2) or the page descriptor (lvl3). - OUTPUT_ADDR_64KiB OFFSET(16) NUMBITS(32) [], // [47:16] - OUTPUT_ADDR_4KiB OFFSET(21) NUMBITS(27) [], // [47:21] - - /// Access flag - AF OFFSET(10) NUMBITS(1) [ - NotAccessed = 0, - Accessed = 1 - ], - - /// Shareability field - SH OFFSET(8) NUMBITS(2) [ - OuterShareable = 0b10, - InnerShareable = 0b11 - ], - - /// Access Permissions - AP OFFSET(6) NUMBITS(2) [ - RW_EL1 = 0b00, - RW_EL1_EL0 = 0b01, - RO_EL1 = 0b10, - RO_EL1_EL0 = 0b11 - ], - - /// Memory attributes index into the MAIR_EL1 register - AttrIndx OFFSET(2) NUMBITS(3) [], - - TYPE OFFSET(1) NUMBITS(1) [ - Reserved_Invalid = 0, - Page = 1 - ], - - VALID OFFSET(0) NUMBITS(1) [ - False = 0, - True = 1 - ] - ] -} - -/// A table descriptor with 64 KiB aperture. -/// -/// The output points to the next table. -#[derive(Copy, Clone)] -#[repr(C)] -pub struct TableDescriptor { - value: u64, -} - -/// A page descriptor with 64 KiB aperture. -/// -/// The output points to physical memory. -#[derive(Copy, Clone)] -#[repr(C)] -pub struct PageDescriptor { - value: u64, -} - -pub trait BaseAddr { - fn phys_start_addr(&self) -> Address; - fn base_addr_u64(&self) -> u64; - fn base_addr_usize(&self) -> usize; -} - -// const NUM_LVL2_TABLES: usize = platform::memory::mmu::KernelAddrSpace::SIZE >> Granule512MiB::SHIFT; - -//-------------------------------------------------------------------------------------------------- -// Public Definitions -//-------------------------------------------------------------------------------------------------- - -/// Big monolithic struct for storing the translation tables. Individual levels must be 64 KiB -/// aligned, so the lvl3 is put first. -#[repr(C)] -#[repr(align(65536))] -pub struct FixedSizeTranslationTable { - /// Page descriptors, covering 64 KiB windows per entry. - lvl3: [[PageDescriptor; 8192]; NUM_TABLES], - - /// Table descriptors, covering 512 MiB windows. - lvl2: [TableDescriptor; NUM_TABLES], - - /// Have the tables been initialized? - initialized: bool, -} - -// /// A translation table type for the kernel space. -// pub type KernelTranslationTable = FixedSizeTranslationTable; - -//-------------------------------------------------------------------------------------------------- -// Private Implementations -//-------------------------------------------------------------------------------------------------- - -impl BaseAddr for [T; N] { - // The binary is still identity mapped, so we don't need to convert here. - fn phys_start_addr(&self) -> Address { - Address::new(core::ptr::from_ref(self) as usize) - } - - fn base_addr_u64(&self) -> u64 { - core::ptr::from_ref(self) as u64 - } - - fn base_addr_usize(&self) -> usize { - core::ptr::from_ref(self) as usize - } -} - -impl TableDescriptor { - /// Create an instance. - /// - /// Descriptor is invalid by default. - pub const fn new_zeroed() -> Self { - Self { value: 0 } - } - - /// Create an instance pointing to the supplied address. - pub fn from_next_lvl_table_addr(phys_next_lvl_table_addr: Address) -> Self { - let val = InMemoryRegister::::new(0); - - let shifted = phys_next_lvl_table_addr.as_usize() >> Granule64KiB::SHIFT; - val.write( - STAGE1_TABLE_DESCRIPTOR::NEXT_LEVEL_TABLE_ADDR_64KiB.val(shifted as u64) - + STAGE1_TABLE_DESCRIPTOR::TYPE::Table - + STAGE1_TABLE_DESCRIPTOR::VALID::True, - ); - - TableDescriptor { value: val.get() } - } -} - -impl PageDescriptor { - /// Create an instance. - /// - /// Descriptor is invalid by default. - pub const fn new_zeroed() -> Self { - Self { value: 0 } - } - - /// Create an instance. - pub fn from_output_page_addr( - phys_output_page_addr: PageAddress, - attribute_fields: AttributeFields, - ) -> Self { - let val = InMemoryRegister::::new(0); - - let shifted = phys_output_page_addr.into_inner().as_usize() >> Granule64KiB::SHIFT; - val.write( - STAGE1_PAGE_DESCRIPTOR::OUTPUT_ADDR_64KiB.val(shifted as u64) - + STAGE1_PAGE_DESCRIPTOR::AF::Accessed - + STAGE1_PAGE_DESCRIPTOR::TYPE::Page - + STAGE1_PAGE_DESCRIPTOR::VALID::True - + attribute_fields.into(), - ); - - Self { value: val.get() } - } - - /// Returns the valid bit. - fn is_valid(self) -> bool { - InMemoryRegister::::new(self.value) - .is_set(STAGE1_PAGE_DESCRIPTOR::VALID) - } -} - -/// Convert the kernel's generic memory attributes to HW-specific attributes of the MMU. -impl convert::From - for tock_registers::fields::FieldValue -{ - fn from(attribute_fields: AttributeFields) -> Self { - // Memory attributes - let mut desc = match attribute_fields.mem_attributes { - MemAttributes::CacheableDRAM => { - STAGE1_PAGE_DESCRIPTOR::SH::InnerShareable - + STAGE1_PAGE_DESCRIPTOR::AttrIndx.val(mair::attr::NORMAL) - } - MemAttributes::NonCacheableDRAM => { - STAGE1_PAGE_DESCRIPTOR::SH::InnerShareable - + STAGE1_PAGE_DESCRIPTOR::AttrIndx.val(mair::attr::NORMAL_NON_CACHEABLE) - } - MemAttributes::Device => { - STAGE1_PAGE_DESCRIPTOR::SH::OuterShareable - + STAGE1_PAGE_DESCRIPTOR::AttrIndx.val(mair::attr::DEVICE_NGNRE) - } - }; - - // Access Permissions - desc += match attribute_fields.acc_perms { - AccessPermissions::ReadOnly => STAGE1_PAGE_DESCRIPTOR::AP::RO_EL1, - AccessPermissions::ReadWrite => STAGE1_PAGE_DESCRIPTOR::AP::RW_EL1, - }; - - // The execute-never attribute is mapped to PXN in AArch64. - desc += if attribute_fields.execute_never { - STAGE1_PAGE_DESCRIPTOR::PXN::NeverExecute - } else { - STAGE1_PAGE_DESCRIPTOR::PXN::Execute - }; - - // Always set unprivileged execute-never as long as userspace is not implemented yet. - desc += STAGE1_PAGE_DESCRIPTOR::UXN::NeverExecute; - - desc - } -} - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- -use crate::mmu::{AddressSpace, AssociatedTranslationTable}; - -impl AssociatedTranslationTable for AddressSpace -where - [u8; Self::SIZE >> Granule512MiB::SHIFT]: Sized, -{ - type TableStartFromBottom = FixedSizeTranslationTable<{ Self::SIZE >> Granule512MiB::SHIFT }>; -} - -impl Default for FixedSizeTranslationTable { - fn default() -> Self { - Self::new() - } -} - -impl FixedSizeTranslationTable { - /// Create an instance. - #[allow(clippy::assertions_on_constants)] - pub const fn new() -> Self { - assert!(platform::memory::mmu::KernelGranule::SIZE == Granule64KiB::SIZE); // assert! is const-fn-friendly - - // Can't have a zero-sized address space. - assert!(NUM_TABLES > 0); - - Self { - #[allow(clippy::large_stack_arrays)] - lvl3: [[PageDescriptor::new_zeroed(); 8192]; NUM_TABLES], - lvl2: [TableDescriptor::new_zeroed(); NUM_TABLES], - initialized: false, - } - } - - /// Helper to calculate the lvl2 and lvl3 indices from an address. - #[inline(always)] - fn lvl2_lvl3_index_from_page_addr( - virt_page_addr: PageAddress, - ) -> Result<(usize, usize), &'static str> { - let addr = virt_page_addr.into_inner().as_usize(); - let lvl2_index = addr >> Granule512MiB::SHIFT; - let lvl3_index = (addr & Granule512MiB::MASK) >> Granule64KiB::SHIFT; - - if lvl2_index >= NUM_TABLES { - return Err("Virtual page is out of bounds of translation table"); - } - - Ok((lvl2_index, lvl3_index)) - } - - /// Sets the `PageDescriptor` corresponding to the supplied page address. - /// - /// Doesn't allow overriding an already valid page. - #[inline(always)] - fn set_page_descriptor_from_page_addr( - &mut self, - virt_page_addr: PageAddress, - new_desc: PageDescriptor, - ) -> Result<(), &'static str> { - let (lvl2_index, lvl3_index) = Self::lvl2_lvl3_index_from_page_addr(virt_page_addr)?; - let desc = self - .lvl3 - .get_mut(lvl2_index) - .and_then(|lvl3_table: &mut [PageDescriptor; 8192]| lvl3_table.get_mut(lvl3_index)) - .ok_or("Invalid page address for translation table")?; - - if desc.is_valid() { - return Err("Virtual page is already mapped"); - } - - *desc = new_desc; - Ok(()) - } -} - -//------------------------------------------------------------------------------ -// OS Interface Code -//------------------------------------------------------------------------------ - -impl crate::mmu::translation_table::interface::TranslationTable - for FixedSizeTranslationTable -{ - /// Iterates over all static translation table entries and fills them at once. - /// - /// # Safety - /// - /// - Modifies a `static mut`. Ensure it only happens from here. - // pub unsafe fn populate_translation_table_entries(&mut self) -> Result<(), &'static str> { - // for (l2_nr, l2_entry) in self.lvl2.iter_mut().enumerate() { - // *l2_entry = - // TableDescriptor::from_next_lvl_table_addr(self.lvl3[l2_nr].base_addr_usize()); - // - // for (l3_nr, l3_entry) in self.lvl3[l2_nr].iter_mut().enumerate() { - // let virt_addr = (l2_nr << Granule512MiB::SHIFT) + (l3_nr << Granule64KiB::SHIFT); - // - // let (phys_output_addr, attribute_fields) = - // platform::memory::mmu::virt_mem_layout().virt_addr_properties(virt_addr)?; - // - // *l3_entry = PageDescriptor::from_output_addr(phys_output_addr, &attribute_fields); - // } - // } - // - // Ok(()) - // } - fn init(&mut self) -> Result<(), &'static str> { - if self.initialized { - return Ok(()); - } - - // Populate the l2 entries. - for (lvl2_nr, lvl2_entry) in self.lvl2.iter_mut().enumerate() { - let phys_table_addr = self - .lvl3 - .get(lvl2_nr) - .ok_or("Invalid lvl2 index")? - .phys_start_addr(); - - let new_desc = TableDescriptor::from_next_lvl_table_addr(phys_table_addr); - *lvl2_entry = new_desc; - } - - self.initialized = true; - Ok(()) - } - - fn phys_base_address(&self) -> Address { - self.lvl2.phys_start_addr() - } - - unsafe fn map_at( - &mut self, - virt_region: &MemoryRegion, - phys_region: &MemoryRegion, - attr: AttributeFields, - ) -> Result<(), &'static str> { - assert!(self.initialized, "Translation tables not initialized"); - - if virt_region.size() != phys_region.size() { - return Err("Tried to map memory regions with different sizes"); - } - - if phys_region.end_exclusive_page_addr() - > crate::platform::memory::phys_addr_space_end_exclusive_addr() - { - return Err("Tried to map outside of physical address space"); - } - - #[allow(clippy::useless_conversion)] - let iter = phys_region.into_iter().zip(virt_region.into_iter()); - for (phys_page_addr, virt_page_addr) in iter { - let new_desc = PageDescriptor::from_output_page_addr(phys_page_addr, attr); - let virt_page = virt_page_addr; - - self.set_page_descriptor_from_page_addr(virt_page, new_desc)?; - } - - Ok(()) - } -} diff --git a/libs/memory/src/arch/aarch64/mod.rs b/libs/memory/src/arch/aarch64/mod.rs index 8d551f9d9..5a93cebbc 100644 --- a/libs/memory/src/arch/aarch64/mod.rs +++ b/libs/memory/src/arch/aarch64/mod.rs @@ -5,13 +5,12 @@ //! Memory management functions for aarch64. -mod addr; -pub mod mmu; - -// pub use addr::{PhysAddr, VirtAddr}; +mod common; +mod granule_16k; +mod granule_4k; +mod granule_64k; -// aarch64 granules and page sizes howto: -// https://stackoverflow.com/questions/34269185/simultaneous-existence-of-different-sized-pages-on-aarch64 +pub mod features; +pub mod mmu; -/// Default page size used by the kernel. -pub const PAGE_SIZE: usize = 65536; +pub use {granule_4k::Aarch64_4K, granule_16k::Aarch64_16K, granule_64k::Aarch64_64K}; diff --git a/libs/memory/src/arch/mod.rs b/libs/memory/src/arch/mod.rs index 3fd4b4298..07641623b 100644 --- a/libs/memory/src/arch/mod.rs +++ b/libs/memory/src/arch/mod.rs @@ -1,5 +1,6 @@ #[cfg(target_arch = "aarch64")] pub mod aarch64; -#[cfg(target_arch = "aarch64")] -pub use aarch64::*; +pub mod powerpc; +pub mod riscv64; +pub mod x86_64; diff --git a/libs/memory/src/arch/powerpc.rs b/libs/memory/src/arch/powerpc.rs new file mode 100644 index 000000000..3bec81944 --- /dev/null +++ b/libs/memory/src/arch/powerpc.rs @@ -0,0 +1,331 @@ +use { + crate::arch_trait::{EntryKind, LevelCapabilities, TranslationArch}, + libaddress::{PhysAddr, VirtAddr}, + libmapping::{AccessPermissions, AttributeFields, MemAttributes}, +}; + +// --------------------------------------------------------------------------- +// PowerPC 970MP Hashed Page Table (HPT) entry layout +// --------------------------------------------------------------------------- +// +// The 970MP uses a hashed page table (HPT) rather than hierarchical page +// tables. Translation is a two-step process: +// +// 1. SLB (Segment Lookaside Buffer) translates the effective address (EA) +// into a virtual address (VA) by providing a VSID. +// +// 2. The HPT maps the VA to a real (physical) address via hash lookup. +// +// The HPT is a flat array of PTEGs (Page Table Entry Groups). Each PTEG +// contains 8 PTEs. Each PTE is 16 bytes (two 64-bit words). +// +// PTEG selection: +// primary_hash = (VSID ^ page_index) & htab_mask +// secondary_hash = ~primary_hash & htab_mask +// +// PTE format (16 bytes = 2 × u64): +// +// pte_hi (word 0): +// [63] V — Valid +// [62:7] AVPN — Abbreviated Virtual Page Number +// [6:2] (reserved, software use) +// [1] H — Hash function identifier (0=primary, 1=secondary) +// [0] (reserved) +// +// For large pages (16MB): +// [63] V — Valid +// [62:7] AVPN — but bits [22:12] of the VA are in pte_hi[24:14] +// [3] L — Large page (1 = 16MB) +// [1] H — Hash function identifier +// +// pte_lo (word 1): +// [63:12] RPGN — Real Page Group Number (physical page frame) +// [11:9] (reserved) +// [8] R — Referenced (set by hardware) +// [7] C — Changed (set by hardware) +// [6:3] WIMG — Memory/cache control +// W = Write-through +// I = Cache-inhibited +// M = Memory coherence required +// G = Guarded (no speculative access) +// [2] N — No-execute +// [1:0] PP — Page protection +// 00 = no access (supervisor only) +// 01 = supervisor read/write +// 10 = supervisor read/write, user read/write +// 11 = supervisor read-only, user read-only + +// --- pte_hi bits --- +const PTE_HI_VALID: u64 = 1 << 63; +const PTE_HI_HASH: u64 = 1 << 1; +const PTE_HI_LARGE: u64 = 1 << 3; + +// AVPN occupies bits [62:7] of pte_hi. +// For a 4KB page, AVPN = VA[77:23] (the upper bits of the virtual page number). +const PTE_HI_AVPN_MASK: u64 = 0x7FFF_FFFF_FFFF_FF80; + +// --- pte_lo bits --- +const PTE_LO_RPGN_MASK: u64 = 0xFFFF_FFFF_FFFF_F000; // bits [63:12] +const PTE_LO_REF: u64 = 1 << 8; +const PTE_LO_CHG: u64 = 1 << 7; +const PTE_LO_NOEXEC: u64 = 1 << 2; + +// WIMG field: bits [6:3] +const PTE_LO_WIMG_SHIFT: u64 = 3; +const _PTE_LO_WIMG_MASK: u64 = 0xF << PTE_LO_WIMG_SHIFT; + +// WIMG bit values (shifted) +const WIMG_M: u64 = 0b0010 << PTE_LO_WIMG_SHIFT; // Memory coherence +const WIMG_IG: u64 = 0b0101 << PTE_LO_WIMG_SHIFT; // Cache-inhibited + Guarded + +// PP field: bits [1:0] +const _PP_NO_ACCESS: u64 = 0b00; +const PP_RW_SUPER: u64 = 0b01; +const _PP_RW_ALL: u64 = 0b10; +const PP_RO_ALL: u64 = 0b11; + +// Page sizes +const SIZE_4K: usize = 4096; +const _SIZE_16M: usize = 16 * 1024 * 1024; + +// PTEs per PTEG +const PTES_PER_PTEG: usize = 8; + +/// PowerPC 970MP hashed page table with 4KiB base page size. +/// +/// The 970MP uses a fundamentally different translation scheme from +/// hierarchical page tables. Instead of a multi-level tree, it uses: +/// +/// - **SLB** (Segment Lookaside Buffer): hardware-managed register file +/// that maps effective address segments to VSIDs. Managed separately +/// from this module. +/// +/// - **HPT** (Hashed Page Table): a flat hash table of PTEGs. Each PTEG +/// contains 8 PTEs (16 bytes each). The hash is computed from the VSID +/// and virtual page number. +/// +/// Since the HPT is a single flat structure, `NUM_LEVELS = 1` and +/// `ENTRY_WIDTH = 2` (each PTE is two u64 words: `pte_hi` and `pte_lo`). +/// +/// Page sizes: 4KiB (base) and 16MiB (large, indicated by the L bit). +#[allow(non_camel_case_types)] +pub struct PowerPC_970; + +impl PowerPC_970 { + /// Number of PTEGs for a given HPT size in bytes. + /// HPT size must be a power of 2, minimum 256KB (2^18). + pub fn ptegs_for_htab_size(htab_size_bytes: usize) -> usize { + htab_size_bytes / (PTES_PER_PTEG * 16) + } + + /// Compute the `htab_mask` from the HPT size in bytes. + /// This masks the hash to select a PTEG within the table. + pub fn htab_mask(htab_size_bytes: usize) -> u64 { + (Self::ptegs_for_htab_size(htab_size_bytes) - 1) as u64 + } + + /// Build the AVPN field for `pte_hi` from a VSID and VA page index. + /// + /// For 4KB pages: AVPN = VSID[36:0] << 23 | `page_index`[15:11] + /// (bits of the VA that are NOT part of the hash). + pub fn build_avpn(vsid: u64, va_page_index: u64) -> u64 { + // AVPN goes in bits [62:7] of pte_hi. + // The AVPN encodes VSID and the upper bits of the page index. + let avpn = (vsid << 12) | (va_page_index >> 4); + (avpn << 7) & PTE_HI_AVPN_MASK + } + + /// Extract the page index from a virtual address for 4KB pages. + pub fn page_index_4k(vaddr: VirtAddr) -> u64 { + (vaddr.as_u64() >> 12) & 0xFFFF + } + + /// Extract the page index from a virtual address for 16MB large pages. + pub fn page_index_16m(vaddr: VirtAddr) -> u64 { + (vaddr.as_u64() >> 24) & 0xFF + } +} + +impl TranslationArch for PowerPC_970 { + const NUM_LEVELS: usize = 1; + const ENTRY_WIDTH: usize = 2; // 16-byte PTEs: [pte_hi, pte_lo] + const HASHED: bool = true; + + fn entries_per_table(_level: usize) -> usize { + // The actual number of entries depends on the HPT size, which is + // runtime-configurable. For trait purposes we return a nominal + // value. Real table creation uses the runtime HPT size. + // + // A minimum HPT is 256KB = 16384 entries (2048 PTEGs × 8 PTEs). + // Callers should use PowerPC_970::ptegs_for_htab_size() and + // construct tables with the appropriate size. + 2048 * PTES_PER_PTEG + } + + fn table_alignment(_level: usize) -> usize { + // HPT must be naturally aligned to its size. The minimum is 256KB. + // In practice the kernel allocates a power-of-2 aligned region. + 256 * 1024 + } + + fn level_capabilities(level: usize) -> LevelCapabilities { + match level { + 0 => LevelCapabilities { + supports_table_pointer: false, + supports_block: true, + block_size: SIZE_4K, + }, + _ => LevelCapabilities { + supports_table_pointer: false, + supports_block: false, + block_size: 0, + }, + } + } + + fn index_from_vaddr(_vaddr: VirtAddr, _level: usize) -> usize { + // HPT does not use hierarchical index extraction. + // Use hash_primary/hash_secondary instead. + 0 + } + + // -- Single-u64 methods are not meaningful for HPT -- + + fn decode_entry(_raw: u64, _level: usize) -> EntryKind { + // Use decode_entry_wide for 16-byte PTEs. + EntryKind::Invalid + } + + fn encode_table_entry(_next_table_phys: PhysAddr, _level: usize) -> u64 { + // HPT has no table pointers. + 0 + } + + fn encode_block_entry(_phys: PhysAddr, _attr: AttributeFields, _level: usize) -> u64 { + // Use encode_page_entry_wide for HPT entries. + 0 + } + + fn encode_page_entry(_phys: PhysAddr, _attr: AttributeFields) -> u64 { + // Use encode_page_entry_wide for HPT entries. + 0 + } + + fn output_address(_raw: u64, _level: usize) -> PhysAddr { + // Use output_address_wide for HPT entries. + PhysAddr::new(0) + } + + // -- Wide entry methods (the real API for HPT) -- + + fn decode_entry_wide(raw: &[u64], _level: usize) -> EntryKind { + debug_assert_eq!(raw.len(), 2); + let pte_hi = raw[0]; + let pte_lo = raw[1]; + + if pte_hi & PTE_HI_VALID == 0 { + return EntryKind::Invalid; + } + + let rpgn = pte_lo & PTE_LO_RPGN_MASK; + EntryKind::Block(PhysAddr::new(rpgn)) + } + + fn encode_page_entry_wide(phys: PhysAddr, attr: AttributeFields, buf: &mut [u64]) { + debug_assert_eq!(buf.len(), 2); + let addr = phys.as_u64(); + debug_assert!( + addr.trailing_zeros() >= 12, + "Physical address not 4K aligned" + ); + + // Build pte_lo: RPGN + WIMG + PP + N + R + C pre-set + let mut pte_lo = addr & PTE_LO_RPGN_MASK; + pte_lo |= encode_wimg(attr.mem_attributes); + pte_lo |= encode_pp(attr.acc_perms); + if !attr.executable { + pte_lo |= PTE_LO_NOEXEC; + } + // Pre-set Referenced and Changed to avoid software faults. + pte_lo |= PTE_LO_REF | PTE_LO_CHG; + + // Build pte_hi: Valid bit set. AVPN and H bit must be filled in + // by the caller (they depend on VSID and hash group), so we just + // set the valid bit here. The caller uses write_raw_wide to + // compose the final entry with AVPN. + let pte_hi = PTE_HI_VALID; + + buf[0] = pte_hi; + buf[1] = pte_lo; + } + + fn output_address_wide(raw: &[u64], _level: usize) -> PhysAddr { + debug_assert_eq!(raw.len(), 2); + let pte_lo = raw[1]; + PhysAddr::new(pte_lo & PTE_LO_RPGN_MASK) + } + + // -- Hash-based lookup -- + + fn hash_primary(vaddr: VirtAddr, vsid: u64, htab_mask: u64) -> usize { + // Primary hash = (VSID ^ page_index) & htab_mask + let page_index = PowerPC_970::page_index_4k(vaddr); + usize::try_from((vsid ^ page_index) & htab_mask).unwrap() + } + + fn hash_secondary(primary_hash: usize, htab_mask: u64) -> usize { + // Secondary hash = ~primary_hash & htab_mask + usize::try_from(!(primary_hash as u64) & htab_mask).unwrap() + } +} + +/// Encode memory attributes into WIMG bits for `pte_lo`. +fn encode_wimg(mem_attr: MemAttributes) -> u64 { + match mem_attr { + // Normal cacheable memory: M=1 (coherence required for SMP) + MemAttributes::CacheableDRAM => WIMG_M, + // Non-cacheable memory: W=0, I=1, M=0, G=0 + MemAttributes::NonCacheableDRAM => 0b0100 << PTE_LO_WIMG_SHIFT, + // Device/MMIO: I=1, G=1 (cache-inhibited + guarded) + MemAttributes::Device => WIMG_IG, + } +} + +/// Encode access permissions into PP bits for `pte_lo`. +fn encode_pp(acc_perms: AccessPermissions) -> u64 { + match acc_perms { + // Supervisor read/write (user no access for now) + AccessPermissions::ReadWrite => PP_RW_SUPER, + // Read-only for all + AccessPermissions::ReadOnly => PP_RO_ALL, + } +} + +/// Helper for constructing a complete HPT PTE with AVPN and hash info. +/// +/// `encode_page_entry_wide` sets the valid bit and physical attributes but +/// leaves AVPN and H zeroed. This function builds the complete `pte_hi` +/// word with the full AVPN, hash-group indicator, and optional large-page bit. +/// +/// Typical usage: call `encode_page_entry_wide` to get the base [`pte_hi`, `pte_lo`], +/// then OR the result of this function into `buf[0]` to set AVPN and H. +#[allow(dead_code)] +pub fn build_complete_pte_hi(vsid: u64, vaddr: VirtAddr, secondary: bool, large_page: bool) -> u64 { + let page_index = if large_page { + PowerPC_970::page_index_16m(vaddr) + } else { + PowerPC_970::page_index_4k(vaddr) + }; + + let mut pte_hi = PTE_HI_VALID; + pte_hi |= PowerPC_970::build_avpn(vsid, page_index); + + if secondary { + pte_hi |= PTE_HI_HASH; + } + if large_page { + pte_hi |= PTE_HI_LARGE; + } + + pte_hi +} diff --git a/libs/memory/src/arch/riscv64.rs b/libs/memory/src/arch/riscv64.rs new file mode 100644 index 000000000..259b2c68e --- /dev/null +++ b/libs/memory/src/arch/riscv64.rs @@ -0,0 +1,264 @@ +use { + crate::arch_trait::{EntryKind, LevelCapabilities, TranslationArch}, + libaddress::{PhysAddr, VirtAddr}, + libmapping::{AccessPermissions, AttributeFields, MemAttributes}, +}; + +// --------------------------------------------------------------------------- +// RISC-V Sv48 page table entry bit layout (4-level, 4KiB pages) +// --------------------------------------------------------------------------- +// +// Sv48 virtual address split (48-bit, sign-extended from bit 47): +// +// 63-48 47-39 38-30 29-21 20-12 11-0 +// signx VPN[3] VPN[2] VPN[1] VPN[0] off +// L0 L1 L2 L3 +// +// Each level indexes 512 entries (9 bits). Table size = 512 * 8 = 4096 bytes. +// +// Any non-leaf level can produce a superpage if R, W, or X bits are set: +// L0 (VPN[3]): 512 GiB terapage (rare, usually table-only) +// L1 (VPN[2]): 1 GiB gigapage +// L2 (VPN[1]): 2 MiB megapage +// L3 (VPN[0]): 4 KiB page (always leaf) +// +// PTE format (64 bits): +// +// [63:54] reserved (must be 0) +// [53:10] PPN (Physical Page Number) — PPN[2] [53:28], PPN[1] [27:19], PPN[0] [18:10] +// [9:8] RSW (reserved for supervisor software) +// [7] D (Dirty) +// [6] A (Accessed) +// [5] G (Global) +// [4] U (User accessible) +// [3] X (Executable) +// [2] W (Writable) +// [1] R (Readable) +// [0] V (Valid) +// +// Leaf vs non-leaf determination: +// V=1 and R=0, W=0, X=0 => pointer to next-level table +// V=1 and (R=1 or X=1) => leaf page/superpage +// V=0 => invalid +// W=1 and R=0 => reserved (invalid) + +// PTE flag bits +const V: u64 = 1 << 0; // Valid +const R: u64 = 1 << 1; // Readable +const W: u64 = 1 << 2; // Writable +const X: u64 = 1 << 3; // Executable +const _U: u64 = 1 << 4; // User accessible (reserved for userspace support) +const _G: u64 = 1 << 5; // Global (reserved for global mappings) +const A: u64 = 1 << 6; // Accessed +const D: u64 = 1 << 7; // Dirty + +// PPN extraction: PTE bits [53:10] contain the physical page number. +// Physical address = PPN << 12. +const PPN_SHIFT: u64 = 10; +const PPN_MASK: u64 = 0x003F_FFFF_FFFF_FC00; // bits [53:10] + +// For superpages, lower PPN fields must be zero: +// 2MiB megapage (L2): PPN[0] (bits [18:10]) must be 0 +// 1GiB gigapage (L1): PPN[0] and PPN[1] (bits [27:10]) must be 0 +// 512GiB terapage (L0): PPN[0], PPN[1], PPN[2] partially (bits [37:10]) must be 0 + +// Block sizes +const SIZE_4K: usize = 4096; +const SIZE_2M: usize = 2 * 1024 * 1024; +const SIZE_1G: usize = 1024 * 1024 * 1024; +const SIZE_512G: usize = 512 * 1024 * 1024 * 1024; + +/// Convert a PPN from a PTE into a physical address. +fn ppn_to_phys(raw: u64) -> u64 { + ((raw & PPN_MASK) >> PPN_SHIFT) << 12 +} + +/// Convert a physical address into PPN bits positioned for a PTE. +fn phys_to_ppn(phys: u64) -> u64 { + (phys >> 12) << PPN_SHIFT +} + +/// Is this PTE a leaf (page/superpage)? +/// A valid entry is a leaf if any of R, W, X are set. +fn is_leaf(raw: u64) -> bool { + raw & (R | W | X) != 0 +} + +/// RISC-V Sv48 4-level paging. +/// +/// 4-level hierarchy with 512 entries per table, 4KiB alignment. +/// Supports superpages at L0 (512GiB), L1 (1GiB), L2 (2MiB), and +/// regular 4KiB pages at L3. +/// +/// Unlike `AArch64` and `x86_64`, RISC-V uses the same PTE format at every +/// level. The distinction between table pointer and leaf is purely based +/// on the R/W/X permission bits: if none are set, it's a table pointer. +#[allow(non_camel_case_types)] +pub struct RiscV_Sv48; + +impl TranslationArch for RiscV_Sv48 { + const NUM_LEVELS: usize = 4; + + fn entries_per_table(_level: usize) -> usize { + 512 + } + + fn table_alignment(_level: usize) -> usize { + SIZE_4K + } + + fn level_capabilities(level: usize) -> LevelCapabilities { + match level { + // L0: table pointer or 512GiB terapage (generally table-only in practice) + 0 => LevelCapabilities { + supports_table_pointer: true, + supports_block: true, + block_size: SIZE_512G, + }, + // L1: table pointer or 1GiB gigapage + 1 => LevelCapabilities { + supports_table_pointer: true, + supports_block: true, + block_size: SIZE_1G, + }, + // L2: table pointer or 2MiB megapage + 2 => LevelCapabilities { + supports_table_pointer: true, + supports_block: true, + block_size: SIZE_2M, + }, + // L3: 4KiB page only (leaf only) + 3 => LevelCapabilities { + supports_table_pointer: false, + supports_block: true, + block_size: SIZE_4K, + }, + _ => LevelCapabilities { + supports_table_pointer: false, + supports_block: false, + block_size: 0, + }, + } + } + + fn index_from_vaddr(vaddr: VirtAddr, level: usize) -> usize { + let shift = match level { + 0 => 39, // VPN[3] + 1 => 30, // VPN[2] + 2 => 21, // VPN[1] + 3 => 12, // VPN[0] + _ => return 0, + }; + ((vaddr.as_u64() >> shift) & 0x1FF) as usize + } + + fn decode_entry(raw: u64, _level: usize) -> EntryKind { + if raw & V == 0 { + return EntryKind::Invalid; + } + + // W=1, R=0 is a reserved combination + if raw & W != 0 && raw & R == 0 { + return EntryKind::Invalid; + } + + if is_leaf(raw) { + EntryKind::Block(PhysAddr::new(ppn_to_phys(raw))) + } else { + EntryKind::Table(PhysAddr::new(ppn_to_phys(raw))) + } + } + + fn encode_table_entry(next_table_phys: PhysAddr, _level: usize) -> u64 { + let addr = next_table_phys.as_u64(); + debug_assert!(addr.trailing_zeros() >= 12, "Table address not 4K aligned"); + // V=1, R=W=X=0 => non-leaf (table pointer) + phys_to_ppn(addr) | V + } + + fn encode_block_entry(phys: PhysAddr, attr: AttributeFields, level: usize) -> u64 { + let addr = phys.as_u64(); + + // Verify superpage alignment: lower PPN fields must be zero + match level { + 0 => debug_assert!( + addr & (SIZE_512G as u64 - 1) == 0, + "512GiB terapage not aligned" + ), + 1 => debug_assert!( + addr & (SIZE_1G as u64 - 1) == 0, + "1GiB gigapage not aligned" + ), + 2 => debug_assert!( + addr & (SIZE_2M as u64 - 1) == 0, + "2MiB megapage not aligned" + ), + _ => panic!("Use encode_page_entry for L3 leaf entries"), + } + + phys_to_ppn(addr) | encode_attributes(attr) | V + } + + fn encode_page_entry(phys: PhysAddr, attr: AttributeFields) -> u64 { + let addr = phys.as_u64(); + debug_assert!(addr.trailing_zeros() >= 12, "Page address not 4K aligned"); + phys_to_ppn(addr) | encode_attributes(attr) | V + } + + fn output_address(raw: u64, _level: usize) -> PhysAddr { + PhysAddr::new(ppn_to_phys(raw)) + } +} + +/// Encode `AttributeFields` into RISC-V PTE permission/attribute bits. +/// +/// RISC-V is simpler than ARM/x86 — no MAIR or PWT/PCD. Memory type +/// is controlled by the Svpbmt extension (bits [62:61]) when available, +/// but the base spec treats all memory as cacheable. For device memory +/// we rely on Svpbmt or PMA (Physical Memory Attributes) configured +/// elsewhere. +fn encode_attributes(attr: AttributeFields) -> u64 { + let mut bits: u64 = 0; + + // Pre-set Accessed and Dirty to avoid page faults on first access. + bits |= A | D; + + // RISC-V permission model: R, W, X bits directly. + // At minimum, a leaf entry must have R or X set. + match attr.acc_perms { + AccessPermissions::ReadWrite => bits |= R | W, + AccessPermissions::ReadOnly => bits |= R, + } + + if attr.executable { + bits |= X; + } + + // Supervisor-only for now (U=0). + // When userspace is implemented, set U based on context. + + // Memory type hints via Svpbmt (bits [62:61]) when the extension is + // available. Without Svpbmt, PMAs define memory type and these bits + // are reserved-zero. + // + // Svpbmt encoding: + // 00 = PMA (default, use Physical Memory Attributes) + // 01 = NC (non-cacheable) + // 10 = IO (I/O, strongly ordered) + // 11 = reserved + match attr.mem_attributes { + MemAttributes::CacheableDRAM => { + // 00 = PMA default (cacheable for normal RAM) + } + MemAttributes::NonCacheableDRAM => { + // 01 = NC + bits |= 1 << 61; + } + MemAttributes::Device => { + // 10 = IO + bits |= 2 << 61; + } + } + + bits +} diff --git a/libs/memory/src/arch/x86_64.rs b/libs/memory/src/arch/x86_64.rs new file mode 100644 index 000000000..9b0895e64 --- /dev/null +++ b/libs/memory/src/arch/x86_64.rs @@ -0,0 +1,257 @@ +use { + crate::arch_trait::{EntryKind, LevelCapabilities, TranslationArch}, + libaddress::{PhysAddr, VirtAddr}, + libmapping::{AccessPermissions, AttributeFields, MemAttributes}, +}; + +// --------------------------------------------------------------------------- +// x86_64 page table entry bit layout (4-level, 4KiB pages) +// --------------------------------------------------------------------------- +// +// Virtual address split (48-bit canonical): +// +// 63-48 47-39 38-30 29-21 20-12 11-0 +// signx PML4 PDPT PD PT off +// (copy L0 L1 L2 L3 +// of b47) +// +// Each level indexes 512 entries (9 bits). Table size = 512 * 8 = 4096 bytes. +// +// L0 (PML4): table pointer only +// L1 (PDPT): table pointer or 1GiB page (if PS bit set) +// L2 (PD): table pointer or 2MiB page (if PS bit set) +// L3 (PT): 4KiB page only +// +// Common entry format: +// +// [63] NX (No Execute) +// [62:52] available / reserved (depending on feature) +// [51:12] physical address (4K aligned; [51:M] must be 0 where M = MAXPHYADDR) +// [11:9] available +// [8] G (Global, ignored in PML4E/PDPTE) +// [7] PS (Page Size: 0=points to table, 1=maps large page; must be 0 in PML4E and PTE) +// [6] D (Dirty, only on leaf entries) +// [5] A (Accessed) +// [4] PCD (Page-level Cache Disable) +// [3] PWT (Page-level Write-Through) +// [2] U/S (User/Supervisor: 0=supervisor only, 1=user accessible) +// [1] R/W (Read/Write: 0=read-only, 1=writable) +// [0] P (Present) +// +// Large page (1GiB at L1, 2MiB at L2): +// Same as above but PS=1, and address bits below the page size are +// repurposed: [20:13] = PAT/reserved for 2MiB, [29:13] for 1GiB. + +// Entry bit positions +const PRESENT: u64 = 1 << 0; +const WRITABLE: u64 = 1 << 1; +const USER_ACCESSIBLE: u64 = 1 << 2; +const WRITE_THROUGH: u64 = 1 << 3; +const CACHE_DISABLE: u64 = 1 << 4; +const ACCESSED: u64 = 1 << 5; +const DIRTY: u64 = 1 << 6; +const PAGE_SIZE: u64 = 1 << 7; // PS bit — large page at L1/L2 +const NO_EXECUTE: u64 = 1 << 63; + +// Address masks +const ADDR_MASK_4K: u64 = 0x000F_FFFF_FFFF_F000; // [51:12] +const ADDR_MASK_2M: u64 = 0x000F_FFFF_FFE0_0000; // [51:21] +const ADDR_MASK_1G: u64 = 0x000F_FFFF_C000_0000; // [51:30] + +// Block sizes +const SIZE_4K: usize = 4096; +const SIZE_2M: usize = 2 * 1024 * 1024; +const SIZE_1G: usize = 1024 * 1024 * 1024; + +/// `x86_64` 4-level paging with 4KiB pages. +/// +/// 4-level hierarchy: PML4 (L0) -> PDPT (L1) -> PD (L2) -> PT (L3). +/// 512 entries per table, 4096-byte table size, 4096-byte alignment. +/// +/// Supports 1GiB huge pages at L1 (PDPT) and 2MiB large pages at L2 (PD) +/// via the PS (Page Size) bit. +pub struct X86_64_4K; + +impl TranslationArch for X86_64_4K { + const NUM_LEVELS: usize = 4; + + fn entries_per_table(_level: usize) -> usize { + 512 + } + + fn table_alignment(_level: usize) -> usize { + SIZE_4K + } + + fn level_capabilities(level: usize) -> LevelCapabilities { + match level { + // PML4: table pointer only (no large pages) + 0 => LevelCapabilities { + supports_table_pointer: true, + supports_block: false, + block_size: 0, + }, + // PDPT: table pointer or 1GiB huge page + 1 => LevelCapabilities { + supports_table_pointer: true, + supports_block: true, + block_size: SIZE_1G, + }, + // PD: table pointer or 2MiB large page + 2 => LevelCapabilities { + supports_table_pointer: true, + supports_block: true, + block_size: SIZE_2M, + }, + // PT: 4KiB page only + 3 => LevelCapabilities { + supports_table_pointer: false, + supports_block: true, + block_size: SIZE_4K, + }, + _ => LevelCapabilities { + supports_table_pointer: false, + supports_block: false, + block_size: 0, + }, + } + } + + fn index_from_vaddr(vaddr: VirtAddr, level: usize) -> usize { + let shift = match level { + 0 => 39, // PML4 index + 1 => 30, // PDPT index + 2 => 21, // PD index + 3 => 12, // PT index + _ => return 0, + }; + ((vaddr.as_u64() >> shift) & 0x1FF) as usize + } + + fn decode_entry(raw: u64, level: usize) -> EntryKind { + if raw & PRESENT == 0 { + return EntryKind::Invalid; + } + + match level { + // PML4: always a table pointer (PS must be 0) + 0 => EntryKind::Table(PhysAddr::new(raw & ADDR_MASK_4K)), + // PDPT: PS=1 means 1GiB page, PS=0 means table pointer + 1 => { + if raw & PAGE_SIZE != 0 { + EntryKind::Block(PhysAddr::new(raw & ADDR_MASK_1G)) + } else { + EntryKind::Table(PhysAddr::new(raw & ADDR_MASK_4K)) + } + } + // PD: PS=1 means 2MiB page, PS=0 means table pointer + 2 => { + if raw & PAGE_SIZE != 0 { + EntryKind::Block(PhysAddr::new(raw & ADDR_MASK_2M)) + } else { + EntryKind::Table(PhysAddr::new(raw & ADDR_MASK_4K)) + } + } + // PT: always a 4KiB page (PS is ignored / must be 0) + 3 => EntryKind::Block(PhysAddr::new(raw & ADDR_MASK_4K)), + _ => EntryKind::Invalid, + } + } + + fn encode_table_entry(next_table_phys: PhysAddr, _level: usize) -> u64 { + let addr = next_table_phys.as_u64(); + debug_assert!(addr & !ADDR_MASK_4K == 0, "Table address not 4K aligned"); + // Table entries: P=1, R/W=1, U/S=1 (permissive — leaf entries restrict further) + (addr & ADDR_MASK_4K) | USER_ACCESSIBLE | WRITABLE | PRESENT + } + + fn encode_block_entry(phys: PhysAddr, attr: AttributeFields, level: usize) -> u64 { + let addr = phys.as_u64(); + let addr_mask = match level { + 1 => ADDR_MASK_1G, + 2 => ADDR_MASK_2M, + _ => panic!("Large page entries only valid at L1 (1GiB) and L2 (2MiB)"), + }; + debug_assert!( + addr & !addr_mask == 0, + "Block address not aligned for level" + ); + // PS=1 for large pages at L1/L2 + (addr & addr_mask) | PAGE_SIZE | encode_attributes(attr) | PRESENT + } + + fn encode_page_entry(phys: PhysAddr, attr: AttributeFields) -> u64 { + let addr = phys.as_u64(); + debug_assert!(addr & !ADDR_MASK_4K == 0, "Page address not 4K aligned"); + // PT entries: no PS bit, just P=1 + (addr & ADDR_MASK_4K) | encode_attributes(attr) | PRESENT + } + + fn output_address(raw: u64, level: usize) -> PhysAddr { + let mask = match level { + 0 | 3 => ADDR_MASK_4K, + 1 => { + if raw & PAGE_SIZE != 0 { + ADDR_MASK_1G + } else { + ADDR_MASK_4K + } + } + 2 => { + if raw & PAGE_SIZE != 0 { + ADDR_MASK_2M + } else { + ADDR_MASK_4K + } + } + _ => 0, + }; + PhysAddr::new(raw & mask) + } +} + +/// Encode `AttributeFields` into the `x86_64` PTE flag bits. +/// +/// `x86_64` uses a different model from ARM: +/// - No MAIR — caching is controlled by PWT and PCD bits directly +/// - Permissions are R/W (bit 1) and U/S (bit 2) +/// - Execute control is NX (bit 63), opt-out rather than opt-in +fn encode_attributes(attr: AttributeFields) -> u64 { + let mut bits: u64 = 0; + + // Accessed and Dirty — pre-set to avoid faults on first access. + // The kernel can clear these for tracking if needed. + bits |= ACCESSED | DIRTY; + + // Memory type via PWT/PCD + match attr.mem_attributes { + MemAttributes::CacheableDRAM => { + // PWT=0, PCD=0: write-back cacheable + } + MemAttributes::NonCacheableDRAM => { + // PWT=1, PCD=1: uncacheable (UC) + // For true WC you'd use PAT, but UC is the safe no_std default. + bits |= WRITE_THROUGH | CACHE_DISABLE; + } + MemAttributes::Device => { + // PWT=0, PCD=1: uncacheable, strong ordering + bits |= CACHE_DISABLE; + } + } + + // Access permissions — x86_64 is permissive by default (read always allowed). + // R/W bit: 0 = read-only, 1 = read-write + if let AccessPermissions::ReadWrite = attr.acc_perms { + bits |= WRITABLE; + } + + // Execute-never via NX bit + if !attr.executable { + bits |= NO_EXECUTE; + } + + // Supervisor-only for now (U/S = 0) + // When userspace is implemented, set USER_ACCESSIBLE based on context. + + bits +} diff --git a/libs/memory/src/arch_trait.rs b/libs/memory/src/arch_trait.rs new file mode 100644 index 000000000..9fe43c5f0 --- /dev/null +++ b/libs/memory/src/arch_trait.rs @@ -0,0 +1,142 @@ +use { + libaddress::{PhysAddr, VirtAddr}, + libmapping::AttributeFields, +}; + +/// What kinds of entries a given table level supports. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct LevelCapabilities { + /// This level can contain pointers to the next-level table. + pub supports_table_pointer: bool, + /// This level can contain block/page mappings to physical memory. + pub supports_block: bool, + /// If block mappings are supported, the block size in bytes. + /// For the leaf level (e.g. L3 with 4K pages), this is the page size. + pub block_size: usize, +} + +/// A decoded translation table entry. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum EntryKind { + /// Entry is invalid / not present. + Invalid, + /// Entry points to the next-level translation table at the given physical address. + Table(PhysAddr), + /// Entry maps a block (or page) of physical memory. + Block(PhysAddr), +} + +/// Architecture-specific translation table format. +/// +/// Implementations of this trait encode all the knowledge about a specific +/// MMU translation scheme: how many levels, how entries are encoded, what +/// block sizes each level supports, etc. +/// +/// The trait uses only associated functions (no `&self`) so implementations +/// are zero-sized type tags (e.g. `struct Aarch64_4K;`). +/// +/// ## Hierarchical vs Hashed Page Tables +/// +/// Most architectures (`AArch64`, `x86_64`, RISC-V) use hierarchical multi-level +/// page tables where each entry is a single u64. For these, the default +/// `ENTRY_WIDTH` of 1 applies and the standard encode/decode methods work +/// directly. +/// +/// Some architectures (PowerPC 970MP) use hashed page tables (HPT) with +/// wider entries (e.g. 16 bytes = 2 × u64). These set `ENTRY_WIDTH` to 2 +/// and override the `_wide` encode/decode methods that operate on `&[u64]` +/// slices. The single-u64 methods have default panicking implementations +/// for HPT architectures since they are not meaningful. +pub trait TranslationArch { + /// Number of levels in the translation hierarchy (e.g. 4 for `AArch64` 4K granule). + /// For hashed page tables, this is 1 (the hash table itself). + const NUM_LEVELS: usize; + + /// Number of u64 words per entry. Default is 1 for hierarchical page tables. + /// PowerPC HPT uses 2 (16-byte PTEs: `pte_hi` + `pte_lo`). + const ENTRY_WIDTH: usize = 1; + + /// Whether this architecture uses a hashed (non-hierarchical) page table. + /// When true, `index_from_vaddr` may not be meaningful — use + /// `hash_from_vaddr` instead. + const HASHED: bool = false; + + /// Number of entries in a table at the given level. + fn entries_per_table(level: usize) -> usize; + + /// Required alignment in bytes for a table at the given level. + fn table_alignment(level: usize) -> usize; + + /// Size in bytes of a table at the given level. + /// For `ENTRY_WIDTH=1`: entries * 8. For `ENTRY_WIDTH=2`: entries * 16. + fn table_size(level: usize) -> usize { + Self::entries_per_table(level) * Self::ENTRY_WIDTH * core::mem::size_of::() + } + + /// What kinds of entries this level supports. + fn level_capabilities(level: usize) -> LevelCapabilities; + + /// Extract the table index from a virtual address for a given level. + /// For hierarchical page tables, this extracts the VPN bits. + /// For hashed page tables, use `hash_from_vaddr` instead. + fn index_from_vaddr(vaddr: VirtAddr, level: usize) -> usize; + + // -- Single-u64 entry methods (hierarchical page tables) -- + + /// Decode a raw 64-bit entry at a given level into its semantic meaning. + /// Default implementation panics for hashed architectures. + fn decode_entry(raw: u64, level: usize) -> EntryKind; + + /// Encode a table pointer entry (pointing to the next-level table). + fn encode_table_entry(next_table_phys: PhysAddr, level: usize) -> u64; + + /// Encode a block mapping entry at levels that support blocks (L1 1G, L2 2M). + fn encode_block_entry(phys: PhysAddr, attr: AttributeFields, level: usize) -> u64; + + /// Encode a page mapping entry at the leaf level (L3 for 4K granule). + fn encode_page_entry(phys: PhysAddr, attr: AttributeFields) -> u64; + + /// Extract the output physical address from a raw entry at a given level. + fn output_address(raw: u64, level: usize) -> PhysAddr; + + // -- Wide entry methods (hashed page tables with ENTRY_WIDTH > 1) -- + + /// Decode a wide entry (multiple u64 words) at a given level. + /// The slice length must equal `ENTRY_WIDTH`. + /// Default implementation delegates to `decode_entry` for `ENTRY_WIDTH=1`. + fn decode_entry_wide(raw: &[u64], level: usize) -> EntryKind { + debug_assert_eq!(raw.len(), Self::ENTRY_WIDTH); + Self::decode_entry(raw[0], level) + } + + /// Encode a page mapping into a wide entry. + /// Returns the entry words. Default delegates to `encode_page_entry`. + fn encode_page_entry_wide(phys: PhysAddr, attr: AttributeFields, buf: &mut [u64]) { + debug_assert_eq!(buf.len(), Self::ENTRY_WIDTH); + buf[0] = Self::encode_page_entry(phys, attr); + } + + /// Extract the output physical address from a wide entry. + /// Default delegates to `output_address`. + fn output_address_wide(raw: &[u64], level: usize) -> PhysAddr { + debug_assert_eq!(raw.len(), Self::ENTRY_WIDTH); + Self::output_address(raw[0], level) + } + + // -- Hash-based lookup (hashed page tables) -- + + /// For hashed page tables: compute the primary hash index (PTEG index) + /// from a virtual address and the VSID (Virtual Segment ID). + /// + /// Returns the PTEG index within the hash table. + /// Default implementation returns 0 (unused for hierarchical tables). + fn hash_primary(_vaddr: VirtAddr, _vsid: u64, _htab_mask: u64) -> usize { + 0 + } + + /// For hashed page tables: compute the secondary hash index. + /// Default implementation returns 0 (unused for hierarchical tables). + fn hash_secondary(_primary_hash: usize, _htab_mask: u64) -> usize { + 0 + } +} diff --git a/libs/memory/src/error.rs b/libs/memory/src/error.rs new file mode 100644 index 000000000..cfd505d84 --- /dev/null +++ b/libs/memory/src/error.rs @@ -0,0 +1,27 @@ +use snafu::Snafu; + +/// Errors from translation table operations. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Snafu)] +pub enum TableError { + /// Entry index is out of bounds for this table level. + #[snafu(display("entry index out of bounds"))] + IndexOutOfBounds, + /// The provided memory slice has wrong size for the given table level. + #[snafu(display("memory slice has wrong size for table level"))] + InvalidTableSize, + /// The table level is not valid for this architecture. + #[snafu(display("invalid table level for this architecture"))] + InvalidLevel, + /// This table level does not support block mappings. + #[snafu(display("block mappings not supported at this level"))] + BlockNotSupported, + /// This table level does not support table pointers. + #[snafu(display("table pointers not supported at this level"))] + TablePointerNotSupported, + /// The provided physical address is not properly aligned for this entry type. + #[snafu(display("physical address not properly aligned"))] + MisalignedAddress, + /// Attempted to overwrite a valid entry without clearing it first. + #[snafu(display("entry is already valid"))] + EntryAlreadyValid, +} diff --git a/libs/memory/src/lib.rs b/libs/memory/src/lib.rs index cd3647e19..d78b0b66d 100644 --- a/libs/memory/src/lib.rs +++ b/libs/memory/src/lib.rs @@ -1,154 +1,39 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 -// -// Copyright (c) 2018-2022 Andre Richter - -//! Memory Management. +/* + * SPDX-License-Identifier: BlueOak-1.0.0 + * Copyright (c) Berkus Decker + */ + +//! Translation table management for the Vesper nanokernel. +//! +//! This library provides arch-independent abstractions for MMU translation +//! tables. Each table level is a first-class object, matching the kernel's +//! capability-based syscall API where `GlobalDirectory`, `PageDirectory`, Frame +//! etc. are separate capabilities. +//! +//! All table memory is externally provided — this library never allocates. #![no_std] -#![allow(dead_code)] // while refactoring -#![allow(incomplete_features)] -#![feature(generic_const_exprs)] // incomplete_features -#![feature(format_args_nl)] #![allow(internal_features)] -#![feature(allocator_api)] -#![feature(core_intrinsics)] -#![feature(step_trait)] - -use core::{ - fmt, - marker::PhantomData, - ops::{Add, Sub}, -}; - -pub mod arch; -pub mod mm; -pub mod mmu; -pub mod platform; - -pub mod phys_addr; // merge with Address? -pub mod virt_addr; // merge with Address? - -//-------------------------------------------------------------------------------------------------- -// Public Definitions -//-------------------------------------------------------------------------------------------------- - -/// Metadata trait for marking the type of an address. -pub trait AddressType: Copy + Clone + PartialOrd + PartialEq + Ord + Eq {} - -/// Zero-sized type to mark a physical address. -#[derive(Copy, Clone, Debug, PartialOrd, PartialEq, Ord, Eq)] -pub enum Physical {} - -/// Zero-sized type to mark a virtual address. -#[derive(Copy, Clone, Debug, PartialOrd, PartialEq, Ord, Eq)] -pub enum Virtual {} - -/// Generic address type. -#[derive(Copy, Clone, Debug, PartialOrd, PartialEq, Ord, Eq)] -pub struct Address { - value: usize, - _address_type: PhantomData ATYPE>, -} - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- - -impl AddressType for Physical {} -impl AddressType for Virtual {} - -impl Address { - /// Create an instance. - pub const fn new(value: usize) -> Self { - Self { - value, - _address_type: PhantomData, - } - } - - /// Convert to usize. - pub const fn as_usize(self) -> usize { - self.value - } - - /// Align down to page size. - #[must_use] - pub const fn align_down_page(self) -> Self { - let aligned = mm::align_down(self.value, platform::KernelGranule::SIZE); - - Self::new(aligned) - } - - /// Align up to page size. - #[must_use] - pub const fn align_up_page(self) -> Self { - let aligned = mm::align_up(self.value, platform::KernelGranule::SIZE); - - Self::new(aligned) - } - - /// Checks if the address is page aligned. - pub const fn is_page_aligned(&self) -> bool { - mm::is_aligned(self.value, platform::KernelGranule::SIZE) - } - - /// Return the address' offset into the corresponding page. - pub const fn offset_into_page(&self) -> usize { - self.value & platform::KernelGranule::MASK - } -} - -impl Add for Address { - type Output = Self; - - #[inline(always)] - fn add(self, rhs: usize) -> Self::Output { - match self.value.checked_add(rhs) { - None => panic!("Overflow on Address::add"), - Some(x) => Self::new(x), - } - } -} - -impl Sub> for Address { - type Output = Self; +#![feature(core_intrinsics)] // internal feature +#![feature(format_args_nl)] - #[inline(always)] - fn sub(self, rhs: Address) -> Self::Output { - match self.value.checked_sub(rhs.value) { - None => panic!("Overflow on Address::sub"), - Some(x) => Self::new(x), - } - } -} +pub mod arch_trait; +pub mod error; +pub mod table; +pub mod walk; -impl fmt::Display for Address { - // Don't expect to see physical addresses greater than 40 bit. - #[allow(clippy::cast_possible_truncation)] - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - let q3: u8 = ((self.value >> 32) & 0xff) as u8; - let q2: u16 = ((self.value >> 16) & 0xffff) as u16; - let q1: u16 = (self.value & 0xffff) as u16; +mod arch; - write!(f, "0x")?; - write!(f, "{q3:02x}_")?; - write!(f, "{q2:04x}_")?; - write!(f, "{q1:04x}") - } -} +// Re-export core types at crate root. +pub use { + arch_trait::{EntryKind, LevelCapabilities, TranslationArch}, + error::TableError, + table::{Table, TableRef}, + walk::{TranslationResult, translate, translate_hashed}, +}; -impl fmt::Display for Address { - #[allow(clippy::cast_possible_truncation)] - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - let q4: u16 = ((self.value >> 48) & 0xffff) as u16; - let q3: u16 = ((self.value >> 32) & 0xffff) as u16; - let q2: u16 = ((self.value >> 16) & 0xffff) as u16; - let q1: u16 = (self.value & 0xffff) as u16; +// Re-export arch implementations. +#[cfg(target_arch = "aarch64")] +pub use arch::aarch64::{Aarch64_4K, Aarch64_16K, Aarch64_64K, features, mmu}; - write!(f, "0x")?; - write!(f, "{q4:04x}_")?; - write!(f, "{q3:04x}_")?; - write!(f, "{q2:04x}_")?; - write!(f, "{q1:04x}") - } -} +pub use arch::{powerpc::PowerPC_970, riscv64::RiscV_Sv48, x86_64::X86_64_4K}; diff --git a/libs/memory/src/mmu/mod.rs b/libs/memory/src/mmu/mod.rs deleted file mode 100644 index c9c13ff72..000000000 --- a/libs/memory/src/mmu/mod.rs +++ /dev/null @@ -1,272 +0,0 @@ -use { - crate::{Address, Physical, Virtual, platform}, - core::num::NonZeroUsize, - liblog::warn, - snafu::Snafu, -}; - -#[cfg(target_arch = "aarch64")] -use crate::arch::aarch64::mmu as arch_mmu; - -mod mapping_record; -pub mod page_alloc; -pub mod translation_table; -mod types; - -pub use types::*; - -//-------------------------------------------------------------------------------------------------- -// Architectural Public Reexports -//-------------------------------------------------------------------------------------------------- -// pub use arch_mmu::mmu; - -//-------------------------------------------------------------------------------------------------- -// Public Definitions -//-------------------------------------------------------------------------------------------------- - -/// MMU enable errors variants. -// @todo rework error types -#[allow(missing_docs)] -#[derive(Debug, Snafu)] -pub enum MMUEnableError { - #[snafu(display("MMU is already enabled"))] - AlreadyEnabled, - #[snafu(display("{}", err))] - Other { err: &'static str }, -} - -/// Memory Management interfaces. -pub mod interface { - use super::*; - - /// MMU functions. - pub trait MMU { - /// Turns on the MMU for the first time and enables data and instruction caching. - /// - /// # Safety - /// - /// - Changes the hardware's global state. - unsafe fn enable_mmu_and_caching( - &self, - phys_tables_base_addr: Address, - ) -> Result<(), MMUEnableError>; - - /// Returns true if the MMU is enabled, false otherwise. - fn is_enabled(&self) -> bool; - - fn print_features(&self); // debug - } -} - -/// Describes the characteristics of a translation granule. -pub struct TranslationGranule; - -/// Describes properties of an address space. -pub struct AddressSpace; - -/// Intended to be implemented for [`AddressSpace`]. -pub trait AssociatedTranslationTable { - /// A translation table whose address range is: - /// - /// [`AS_SIZE` - 1, 0] - type TableStartFromBottom; -} - -//-------------------------------------------------------------------------------------------------- -// Private Code -//-------------------------------------------------------------------------------------------------- -use {interface::MMU, liblocking::interface::*, translation_table::interface::TranslationTable}; - -/// Query the platform for the reserved virtual addresses for MMIO remapping -/// and initialize the kernel's MMIO VA allocator with it. -fn kernel_init_mmio_va_allocator() { - let region = platform::memory::mmu::virt_mmio_remap_region(); - - page_alloc::kernel_mmio_va_allocator().lock(|allocator| allocator.init(region)); -} - -/// Map a region in the kernel's translation tables. -/// -/// No input checks done, input is passed through to the architectural implementation. -/// -/// # Safety -/// -/// - See `map_at()`. -/// - Does not prevent aliasing. -unsafe fn kernel_map_at_unchecked( - name: &'static str, - virt_region: &MemoryRegion, - phys_region: &MemoryRegion, - attr: AttributeFields, -) -> Result<(), &'static str> { - crate::platform::memory::mmu::kernel_translation_tables().write(|tables| - // SAFETY: Make a mistake and you're dead, gaijin! - unsafe { tables.map_at(virt_region, phys_region, attr) })?; - - if let Err(x) = mapping_record::kernel_add(name, virt_region, phys_region, attr) { - warn!("{x}"); - } - - Ok(()) -} - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- - -impl TranslationGranule { - /// The granule's size. - pub const SIZE: usize = Self::size_checked(); - - /// The granule's mask. - pub const MASK: usize = Self::SIZE - 1; - - /// The granule's shift, aka log2(size). - pub const SHIFT: usize = Self::SIZE.trailing_zeros() as usize; - - const fn size_checked() -> usize { - assert!(GRANULE_SIZE.is_power_of_two()); - - GRANULE_SIZE - } -} - -impl AddressSpace { - /// The address space size. - pub const SIZE: usize = Self::size_checked(); - - /// The address space shift, aka log2(size). - pub const SIZE_SHIFT: usize = Self::SIZE.trailing_zeros() as usize; - - const fn size_checked() -> usize { - assert!(AS_SIZE.is_power_of_two()); - - // Check for architectural restrictions as well. - Self::arch_address_space_size_sanity_checks(); - - AS_SIZE - } -} - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- - -/// Raw mapping of a virtual to physical region in the kernel translation tables. -/// -/// Prevents mapping into the MMIO range of the tables. -/// -/// # Safety -/// -/// - See `kernel_map_at_unchecked()`. -/// - Does not prevent aliasing. Currently, the callers must be trusted. -pub unsafe fn kernel_map_at( - name: &'static str, - virt_region: &MemoryRegion, - phys_region: &MemoryRegion, - attr: AttributeFields, -) -> Result<(), &'static str> { - if platform::memory::mmu::virt_mmio_remap_region().overlaps(virt_region) { - return Err("Attempt to manually map into MMIO region"); - } - - // SAFETY: Make a mistake and you're dead, gaijin! - unsafe { - kernel_map_at_unchecked(name, virt_region, phys_region, attr)?; - } - - Ok(()) -} - -/// MMIO remapping in the kernel translation tables. -/// -/// Typically used by device drivers. -/// -/// # Safety -/// -/// - Same as `kernel_map_at_unchecked()`, minus the aliasing part. -pub unsafe fn kernel_map_mmio( - name: &'static str, - mmio_descriptor: &MMIODescriptor, -) -> Result, &'static str> { - let phys_region = MemoryRegion::from(*mmio_descriptor); - let offset_into_start_page = mmio_descriptor.start_addr().offset_into_page(); - - // Check if an identical region has been mapped for another driver. If so, reuse it. - let virt_addr = if let Some(addr) = - mapping_record::kernel_find_and_insert_mmio_duplicate(mmio_descriptor, name) - { - addr - // Otherwise, allocate a new region and map it. - } else { - let Some(num_pages) = NonZeroUsize::new(phys_region.num_pages()) else { - return Err("Requested 0 pages"); - }; - - let virt_region = - page_alloc::kernel_mmio_va_allocator().lock(|allocator| allocator.alloc(num_pages))?; - - // SAFETY: Make a mistake and you're dead, gaijin! - unsafe { - kernel_map_at_unchecked( - name, - &virt_region, - &phys_region, - AttributeFields { - mem_attributes: MemAttributes::Device, - acc_perms: AccessPermissions::ReadWrite, - execute_never: true, - }, - )?; - } - - virt_region.start_addr() - }; - - Ok(virt_addr + offset_into_start_page) -} - -/// Map the kernel's binary. Returns the translation table's base address. -/// -/// # Safety -/// -/// - See [`bsp::memory::mmu::kernel_map_binary()`]. -pub unsafe fn kernel_map_binary() -> Result, &'static str> { - let phys_kernel_tables_base_addr = - platform::memory::mmu::kernel_translation_tables().write(|tables| { - tables.init().unwrap(); - tables.phys_base_address() - }); - - // SAFETY: Make a mistake and you're dead, gaijin! - unsafe { - platform::memory::mmu::kernel_map_binary()?; - } - - Ok(phys_kernel_tables_base_addr) -} - -/// Enable the MMU and data + instruction caching. -/// -/// # Safety -/// -/// - Crucial function during kernel init. Changes the the complete memory view of the processor. -#[inline] -pub unsafe fn enable_mmu_and_caching( - phys_tables_base_addr: Address, -) -> Result<(), MMUEnableError> { - // SAFETY: Make a mistake and you're dead, gaijin! - unsafe { arch_mmu::mmu().enable_mmu_and_caching(phys_tables_base_addr) } -} - -/// Finish initialization of the MMU subsystem. -#[inline] -pub fn post_enable_init() { - kernel_init_mmio_va_allocator(); -} - -/// Human-readable print of all recorded kernel mappings. -#[inline] -pub fn kernel_print_mappings() { - mapping_record::kernel_print(); -} diff --git a/libs/memory/src/mmu/page_alloc.rs b/libs/memory/src/mmu/page_alloc.rs deleted file mode 100644 index 4492cf7a0..000000000 --- a/libs/memory/src/mmu/page_alloc.rs +++ /dev/null @@ -1,76 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 -// -// Copyright (c) 2021-2022 Andre Richter - -//! Page allocation. - -use { - super::MemoryRegion, - crate::{AddressType, Virtual}, - core::num::NonZeroUsize, - liblocking::IRQSafeNullLock, - liblog::warn, -}; - -//-------------------------------------------------------------------------------------------------- -// Public Definitions -//-------------------------------------------------------------------------------------------------- - -/// A page allocator that can be lazyily initialized. -pub struct PageAllocator { - pool: Option>, -} - -//-------------------------------------------------------------------------------------------------- -// Global instances -//-------------------------------------------------------------------------------------------------- - -static KERNEL_MMIO_VA_ALLOCATOR: IRQSafeNullLock> = - IRQSafeNullLock::new(PageAllocator::new()); - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- - -/// Return a reference to the kernel's MMIO virtual address allocator. -pub fn kernel_mmio_va_allocator() -> &'static IRQSafeNullLock> { - &KERNEL_MMIO_VA_ALLOCATOR -} - -impl Default for PageAllocator { - fn default() -> Self { - Self::new() - } -} - -impl PageAllocator { - /// Create an instance. - pub const fn new() -> Self { - Self { pool: None } - } - - /// Initialize the allocator. - pub fn init(&mut self, pool: MemoryRegion) { - if self.pool.is_some() { - warn!("Already initialized"); - return; - } - - self.pool = Some(pool); - } - - /// Allocate a number of pages. - pub fn alloc( - &mut self, - num_requested_pages: NonZeroUsize, - ) -> Result, &'static str> { - if self.pool.is_none() { - return Err("Allocator not initialized"); - } - - self.pool - .as_mut() - .unwrap() - .take_first_n_pages(num_requested_pages) - } -} diff --git a/libs/memory/src/mmu/translation_table.rs b/libs/memory/src/mmu/translation_table.rs deleted file mode 100644 index dff6fb61b..000000000 --- a/libs/memory/src/mmu/translation_table.rs +++ /dev/null @@ -1,52 +0,0 @@ -//! Translation table. - -#[cfg(target_arch = "aarch64")] -use crate::arch::aarch64::mmu::translation_table as arch_translation_table; - -use { - super::{AttributeFields, MemoryRegion}, - crate::{Address, Physical, Virtual}, -}; - -//-------------------------------------------------------------------------------------------------- -// Architectural Public Reexports -//-------------------------------------------------------------------------------------------------- -#[cfg(target_arch = "aarch64")] -pub use arch_translation_table::FixedSizeTranslationTable; - -//-------------------------------------------------------------------------------------------------- -// Public Definitions -//-------------------------------------------------------------------------------------------------- - -/// Translation table interfaces. -pub mod interface { - use super::*; - - /// Translation table operations. - pub trait TranslationTable { - /// Anything that needs to run before any of the other provided functions can be used. - /// - /// - Implementor must ensure that this function can run only once or is harmless if invoked - /// multiple times. - fn init(&mut self) -> Result<(), &'static str>; - - /// The translation table's base address to be used for programming the MMU. - fn phys_base_address(&self) -> Address; - - /// Map the given virtual memory region to the given physical memory region. - /// - /// # Safety - /// - /// - Using wrong attributes can cause multiple issues of different nature in the system. - /// - It is not required that the architectural implementation prevents aliasing. That is, - /// mapping to the same physical memory using multiple virtual addresses, which would - /// break Rust's ownership assumptions. This should be protected against in the kernel's - /// generic MMU code. - unsafe fn map_at( - &mut self, - virt_region: &MemoryRegion, - phys_region: &MemoryRegion, - attr: AttributeFields, - ) -> Result<(), &'static str>; - } -} diff --git a/libs/memory/src/mmu/types.rs b/libs/memory/src/mmu/types.rs deleted file mode 100644 index d9d48516d..000000000 --- a/libs/memory/src/mmu/types.rs +++ /dev/null @@ -1,327 +0,0 @@ -//-------------------------------------------------------------------------------------------------- -// Public Definitions -//-------------------------------------------------------------------------------------------------- - -use { - crate::{Address, AddressType, Physical, mm, platform::KernelGranule}, - core::{ - fmt::{self, Formatter}, - iter::Step, - num::NonZeroUsize, - ops::Range, - }, -}; - -/// A wrapper type around [Address] that ensures page alignment. -#[derive(Copy, Clone, Debug, Eq, PartialOrd, PartialEq)] -pub struct PageAddress { - inner: Address, -} - -/// A type that describes a region of memory in quantities of pages. -#[derive(Copy, Clone, Debug, Eq, PartialOrd, PartialEq)] -pub struct MemoryRegion { - start: PageAddress, - end_exclusive: PageAddress, -} - -/// Architecture agnostic memory attributes. -#[derive(Copy, Clone, Debug, Eq, PartialOrd, PartialEq)] -pub enum MemAttributes { - /// Regular memory - CacheableDRAM, - /// Memory without caching - NonCacheableDRAM, - /// Device memory - Device, -} - -/// Architecture agnostic memory region access permissions. -#[derive(Copy, Clone, Debug, Eq, PartialOrd, PartialEq)] -pub enum AccessPermissions { - /// Read-only access - ReadOnly, - /// Read-write access - ReadWrite, -} - -/// Summary structure of memory region properties. -#[derive(Copy, Clone, Debug, Eq, PartialOrd, PartialEq)] -pub struct AttributeFields { - /// Attributes - pub mem_attributes: MemAttributes, - /// Permissions - pub acc_perms: AccessPermissions, - /// Disable executable code in this region - pub execute_never: bool, -} - -/// An MMIO descriptor for use in device drivers. -#[derive(Copy, Clone)] -pub struct MMIODescriptor { - start_addr: Address, - end_addr_exclusive: Address, -} - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- - -//------------------------------------------------------------------------------ -// PageAddress -//------------------------------------------------------------------------------ -impl PageAddress { - /// Unwraps the value. - pub fn into_inner(self) -> Address { - self.inner - } - - /// Calculates the offset from the page address. - /// - /// `count` is in units of [`PageAddress`]. For example, a count of 2 means `result = self + 2 * - /// page_size`. - pub fn checked_offset(self, count: isize) -> Option { - if count == 0 { - return Some(self); - } - - let delta = count.unsigned_abs().checked_mul(KernelGranule::SIZE)?; - let result = if count.is_positive() { - self.inner.as_usize().checked_add(delta)? - } else { - self.inner.as_usize().checked_sub(delta)? - }; - - Some(Self { - inner: Address::new(result), - }) - } -} - -impl From for PageAddress { - fn from(addr: usize) -> Self { - assert!( - mm::is_aligned(addr, KernelGranule::SIZE), - "Input usize not page aligned" - ); - - Self { - inner: Address::new(addr), - } - } -} - -impl From> for PageAddress { - fn from(addr: Address) -> Self { - assert!(addr.is_page_aligned(), "Input Address not page aligned"); - - Self { inner: addr } - } -} - -impl Step for PageAddress { - fn steps_between(start: &Self, end: &Self) -> (usize, Option) { - if start > end { - return (0, None); - } - - // Since start <= end, do unchecked arithmetic. - let steps = (end.inner.as_usize() - start.inner.as_usize()) >> KernelGranule::SHIFT; - (steps, Some(steps)) - } - - fn forward_checked(start: Self, count: usize) -> Option { - start.checked_offset(count.cast_signed()) - } - - fn backward_checked(start: Self, count: usize) -> Option { - start.checked_offset(-(count.cast_signed())) - } -} - -//------------------------------------------------------------------------------ -// MemoryRegion -//------------------------------------------------------------------------------ -impl MemoryRegion { - /// Create an instance. - pub fn new(start: PageAddress, end_exclusive: PageAddress) -> Self { - assert!(start <= end_exclusive); - - Self { - start, - end_exclusive, - } - } - - fn as_range(&self) -> Range> { - self.into_iter() - } - - /// Returns the start page address. - pub fn start_page_addr(&self) -> PageAddress { - self.start - } - - /// Returns the start address. - pub fn start_addr(&self) -> Address { - self.start.into_inner() - } - - /// Returns the exclusive end page address. - pub fn end_exclusive_page_addr(&self) -> PageAddress { - self.end_exclusive - } - - /// Returns the exclusive end page address. - pub fn end_inclusive_page_addr(&self) -> PageAddress { - self.end_exclusive.checked_offset(-1).unwrap() - } - - /// Checks if self contains an address. - pub fn contains(&self, addr: Address) -> bool { - let page_addr = PageAddress::from(addr.align_down_page()); - self.as_range().contains(&page_addr) - } - - /// Checks if there is an overlap with another memory region. - pub fn overlaps(&self, other_region: &Self) -> bool { - let self_range = self.as_range(); - - self_range.contains(&other_region.start_page_addr()) - || self_range.contains(&other_region.end_inclusive_page_addr()) - } - - /// Returns the number of pages contained in this region. - pub fn num_pages(&self) -> usize { - PageAddress::steps_between(&self.start, &self.end_exclusive).0 - } - - /// Returns the size in bytes of this region. - pub fn size(&self) -> usize { - // Invariant: start <= end_exclusive, so do unchecked arithmetic. - let end_exclusive = self.end_exclusive.into_inner().as_usize(); - let start = self.start.into_inner().as_usize(); - - end_exclusive - start - } - - /// Splits the `MemoryRegion` like: - /// - /// -------------------------------------------------------------------------------- - /// | | | | | | | | | | | | | | | | | | | - /// -------------------------------------------------------------------------------- - /// ^ ^ ^ - /// | | | - /// `left_start` `left_end_exclusive` | - /// | - /// ^ | - /// | | - /// `right_start` `right_end_exclusive` - /// - /// Left region is returned to the caller. Right region is the new region for this struct. - pub fn take_first_n_pages(&mut self, num_pages: NonZeroUsize) -> Result { - let count: usize = num_pages.into(); - - let left_end_exclusive = self.start.checked_offset(count.cast_signed()); - let Some(left_end_exclusive) = left_end_exclusive else { - return Err("Overflow while calculating left_end_exclusive"); - }; - - if left_end_exclusive > self.end_exclusive { - return Err("Not enough free pages"); - } - - let allocation = Self { - start: self.start, - end_exclusive: left_end_exclusive, - }; - self.start = left_end_exclusive; - - Ok(allocation) - } -} - -impl IntoIterator for MemoryRegion { - type Item = PageAddress; - type IntoIter = Range; - - fn into_iter(self) -> Self::IntoIter { - Range { - start: self.start, - end: self.end_exclusive, - } - } -} - -impl From for MemoryRegion { - fn from(desc: MMIODescriptor) -> Self { - let start = PageAddress::from(desc.start_addr.align_down_page()); - let end_exclusive = PageAddress::from(desc.end_addr_exclusive().align_up_page()); - - Self { - start, - end_exclusive, - } - } -} - -//------------------------------------------------------------------------------ -// MMIODescriptor -//------------------------------------------------------------------------------ - -impl MMIODescriptor { - /// Create an instance. - pub const fn new(start_addr: Address, size: usize) -> Self { - assert!(size > 0); - let end_addr_exclusive = Address::new(start_addr.as_usize() + size); - - Self { - start_addr, - end_addr_exclusive, - } - } - - /// Return the start address. - pub const fn start_addr(&self) -> Address { - self.start_addr - } - - /// Return the exclusive end address. - pub fn end_addr_exclusive(&self) -> Address { - self.end_addr_exclusive - } -} - -//------------------------------------------------------------------------------ -// AttributeFields -//------------------------------------------------------------------------------ - -impl Default for AttributeFields { - fn default() -> AttributeFields { - AttributeFields { - mem_attributes: MemAttributes::CacheableDRAM, - acc_perms: AccessPermissions::ReadWrite, - execute_never: true, - } - } -} - -/// Human-readable output of `AttributeFields` -impl fmt::Display for AttributeFields { - fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { - let attr = match self.mem_attributes { - MemAttributes::CacheableDRAM => "C", - MemAttributes::NonCacheableDRAM => "NC", - MemAttributes::Device => "Dev", - }; - - let acc_p = match self.acc_perms { - AccessPermissions::ReadOnly => "RO", - AccessPermissions::ReadWrite => "RW", - }; - - let xn = if self.execute_never { "PXN" } else { "PX" }; - - write!(f, "{attr: <3} {acc_p} {xn: <3}") - } -} diff --git a/libs/memory/src/mmu/unused.rs b/libs/memory/src/mmu/unused.rs deleted file mode 100644 index 897c0796f..000000000 --- a/libs/memory/src/mmu/unused.rs +++ /dev/null @@ -1,124 +0,0 @@ -//-------------------------------------------------------------------------------------------------- -// Laterrrr -//-------------------------------------------------------------------------------------------------- - -/// Architecture agnostic memory region translation types. -#[allow(dead_code)] -#[derive(Copy, Clone)] -pub enum Translation { - /// One-to-one address mapping - Identity, - /// Mapping with a specified offset - Offset(usize), -} - -//-------------------------------------------------------------------------------------------------- -// Public Definitions -//-------------------------------------------------------------------------------------------------- - -/// Types used for compiling the virtual memory layout of the kernel using address ranges. -/// -/// Memory region descriptor. -/// -/// Used to construct iterable kernel memory ranges. -pub struct TranslationDescriptor { - /// Name of the region - pub name: &'static str, - /// Virtual memory range - pub virtual_range: fn() -> RangeInclusive, - /// Mapping translation - pub physical_range_translation: Translation, - /// Attributes - pub attribute_fields: AttributeFields, -} - -/// Type for expressing the kernel's virtual memory layout. -pub struct KernelVirtualLayout { - /// The last (inclusive) address of the address space. - max_virt_addr_inclusive: usize, - - /// Array of descriptors for non-standard (normal cacheable DRAM) memory regions. - inner: [TranslationDescriptor; NUM_SPECIAL_RANGES], -} - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- - -/// Human-readable output of a Descriptor. -impl fmt::Display for TranslationDescriptor { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - // Call the function to which self.range points, and dereference the - // result, which causes Rust to copy the value. - let start = *(self.virtual_range)().start(); - let end = *(self.virtual_range)().end(); - let size = end - start + 1; - - // log2(1024) - const KIB_SHIFT: u32 = 10; - - // log2(1024 * 1024) - const MIB_SHIFT: u32 = 20; - - let (size, unit) = if (size >> MIB_SHIFT) > 0 { - (size >> MIB_SHIFT, "MiB") - } else if (size >> KIB_SHIFT) > 0 { - (size >> KIB_SHIFT, "KiB") - } else { - (size, "Byte") - }; - - write!( - f, - " {:#010x} - {:#010x} | {: >3} {} | {} | {}", - start, end, size, unit, self.attribute_fields, self.name - ) - } -} - -impl KernelVirtualLayout<{ NUM_SPECIAL_RANGES }> { - /// Create a new instance. - pub const fn new(max: usize, layout: [TranslationDescriptor; NUM_SPECIAL_RANGES]) -> Self { - Self { - max_virt_addr_inclusive: max, - inner: layout, - } - } - - /// For a given virtual address, find and return the output address and - /// corresponding attributes. - /// - /// If the address is not found in `inner`, return an identity mapped default for normal - /// cacheable DRAM. - pub fn virt_addr_properties( - &self, - virt_addr: usize, - ) -> Result<(usize, AttributeFields), &'static str> { - if virt_addr > self.max_virt_addr_inclusive { - return Err("Address out of range"); - } - - for i in self.inner.iter() { - if (i.virtual_range)().contains(&virt_addr) { - let output_addr = match i.physical_range_translation { - Translation::Identity => virt_addr, - Translation::Offset(a) => a + (virt_addr - (i.virtual_range)().start()), - }; - - return Ok((output_addr, i.attribute_fields)); - } - } - - Ok((virt_addr, AttributeFields::default())) - } - - /// Print the kernel memory layout. - pub fn print_layout(&self) { - println!("[i] Kernel memory layout:"); //info! - - for i in self.inner.iter() { - // for i in KERNEL_VIRTUAL_LAYOUT.iter() { - println!("{}", i); //info! - } - } -} diff --git a/libs/memory/src/phys_addr.rs b/libs/memory/src/phys_addr.rs deleted file mode 100644 index dd1021142..000000000 --- a/libs/memory/src/phys_addr.rs +++ /dev/null @@ -1,236 +0,0 @@ -/* - * SPDX-License-Identifier: BlueOak-1.0.0 - * Copyright (c) Berkus Decker - */ - -use { - crate::mm::{align_down, align_up}, - bit_field::BitField, - core::{ - convert::From, - fmt, - ops::{Add, AddAssign, Shl, Shr, Sub, SubAssign}, - }, - usize_conversions::FromUsize, -}; - -/// A 64-bit physical memory address. -/// -/// This is a wrapper type around an `u64`, so it is always 8 bytes, even when compiled -/// on non 64-bit systems. The `UsizeConversions` trait can be used for performing conversions -/// between `u64` and `usize`. -/// -/// On `aarch64`, only the 52 lower bits of a physical address can be used. The top 12 bits need -/// to be zero. This type guarantees that it always represents a valid physical address. -#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Default)] -#[repr(transparent)] -pub struct PhysAddr(u64); - -/// A passed `u64` was not a valid physical address. -/// -/// This means that bits 52 to 64 were not all null. -#[derive(Debug, PartialEq, Eq, PartialOrd, Ord)] -pub struct PhysAddrNotValid(pub u64); - -impl PhysAddr { - /// Creates a new physical address. - /// - /// Panics if any bits in the bit position 52 to 64 is set. - pub fn new(addr: u64) -> PhysAddr { - assert_eq!( - addr.get_bits(52..64), - 0, - "physical addresses must not have any set bits in positions 52 to 64" - ); - PhysAddr(addr) - } - - /// Tries to create a new physical address. - /// - /// Fails if any bits in the bit positions 52 to 64 are set. - pub fn try_new(addr: u64) -> Result { - match addr.get_bits(52..64) { - 0 => Ok(PhysAddr(addr)), // address is valid - _ => Err(PhysAddrNotValid(addr)), - } - } - - /// Creates a physical address that points to `0`. - pub const fn zero() -> PhysAddr { - PhysAddr(0) - } - - /// Converts the address to an `u64`. - pub fn as_u64(self) -> u64 { - self.0 - } - - /// Convenience method for checking if a physical address is null. - pub fn is_null(&self) -> bool { - self.0 == 0 - } - - /// Aligns the physical address upwards to the given alignment. - /// - /// See the `align_up` function for more information. - #[must_use] - pub fn aligned_up(self, align: usize) -> Self { - PhysAddr( - align_up(self.0.try_into().unwrap(), align) - .try_into() - .unwrap(), - ) - } - - /// Aligns the physical address downwards to the given alignment. - /// - /// See the `align_down` function for more information. - #[must_use] - pub fn aligned_down(self, align: usize) -> Self { - PhysAddr( - align_down(self.0.try_into().unwrap(), align) - .try_into() - .unwrap(), - ) - } - - /// Checks whether the physical address has the demanded alignment. - pub fn is_aligned(self, align: usize) -> bool { - self.aligned_down(align) == self - } -} - -impl fmt::Debug for PhysAddr { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!(f, "PhysAddr({:#x})", self.0) - } -} - -impl fmt::Binary for PhysAddr { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - self.0.fmt(f) - } -} - -impl fmt::LowerHex for PhysAddr { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - self.0.fmt(f) - } -} - -impl fmt::Octal for PhysAddr { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - self.0.fmt(f) - } -} - -impl fmt::UpperHex for PhysAddr { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - self.0.fmt(f) - } -} - -impl From for PhysAddr { - fn from(value: u64) -> Self { - PhysAddr::new(value) - } -} - -impl From for u64 { - fn from(value: PhysAddr) -> Self { - value.as_u64() - } -} - -impl From for u128 { - fn from(value: PhysAddr) -> Self { - u128::from(value.as_u64()) - } -} - -impl Add for PhysAddr { - type Output = Self; - fn add(self, rhs: u64) -> Self::Output { - PhysAddr::new(self.0 + rhs) - } -} - -impl AddAssign for PhysAddr { - fn add_assign(&mut self, rhs: u64) { - *self = *self + rhs; - } -} - -impl Add for PhysAddr -where - u64: FromUsize, -{ - type Output = Self; - fn add(self, rhs: usize) -> Self::Output { - self + u64::from_usize(rhs) - } -} - -impl AddAssign for PhysAddr -where - u64: FromUsize, -{ - fn add_assign(&mut self, rhs: usize) { - self.add_assign(u64::from_usize(rhs)); - } -} - -impl Sub for PhysAddr { - type Output = Self; - fn sub(self, rhs: u64) -> Self::Output { - PhysAddr::new(self.0.checked_sub(rhs).unwrap()) - } -} - -impl SubAssign for PhysAddr { - fn sub_assign(&mut self, rhs: u64) { - *self = *self - rhs; - } -} - -impl Sub for PhysAddr -where - u64: FromUsize, -{ - type Output = Self; - fn sub(self, rhs: usize) -> Self::Output { - self - u64::from_usize(rhs) - } -} - -impl SubAssign for PhysAddr -where - u64: FromUsize, -{ - fn sub_assign(&mut self, rhs: usize) { - self.sub_assign(u64::from_usize(rhs)); - } -} - -impl Sub for PhysAddr { - type Output = u64; - fn sub(self, rhs: PhysAddr) -> Self::Output { - self.as_u64().checked_sub(rhs.as_u64()).unwrap() - } -} - -impl Shr for PhysAddr { - type Output = PhysAddr; - - fn shr(self, shift: usize) -> Self::Output { - PhysAddr::new(self.0 >> shift) - } -} - -impl Shl for PhysAddr { - type Output = PhysAddr; - - fn shl(self, shift: usize) -> Self::Output { - PhysAddr::new(self.0 << shift) - } -} diff --git a/libs/memory/src/platform/README.md b/libs/memory/src/platform/README.md deleted file mode 100644 index 78eb5744e..000000000 --- a/libs/memory/src/platform/README.md +++ /dev/null @@ -1 +0,0 @@ -Temp, until libplatform is introduced. diff --git a/libs/memory/src/platform/mod.rs b/libs/memory/src/platform/mod.rs deleted file mode 100644 index ee8df103f..000000000 --- a/libs/memory/src/platform/mod.rs +++ /dev/null @@ -1,4 +0,0 @@ -pub type KernelGranule = crate::mmu::TranslationGranule<{ 64 * 1024 }>; - -pub mod raspberrypi; -pub use raspberrypi::*; diff --git a/libs/memory/src/platform/raspberrypi/memory/mmu.rs b/libs/memory/src/platform/raspberrypi/memory/mmu.rs deleted file mode 100644 index 06d4000cb..000000000 --- a/libs/memory/src/platform/raspberrypi/memory/mmu.rs +++ /dev/null @@ -1,287 +0,0 @@ -//! Platform memory management unit. - -use { - crate::{ - Physical, Virtual, - mmu::{ - self as generic_mmu, AccessPermissions, AddressSpace, AssociatedTranslationTable, - AttributeFields, MemAttributes, MemoryRegion, PageAddress, TranslationGranule, - }, - }, - liblocking::InitStateLock, -}; - -//-------------------------------------------------------------------------------------------------- -// Private Definitions -//-------------------------------------------------------------------------------------------------- - -type KernelTranslationTable = - ::TableStartFromBottom; - -//-------------------------------------------------------------------------------------------------- -// Public Definitions -//-------------------------------------------------------------------------------------------------- - -/// The translation granule chosen by this platform. This will be used everywhere else -/// in the kernel to derive respective data structures and their sizes. -/// For example, the `crate::memory::mmu::Page`. -pub type KernelGranule = TranslationGranule<{ 64 * 1024 }>; - -/// The kernel's virtual address space defined by this platform. -pub type KernelVirtAddrSpace = AddressSpace<{ 1024 * 1024 * 1024 }>; - -//-------------------------------------------------------------------------------------------------- -// Global instances -//-------------------------------------------------------------------------------------------------- - -/// The kernel translation tables. -/// -/// It is mandatory that `InitStateLock` is transparent. -/// That is, `size_of(InitStateLock) == size_of(KernelTranslationTable)`. -/// There is a unit tests that checks this property. -pub static KERNEL_TABLES: InitStateLock = - InitStateLock::new(KernelTranslationTable::new()); - -//-------------------------------------------------------------------------------------------------- -// Private Code -//-------------------------------------------------------------------------------------------------- - -/// Helper function for calculating the number of pages the given parameter spans. -const fn size_to_num_pages(size: usize) -> usize { - assert!(size > 0); - assert!(size.is_multiple_of(KernelGranule::SIZE)); // assert! is const-fn-friendly - - size >> KernelGranule::SHIFT -} - -/// The code pages of the kernel binary. -pub fn virt_code_region() -> MemoryRegion { - let num_pages = size_to_num_pages(super::code_size()); - - let start_page_addr = super::virt_code_start(); - let end_exclusive_page_addr = start_page_addr - .checked_offset(num_pages.cast_signed()) - .unwrap(); - - MemoryRegion::new(start_page_addr, end_exclusive_page_addr) -} - -/// The data pages of the kernel binary. -pub fn virt_data_region() -> MemoryRegion { - let num_pages = size_to_num_pages(super::data_size()); - - let start_page_addr = super::virt_data_start(); - let end_exclusive_page_addr = start_page_addr - .checked_offset(num_pages.cast_signed()) - .unwrap(); - - MemoryRegion::new(start_page_addr, end_exclusive_page_addr) -} - -/// The boot core stack pages. -pub fn virt_boot_core_stack_region() -> MemoryRegion { - let num_pages = size_to_num_pages(super::boot_core_stack_size()); - - let start_page_addr = super::virt_boot_core_stack_start(); - let end_exclusive_page_addr = start_page_addr - .checked_offset(num_pages.cast_signed()) - .unwrap(); - - MemoryRegion::new(start_page_addr, end_exclusive_page_addr) -} - -// The binary is still identity mapped, so use this trivial conversion function for mapping below. - -fn kernel_virt_to_phys_region(virt_region: MemoryRegion) -> MemoryRegion { - MemoryRegion::new( - PageAddress::from(virt_region.start_page_addr().into_inner().as_usize()), - PageAddress::from( - virt_region - .end_exclusive_page_addr() - .into_inner() - .as_usize(), - ), - ) -} - -//-------------------------------------------------------------------------------------------------- -// Subsumed by the kernel_map_binary() function -//-------------------------------------------------------------------------------------------------- - -// pub static LAYOUT: KernelVirtualLayout = KernelVirtualLayout::new( -// memory_map::END_INCLUSIVE, -// [ -// TranslationDescriptor { -// name: "Remapped Device MMIO", -// virtual_range: remapped_mmio_range_inclusive, -// physical_range_translation: Translation::Offset( -// memory_map::mmio::MMIO_BASE + 0x20_0000, -// ), -// attribute_fields: AttributeFields { -// mem_attributes: MemAttributes::Device, -// acc_perms: AccessPermissions::ReadWrite, -// execute_never: true, -// }, -// }, -// TranslationDescriptor { -// name: "Device MMIO", -// virtual_range: mmio_range_inclusive, -// physical_range_translation: Translation::Identity, -// attribute_fields: AttributeFields { -// mem_attributes: MemAttributes::Device, -// acc_perms: AccessPermissions::ReadWrite, -// execute_never: true, -// }, -// }, -// TranslationDescriptor { -// name: "DMA heap pool", -// virtual_range: dma_range_inclusive, -// physical_range_translation: Translation::Identity, -// attribute_fields: AttributeFields { -// mem_attributes: MemAttributes::NonCacheableDRAM, -// acc_perms: AccessPermissions::ReadWrite, -// execute_never: true, -// }, -// }, -// TranslationDescriptor { -// name: "Framebuffer area (static for now)", -// virtual_range: || { -// RangeInclusive::new( -// memory_map::phys::VIDEOMEM_BASE, -// memory_map::mmio::MMIO_BASE - 1, -// ) -// }, -// physical_range_translation: Translation::Identity, -// attribute_fields: AttributeFields { -// mem_attributes: MemAttributes::Device, -// acc_perms: AccessPermissions::ReadWrite, -// execute_never: true, -// }, -// }, -// ], -// ); - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- - -/// Return a reference to the kernel's translation tables. -pub fn kernel_translation_tables() -> &'static InitStateLock { - &KERNEL_TABLES -} - -/// The MMIO remap pages. -pub fn virt_mmio_remap_region() -> MemoryRegion { - let num_pages = size_to_num_pages(super::mmio_remap_size()); - - let start_page_addr = super::virt_mmio_remap_start(); - let end_exclusive_page_addr = start_page_addr - .checked_offset(num_pages.cast_signed()) - .unwrap(); - - MemoryRegion::new(start_page_addr, end_exclusive_page_addr) -} - -/// Map the kernel binary. -/// -/// # Safety -/// -/// - Any miscalculation or attribute error will likely be fatal. Needs careful manual checking. -pub unsafe fn kernel_map_binary() -> Result<(), &'static str> { - // SAFETY: Make a mistake and you're dead, gaijin! - unsafe { - generic_mmu::kernel_map_at( - "Kernel boot-core stack", - &virt_boot_core_stack_region(), - &kernel_virt_to_phys_region(virt_boot_core_stack_region()), - AttributeFields { - mem_attributes: MemAttributes::CacheableDRAM, - acc_perms: AccessPermissions::ReadWrite, - execute_never: true, - }, - )?; - } - - // TranslationDescriptor { - // name: "Boot code and data", - // virtual_range: boot_range_inclusive, - // physical_range_translation: Translation::Identity, - // attribute_fields: AttributeFields { - // mem_attributes: MemAttributes::CacheableDRAM, - // acc_perms: AccessPermissions::ReadOnly, - // execute_never: false, - // }, - // }, - - // TranslationDescriptor { - // name: "Kernel code and RO data", - // virtual_range: code_range_inclusive, - // physical_range_translation: Translation::Identity, - // attribute_fields: AttributeFields { - // mem_attributes: MemAttributes::CacheableDRAM, - // acc_perms: AccessPermissions::ReadOnly, - // execute_never: false, - // }, - // }, - - // SAFETY: Make a mistake and you're dead, gaijin! - unsafe { - generic_mmu::kernel_map_at( - "Kernel code and RO data", - &virt_code_region(), - &kernel_virt_to_phys_region(virt_code_region()), - AttributeFields { - mem_attributes: MemAttributes::CacheableDRAM, - acc_perms: AccessPermissions::ReadOnly, - execute_never: false, - }, - )?; - } - - // SAFETY: Make a mistake and you're dead, gaijin! - unsafe { - generic_mmu::kernel_map_at( - "Kernel data and bss", - &virt_data_region(), - &kernel_virt_to_phys_region(virt_data_region()), - AttributeFields { - mem_attributes: MemAttributes::CacheableDRAM, - acc_perms: AccessPermissions::ReadWrite, - execute_never: true, - }, - )?; - } - - Ok(()) -} - -//-------------------------------------------------------------------------------------------------- -// Private Code -//-------------------------------------------------------------------------------------------------- - -// fn boot_range_inclusive() -> RangeInclusive { -// RangeInclusive::new(super::boot_start(), super::boot_end_exclusive() - 1) -// } -// -// fn code_range_inclusive() -> RangeInclusive { -// // Notice the subtraction to turn the exclusive end into an inclusive end. -// #[allow(clippy::range_minus_one)] -// RangeInclusive::new(super::code_start(), super::code_end_exclusive() - 1) -// } -// -// fn remapped_mmio_range_inclusive() -> RangeInclusive { -// // The last 64 KiB slot in the first 512 MiB -// RangeInclusive::new(0x1FFF_0000, 0x1FFF_FFFF) -// } -// -// fn mmio_range_inclusive() -> RangeInclusive { -// RangeInclusive::new(memory_map::mmio::MMIO_BASE, memory_map::mmio::MMIO_END) -// // RangeInclusive::new(map::phys::VIDEOMEM_BASE, map::mmio::MMIO_END), -// } -// -// fn dma_range_inclusive() -> RangeInclusive { -// RangeInclusive::new( -// memory_map::virt::DMA_HEAP_START, -// memory_map::virt::DMA_HEAP_END, -// ) -// } diff --git a/libs/memory/src/platform/raspberrypi/memory/mod.rs b/libs/memory/src/platform/raspberrypi/memory/mod.rs deleted file mode 100644 index 77900c6f6..000000000 --- a/libs/memory/src/platform/raspberrypi/memory/mod.rs +++ /dev/null @@ -1,369 +0,0 @@ -//! Platform memory Management. -//! -//! The physical memory layout. -//! -//! The Raspberry's firmware copies the kernel binary to `0x8_0000`. The preceding region will be used -//! as the boot core's stack. -//! -//! +---------------------------------------+ -//! | | `boot_core_stack_start` @ 0x0 -//! | | ^ -//! | Boot-core Stack | | stack -//! | | | growth -//! | | | direction -//! +---------------------------------------+ -//! | | `code_start` @ `0x8_0000` == `boot_core_stack_end_exclusive` -//! | .text | -//! | .rodata | -//! | .got | -//! | | -//! +---------------------------------------+ -//! | | `data_start` == `code_end_exclusive` -//! | .data | -//! | .bss | -//! | | -//! +---------------------------------------+ -//! | | `data_end_exclusive` -//! | | -//! -//! -//! -//! -//! -//! The virtual memory layout is as follows: -//! -//! +---------------------------------------+ -//! | | `boot_core_stack_start` @ 0x0 -//! | | ^ -//! | Boot-core Stack | | stack -//! | | | growth -//! | | | direction -//! +---------------------------------------+ -//! | | `code_start` @ `0x8_0000` == `boot_core_stack_end_exclusive` -//! | .text | -//! | .rodata | -//! | .got | -//! | | -//! +---------------------------------------+ -//! | | `data_start` == `code_end_exclusive` -//! | .data | -//! | .bss | -//! | | -//! +---------------------------------------+ -//! | | `mmio_remap_start` == `data_end_exclusive` -//! | VA region for MMIO remapping | -//! | | -//! +---------------------------------------+ -//! | | `mmio_remap_end_exclusive` -//! | | -pub mod mmu; - -//-------------------------------------------------------------------------------------------------- -// Private Definitions -//-------------------------------------------------------------------------------------------------- - -use { - crate::{Address, Physical, Virtual, mmu::PageAddress}, - core::cell::UnsafeCell, -}; - -// Symbols from the linker script. -unsafe extern "Rust" { - // Boot code. - // - // Using the linker script, we ensure that the boot area is consecutive and 4 - // KiB aligned, and we export the boundaries via symbols: - // - // [__BOOT_START, __BOOT_END) - // - // The inclusive start of the boot area, aka the address of the - // first byte of the area. - static __BOOT_START: UnsafeCell<()>; - - // The exclusive end of the boot area, aka the address of - // the first byte _after_ the RO area. - static __BOOT_END: UnsafeCell<()>; - - // Kernel code and RO data. - // - // Using the linker script, we ensure that the RO area is consecutive and 4 - // KiB aligned, and we export the boundaries via symbols: - // - // [__RO_START, __RO_END) - // - // The inclusive start of the read-only area, aka the address of the - // first byte of the area. - static __CODE_START: UnsafeCell<()>; - // The exclusive end of the read-only area, aka the address of - // the first byte _after_ the RO area. - static __CODE_END: UnsafeCell<()>; - - // The inclusive start of the kernel data/BSS area, aka the address of the - // first byte of the area. - static __DATA_START: UnsafeCell<()>; - // The exclusive end of the kernel data/BSS area, aka the address of - // the first byte _after_ the data/BSS area. - static __DATA_END: UnsafeCell<()>; - - // The inclusive start of the kernel data/BSS area, aka the address of the - // first byte of the area. - static __STACK_BOTTOM: UnsafeCell<()>; - // The exclusive end of the kernel data/BSS area, aka the address of - // the first byte _after_ the data/BSS area. - static __STACK_TOP: UnsafeCell<()>; - - // The inclusive start of the kernel MMIO remap area, aka the address of the - // first byte of the area. - static __MMIO_REMAP_START: UnsafeCell<()>; - // The exclusive end of the kernel MMIO remap area, aka the address of - // the first byte _after_ the MMIO remap area. - static __MMIO_REMAP_END: UnsafeCell<()>; -} - -//-------------------------------------------------------------------------------------------------- -// Public Definitions -//-------------------------------------------------------------------------------------------------- - -/// The board's physical memory map. -/// This is a fixed memory map for Raspberry Pi, -/// @todo we need to infer the memory map from the provided DTB instead. -#[rustfmt::skip] -pub mod map { - use super::*; - - /// Beginning of memory. - pub const START: usize = 0x0000_0000; - /// End of memory - 8Gb `RPi4` - pub const END_INCLUSIVE: usize = 0x1_FFFF_FFFF; - - /// Physical RAM addresses. - pub mod phys { - /// Base address of video (VC) memory. - pub const VIDEOMEM_BASE: usize = 0x3e00_0000; - } - - pub const VIDEOCORE_MBOX_OFFSET: usize = 0x0000_B880; - pub const POWER_OFFSET: usize = 0x0010_0000; - pub const GPIO_OFFSET: usize = 0x0020_0000; - pub const UART_OFFSET: usize = 0x0020_1000; - pub const MINIUART_OFFSET: usize = 0x0021_5000; - - /// Physical devices. - #[cfg(board_rpi3)] - pub mod mmio { - use super::*; - - /// Base address of MMIO register range. - pub const MMIO_BASE: usize = 0x3F00_0000; - - /// Interrupt controller - pub const PERIPHERAL_IC_BASE: Address = Address::new(MMIO_BASE + 0x0000_B200); - pub const PERIPHERAL_IC_SIZE: usize = 0x24; - - /// Base address of ARM<->VC mailbox area. - pub const VIDEOCORE_MBOX_BASE: Address = Address::new(MMIO_BASE + VIDEOCORE_MBOX_OFFSET); - - /// Board power control. - pub const POWER_BASE: Address = Address::new(MMIO_BASE + POWER_OFFSET); - - /// Base address of GPIO registers. - pub const GPIO_BASE: Address = Address::new(MMIO_BASE + GPIO_OFFSET); - pub const GPIO_SIZE: usize = 0xA0; - - pub const PL011_UART_BASE: Address = Address::new(MMIO_BASE + UART_OFFSET); - pub const PL011_UART_SIZE: usize = 0x48; - - /// Base address of `MiniUART`. - pub const MINI_UART_BASE: Address = Address::new(MMIO_BASE + MINIUART_OFFSET); - - /// End of MMIO memory region. - pub const END: Address = Address::new(0x4001_0000); - } - - /// Physical devices. - #[cfg(board_rpi4)] - pub mod mmio { - use super::*; - - /// Base address of MMIO register range. - pub const MMIO_BASE: usize = 0xFE00_0000; - - /// Base address of GPIO registers. - pub const GPIO_BASE: Address = Address::new(MMIO_BASE + GPIO_OFFSET); - pub const GPIO_SIZE: usize = 0xA0; - - /// Base address of regular UART. - pub const PL011_UART_BASE: Address = Address::new(MMIO_BASE + UART_OFFSET); - pub const PL011_UART_SIZE: usize = 0x48; - - /// Base address of `MiniUART`. - pub const MINI_UART_BASE: Address = Address::new(MMIO_BASE + MINIUART_OFFSET); - - /// Interrupt controller - pub const GICD_BASE: Address = Address::new(0xFF84_1000); - pub const GICD_SIZE: usize = 0x824; - - pub const GICC_BASE: Address = Address::new(0xFF84_2000); - pub const GICC_SIZE: usize = 0x14; - - /// Base address of ARM<->VC mailbox area. - pub const VIDEOCORE_MBOX_BASE: usize = MMIO_BASE + VIDEOCORE_MBOX_OFFSET; - - /// End of MMIO memory region. - pub const END: Address = Address::new(0xFF85_0000); - } - - #[cfg(not(any(board_rpi3, board_rpi4)))] - compile_error!("No platform selected - specify TARGET_BOARD in configuration"); - - /// End address of mapped memory. - pub const END: Address = mmio::END; - - //---- - // Unused? - //---- - - /// Virtual (mapped) addresses. - pub mod virt { - /// Start (top) of kernel stack. - pub const KERN_STACK_START: usize = super::START; - /// End (bottom) of kernel stack. SP starts at `KERN_STACK_END` + 1. - pub const KERN_STACK_END: usize = 0x0007_FFFF; - - /// Location of DMA-able memory region (in the second 2 MiB block). - pub const DMA_HEAP_START: usize = 0x0020_0000; - /// End of DMA-able memory region. - pub const DMA_HEAP_END: usize = 0x005F_FFFF; - } -} - -//-------------------------------------------------------------------------------------------------- -// Private Code -//-------------------------------------------------------------------------------------------------- - -/// Start page address of the boot segment. -/// -/// # Safety -/// -/// - Value is provided by the linker script and must be trusted as-is. -#[inline(always)] -fn boot_start() -> usize { - // SAFETY: The linker script ensures the boot code section has a sensible start address. - unsafe { __BOOT_START.get() as usize } -} - -/// Exclusive end page address of the boot segment. -/// # Safety -/// -/// - Value is provided by the linker script and must be trusted as-is. -#[inline(always)] -fn boot_end_exclusive() -> usize { - // SAFETY: The linker script ensures the boot code section has a sensible end address. - unsafe { __BOOT_END.get() as usize } -} - -/// Start page address of the code segment. -/// -/// # Safety -/// -/// - Value is provided by the linker script and must be trusted as-is. -#[inline(always)] -fn code_start() -> usize { - // SAFETY: The linker script ensures the code section has a sensible start address. - unsafe { __CODE_START.get() as usize } -} - -/// Start page address of the code segment. -/// -/// # Safety -/// -/// - Value is provided by the linker script and must be trusted as-is. -#[inline(always)] -fn virt_code_start() -> PageAddress { - // SAFETY: The linker script ensures the code section has a sensible start address. - PageAddress::from(unsafe { __CODE_START.get() as usize }) -} - -/// Size of the code segment. -/// -/// # Safety -/// -/// - Value is provided by the linker script and must be trusted as-is. -#[inline(always)] -fn code_size() -> usize { - // SAFETY: The linker script ensures the code section has a sensible size. - unsafe { (__CODE_END.get() as usize) - (__CODE_START.get() as usize) } -} - -// Exclusive end page address of the code segment. -// # Safety -// -// - Value is provided by the linker script and must be trusted as-is. -// #[inline(always)] -// fn code_end_exclusive() -> usize { -// unsafe { __RO_END.get() as usize } -// } - -/// Start page address of the data segment. -#[inline(always)] -fn virt_data_start() -> PageAddress { - // SAFETY: The linker script ensures the data section has a sensible start address. - PageAddress::from(unsafe { __DATA_START.get() as usize }) -} - -/// Size of the data segment. -/// -/// # Safety -/// -/// - Value is provided by the linker script and must be trusted as-is. -#[inline(always)] -fn data_size() -> usize { - // SAFETY: The linker script ensures the data section has a sensible size. - unsafe { (__DATA_END.get() as usize) - (__DATA_START.get() as usize) } -} - -/// Start page address of the MMIO remap reservation. -/// -/// # Safety -/// -/// - Value is provided by the linker script and must be trusted as-is. -#[inline(always)] -fn virt_mmio_remap_start() -> PageAddress { - // SAFETY: The linker script ensures the MMIO area has a sensible start address. - PageAddress::from(unsafe { __MMIO_REMAP_START.get() as usize }) -} - -/// Size of the MMIO remap reservation. -/// -/// # Safety -/// -/// - Value is provided by the linker script and must be trusted as-is. -#[inline(always)] -fn mmio_remap_size() -> usize { - // SAFETY: The linker script ensures the MMIO area has a sensible size. - unsafe { (__MMIO_REMAP_END.get() as usize) - (__MMIO_REMAP_START.get() as usize) } -} - -/// Start page address of the boot core's stack. -#[inline(always)] -fn virt_boot_core_stack_start() -> PageAddress { - // SAFETY: The linker script ensures the stack bottom has a sensible address. - PageAddress::from(unsafe { __STACK_BOTTOM.get() as usize }) -} - -/// Size of the boot core's stack. -#[inline(always)] -fn boot_core_stack_size() -> usize { - // SAFETY: The linker script ensures the stack has a sensible size. - unsafe { (__STACK_TOP.get() as usize) - (__STACK_BOTTOM.get() as usize) } -} - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- - -/// Exclusive end address of the physical address space. -#[inline(always)] -pub fn phys_addr_space_end_exclusive_addr() -> PageAddress { - PageAddress::from(map::END) -} diff --git a/libs/memory/src/platform/raspberrypi/mod.rs b/libs/memory/src/platform/raspberrypi/mod.rs deleted file mode 100644 index eb291915f..000000000 --- a/libs/memory/src/platform/raspberrypi/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod memory; diff --git a/libs/memory/src/table.rs b/libs/memory/src/table.rs new file mode 100644 index 000000000..574aae644 --- /dev/null +++ b/libs/memory/src/table.rs @@ -0,0 +1,293 @@ +use { + crate::{ + arch_trait::{EntryKind, LevelCapabilities, TranslationArch}, + error::TableError, + }, + core::marker::PhantomData, + libaddress::PhysAddr, + libmapping::AttributeFields, +}; + +/// A single translation table at a specific level. +/// +/// Wraps an externally-provided `&mut [u64]` slice — memory is never +/// allocated internally. The kernel provides table memory through its +/// capability system. +/// +/// The level is stored as a runtime value so that all table capabilities +/// can be stored uniformly in the capability system without needing +/// separate types per level. +/// +/// For architectures with `ENTRY_WIDTH > 1` (e.g. PowerPC HPT with 16-byte +/// PTEs), the slice contains `entries_per_table * ENTRY_WIDTH` u64 words +/// and entry access uses sub-slices of width `ENTRY_WIDTH`. +pub struct Table<'a, A: TranslationArch> { + entries: &'a mut [u64], + level: usize, + _arch: PhantomData, +} + +/// An immutable view of a translation table. +pub struct TableRef<'a, A: TranslationArch> { + entries: &'a [u64], + level: usize, + _arch: PhantomData, +} + +impl<'a, A: TranslationArch> Table<'a, A> { + /// Create a new table from a zeroed memory region. + /// + /// The caller must ensure: + /// - `memory` points to a correctly-sized, correctly-aligned region + /// - `memory` is zeroed + /// - `level` is valid for the architecture (`0..NUM_LEVELS`) + pub fn from_memory(memory: &'a mut [u64], level: usize) -> Result { + if level >= A::NUM_LEVELS { + return Err(TableError::InvalidLevel); + } + let expected = A::entries_per_table(level) * A::ENTRY_WIDTH; + if memory.len() != expected { + return Err(TableError::InvalidTableSize); + } + Ok(Self { + entries: memory, + level, + _arch: PhantomData, + }) + } + + /// Wrap an existing populated table. + /// + /// Same requirements as `from_memory` except the memory need not be zeroed. + pub fn from_existing(memory: &'a mut [u64], level: usize) -> Result { + Self::from_memory(memory, level) + } + + /// The table level (0 = root, NUM_LEVELS-1 = leaf). + pub fn level(&self) -> usize { + self.level + } + + /// Number of entries in this table. + pub fn num_entries(&self) -> usize { + self.entries.len() / A::ENTRY_WIDTH + } + + /// What this level supports. + pub fn capabilities(&self) -> LevelCapabilities { + A::level_capabilities(self.level) + } + + /// Read the raw u64 value at the given index. + /// For architectures with `ENTRY_WIDTH > 1`, returns only the first word. + /// Use `read_raw_wide` for the full entry. + pub fn read_raw(&self, index: usize) -> Result { + let offset = index * A::ENTRY_WIDTH; + self.entries + .get(offset) + .copied() + .ok_or(TableError::IndexOutOfBounds) + } + + /// Read the raw u64 words for the entry at the given index. + /// Returns a slice of `ENTRY_WIDTH` words. + pub fn read_raw_wide(&self, index: usize) -> Result<&[u64], TableError> { + let offset = index * A::ENTRY_WIDTH; + self.entries + .get(offset..offset + A::ENTRY_WIDTH) + .ok_or(TableError::IndexOutOfBounds) + } + + /// Read and decode the entry at the given index. + pub fn read_entry(&self, index: usize) -> Result { + let raw = self.read_raw_wide(index)?; + Ok(A::decode_entry_wide(raw, self.level)) + } + + /// Write a table pointer entry at the given index. + /// + /// The entry must currently be invalid (clear it first if overwriting). + pub fn set_table_entry( + &mut self, + index: usize, + next_table_phys: PhysAddr, + ) -> Result<(), TableError> { + if !A::level_capabilities(self.level).supports_table_pointer { + return Err(TableError::TablePointerNotSupported); + } + let offset = index * A::ENTRY_WIDTH; + let slot = self + .entries + .get(offset..offset + A::ENTRY_WIDTH) + .ok_or(TableError::IndexOutOfBounds)?; + if A::decode_entry_wide(slot, self.level) != EntryKind::Invalid { + return Err(TableError::EntryAlreadyValid); + } + let encoded = A::encode_table_entry(next_table_phys, self.level); + self.entries[offset] = encoded; + Ok(()) + } + + /// Write a block (or page at leaf level) mapping entry. + /// + /// The entry must currently be invalid. + pub fn set_block_entry( + &mut self, + index: usize, + phys: PhysAddr, + attr: AttributeFields, + ) -> Result<(), TableError> { + let caps = A::level_capabilities(self.level); + if !caps.supports_block { + return Err(TableError::BlockNotSupported); + } + let offset = index * A::ENTRY_WIDTH; + let slot = self + .entries + .get(offset..offset + A::ENTRY_WIDTH) + .ok_or(TableError::IndexOutOfBounds)?; + if A::decode_entry_wide(slot, self.level) != EntryKind::Invalid { + return Err(TableError::EntryAlreadyValid); + } + // Use page encoding at the leaf level, block encoding otherwise. + if self.level == A::NUM_LEVELS - 1 || A::ENTRY_WIDTH > 1 { + let buf = self + .entries + .get_mut(offset..offset + A::ENTRY_WIDTH) + .ok_or(TableError::IndexOutOfBounds)?; + A::encode_page_entry_wide(phys, attr, buf); + } else { + self.entries[offset] = A::encode_block_entry(phys, attr, self.level); + }; + Ok(()) + } + + /// Clear (invalidate) the entry at the given index. + pub fn clear_entry(&mut self, index: usize) -> Result<(), TableError> { + let offset = index * A::ENTRY_WIDTH; + let slot = self + .entries + .get_mut(offset..offset + A::ENTRY_WIDTH) + .ok_or(TableError::IndexOutOfBounds)?; + slot.fill(0); + Ok(()) + } + + /// Write a raw u64 value at the given index. + /// + /// This bypasses all validation — use only when you know exactly + /// what descriptor bits to set. For architectures with `ENTRY_WIDTH > 1`, + /// this only writes the first word; use `write_raw_wide` for all words. + /// + /// # Safety + /// The caller must ensure the raw value is a valid descriptor for this level. + pub unsafe fn write_raw(&mut self, index: usize, value: u64) -> Result<(), TableError> { + let offset = index * A::ENTRY_WIDTH; + let slot = self + .entries + .get_mut(offset) + .ok_or(TableError::IndexOutOfBounds)?; + *slot = value; + Ok(()) + } + + /// Write raw u64 words at the given index. + /// + /// # Safety + /// The caller must ensure the raw values form a valid descriptor for this level. + pub unsafe fn write_raw_wide( + &mut self, + index: usize, + values: &[u64], + ) -> Result<(), TableError> { + debug_assert_eq!(values.len(), A::ENTRY_WIDTH); + let offset = index * A::ENTRY_WIDTH; + let slot = self + .entries + .get_mut(offset..offset + A::ENTRY_WIDTH) + .ok_or(TableError::IndexOutOfBounds)?; + slot.copy_from_slice(values); + Ok(()) + } + + /// Iterate over all entries, yielding (index, decoded entry) pairs. + pub fn iter(&self) -> impl Iterator + '_ { + let level = self.level; + self.entries + .chunks_exact(A::ENTRY_WIDTH) + .enumerate() + .map(move |(i, chunk)| (i, A::decode_entry_wide(chunk, level))) + } + + /// Iterate over only valid (non-invalid) entries. + pub fn iter_valid(&self) -> impl Iterator + '_ { + self.iter().filter(|(_, kind)| *kind != EntryKind::Invalid) + } + + /// Get an immutable view of this table. + pub fn as_ref(&self) -> TableRef<'_, A> { + TableRef { + entries: self.entries, + level: self.level, + _arch: PhantomData, + } + } +} + +impl<'a, A: TranslationArch> TableRef<'a, A> { + /// Create a read-only view from a slice. + pub fn from_slice(entries: &'a [u64], level: usize) -> Result { + if level >= A::NUM_LEVELS { + return Err(TableError::InvalidLevel); + } + let expected = A::entries_per_table(level) * A::ENTRY_WIDTH; + if entries.len() != expected { + return Err(TableError::InvalidTableSize); + } + Ok(Self { + entries, + level, + _arch: PhantomData, + }) + } + + pub fn level(&self) -> usize { + self.level + } + + pub fn num_entries(&self) -> usize { + self.entries.len() / A::ENTRY_WIDTH + } + + pub fn read_raw(&self, index: usize) -> Result { + let offset = index * A::ENTRY_WIDTH; + self.entries + .get(offset) + .copied() + .ok_or(TableError::IndexOutOfBounds) + } + + pub fn read_raw_wide(&self, index: usize) -> Result<&[u64], TableError> { + let offset = index * A::ENTRY_WIDTH; + self.entries + .get(offset..offset + A::ENTRY_WIDTH) + .ok_or(TableError::IndexOutOfBounds) + } + + pub fn read_entry(&self, index: usize) -> Result { + let raw = self.read_raw_wide(index)?; + Ok(A::decode_entry_wide(raw, self.level)) + } + + pub fn iter(&self) -> impl Iterator + '_ { + let level = self.level; + self.entries + .chunks_exact(A::ENTRY_WIDTH) + .enumerate() + .map(move |(i, chunk)| (i, A::decode_entry_wide(chunk, level))) + } + + pub fn iter_valid(&self) -> impl Iterator + '_ { + self.iter().filter(|(_, kind)| *kind != EntryKind::Invalid) + } +} diff --git a/libs/memory/src/virt_addr.rs b/libs/memory/src/virt_addr.rs deleted file mode 100644 index efb40db70..000000000 --- a/libs/memory/src/virt_addr.rs +++ /dev/null @@ -1,294 +0,0 @@ -/* - * SPDX-License-Identifier: BlueOak-1.0.0 - * Copyright (c) Berkus Decker - */ - -use { - crate::mm::{align_down, align_up}, - bit_field::BitField, - core::{ - convert::{From, TryInto}, - fmt, - ops::{Add, AddAssign, Rem, RemAssign, Sub, SubAssign}, - }, - usize_conversions::{FromUsize, usize_from}, - ux::*, -}; - -/// A canonical 64-bit virtual memory address. -/// -/// This is a wrapper type around an `u64`, so it is always 8 bytes, even when compiled -/// on non 64-bit systems. The `UsizeConversions` trait can be used for performing conversions -/// between `u64` and `usize`. -/// -/// On `x86_64`, only the 48 lower bits of a virtual address can be used. The top 16 bits need -/// to be copies of bit 47, i.e. the most significant bit. Addresses that fulfil this criterium -/// are called “canonical”. This type guarantees that it always represents a canonical address. -#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Default)] -#[repr(transparent)] -pub struct VirtAddr(u64); - -/// A passed `u64` was not a valid virtual address. -/// -/// This means that bits 48 to 64 are not -/// a valid sign extension and are not null either. So automatic sign extension would have -/// overwritten possibly meaningful bits. This likely indicates a bug, for example an invalid -/// address calculation. -#[derive(Debug)] -pub struct VirtAddrNotValid(u64); - -impl VirtAddr { - /// Creates a new canonical virtual address. - /// - /// This function performs sign extension of bit 47 to make the address canonical. Panics - /// if the bits in the range 48 to 64 contain data (i.e. are not null and not a sign extension). - /// - /// @todo Support ASID byte in top bits of the address. - pub fn new(addr: u64) -> VirtAddr { - Self::try_new(addr).expect( - "address passed to VirtAddr::new must not contain any data \ - in bits 48 to 64", - ) - } - - /// Tries to create a new canonical virtual address. - /// - /// This function tries to performs sign extension of bit 47 to make the address canonical. - /// It succeeds if bits 48 to 64 are either a correct sign extension (i.e. copies of bit 47) - /// or all null. Else, an error is returned. - pub fn try_new(addr: u64) -> Result { - match addr.get_bits(47..64) { - 0 | 0x1ffff => Ok(VirtAddr(addr)), // address is canonical - 1 => Ok(VirtAddr::new_unchecked(addr)), // address needs sign extension - _ => Err(VirtAddrNotValid(addr)), - } - } - - /// Creates a new canonical virtual address without checks. - /// - /// This function performs sign extension of bit 47 to make the address canonical, so - /// bits 48 to 64 are overwritten. If you want to check that these bits contain no data, - /// use `new` or `try_new`. - pub fn new_unchecked(mut addr: u64) -> VirtAddr { - if addr.get_bit(47) { - addr.set_bits(48..64, 0xffff); - } else { - addr.set_bits(48..64, 0); - } - VirtAddr(addr) - } - - /// Creates a virtual address that points to `0`. - pub const fn zero() -> VirtAddr { - VirtAddr(0) - } - - /// Converts the address to an `u64`. - pub fn as_u64(self) -> u64 { - self.0 - } - - /// Creates a virtual address from the given pointer - pub fn from_ptr(ptr: *const T) -> Self { - Self::new(u64::from_usize(ptr as usize)) - } - - /// Converts the address to a raw pointer. - #[cfg(target_pointer_width = "64")] - pub fn as_ptr(self) -> *const T { - usize_from(self.as_u64()) as *const T - } - - /// Converts the address to a mutable raw pointer. - #[cfg(target_pointer_width = "64")] - pub fn as_mut_ptr(self) -> *mut T { - self.as_ptr::().cast_mut() - } - - /// Aligns the virtual address upwards to the given alignment. - /// - /// See the `align_up` free function for more information. - #[must_use] - pub fn aligned_up(self, align: usize) -> Self { - VirtAddr( - align_up(self.0.try_into().unwrap(), align) - .try_into() - .unwrap(), - ) - } - - /// Aligns the virtual address downwards to the given alignment. - /// - /// See the `align_down` free function for more information. - #[must_use] - pub fn aligned_down(self, align: usize) -> Self { - VirtAddr( - align_down(self.0.try_into().unwrap(), align) - .try_into() - .unwrap(), - ) - } - - /// Checks whether the virtual address has the demanded alignment. - pub fn is_aligned(self, align: usize) -> bool { - self.aligned_down(align) == self - } - - /// Returns the 12-bit page offset of this virtual address. - pub fn page_offset(&self) -> u12 { - u12::new((self.0 & 0xfff).try_into().unwrap()) - } - // ^ @todo this only works for 4KiB pages - - /// Returns the 9-bit level 3 page table index. - pub fn l3_index(&self) -> u9 { - u9::new(((self.0 >> 12) & 0o777).try_into().unwrap()) - } - - /// Returns the 9-bit level 2 page table index. - pub fn l2_index(&self) -> u9 { - u9::new(((self.0 >> 12 >> 9) & 0o777).try_into().unwrap()) - } - - /// Returns the 9-bit level 1 page table index. - pub fn l1_index(&self) -> u9 { - u9::new(((self.0 >> 12 >> 9 >> 9) & 0o777).try_into().unwrap()) - } - - /// Returns the 9-bit level 0 page table index. - pub fn l0_index(&self) -> u9 { - u9::new(((self.0 >> 12 >> 9 >> 9 >> 9) & 0o777).try_into().unwrap()) - } -} - -impl fmt::Debug for VirtAddr { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!(f, "VirtAddr({:#x})", self.0) - } -} - -impl fmt::Binary for VirtAddr { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - self.0.fmt(f) - } -} - -impl fmt::LowerHex for VirtAddr { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - self.0.fmt(f) - } -} - -impl fmt::Octal for VirtAddr { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - self.0.fmt(f) - } -} - -impl fmt::UpperHex for VirtAddr { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - self.0.fmt(f) - } -} - -impl From for VirtAddr { - fn from(value: u64) -> Self { - VirtAddr::new(value) - } -} - -impl From for u64 { - fn from(value: VirtAddr) -> Self { - value.as_u64() - } -} - -impl Add for VirtAddr { - type Output = Self; - fn add(self, rhs: u64) -> Self::Output { - VirtAddr::new(self.0 + rhs) - } -} - -impl AddAssign for VirtAddr { - fn add_assign(&mut self, rhs: u64) { - *self = *self + rhs; - } -} - -impl Add for VirtAddr -where - u64: FromUsize, -{ - type Output = Self; - fn add(self, rhs: usize) -> Self::Output { - self + u64::from_usize(rhs) - } -} - -impl AddAssign for VirtAddr -where - u64: FromUsize, -{ - fn add_assign(&mut self, rhs: usize) { - self.add_assign(u64::from_usize(rhs)); - } -} - -impl Sub for VirtAddr { - type Output = Self; - fn sub(self, rhs: u64) -> Self::Output { - VirtAddr::new(self.0.checked_sub(rhs).unwrap()) - } -} - -impl SubAssign for VirtAddr { - fn sub_assign(&mut self, rhs: u64) { - *self = *self - rhs; - } -} - -impl Sub for VirtAddr -where - u64: FromUsize, -{ - type Output = Self; - fn sub(self, rhs: usize) -> Self::Output { - self - u64::from_usize(rhs) - } -} - -impl SubAssign for VirtAddr -where - u64: FromUsize, -{ - fn sub_assign(&mut self, rhs: usize) { - self.sub_assign(u64::from_usize(rhs)); - } -} - -impl Sub for VirtAddr { - type Output = u64; - fn sub(self, rhs: VirtAddr) -> Self::Output { - self.as_u64().checked_sub(rhs.as_u64()).unwrap() - } -} - -impl Rem for VirtAddr -where - u64: FromUsize, -{ - type Output = u64; - fn rem(self, rhs: usize) -> Self::Output { - self.0 % u64::from_usize(rhs) - } -} - -impl RemAssign for VirtAddr -where - u64: FromUsize, -{ - fn rem_assign(&mut self, rhs: usize) { - *self = VirtAddr::new(self.0 % u64::from_usize(rhs)); - } -} diff --git a/libs/memory/src/walk.rs b/libs/memory/src/walk.rs new file mode 100644 index 000000000..16dea7218 --- /dev/null +++ b/libs/memory/src/walk.rs @@ -0,0 +1,141 @@ +use { + crate::arch_trait::{EntryKind, TranslationArch}, + libaddress::{PhysAddr, VirtAddr}, +}; + +/// Result of a successful address translation walk. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct TranslationResult { + /// The resolved physical address. + pub phys_addr: PhysAddr, + /// The level at which the translation terminated (block or page). + pub level: usize, + /// The block/page size at the terminating level. + pub block_size: usize, +} + +/// Walk the translation table hierarchy to translate a virtual address. +/// +/// The `resolve` callback converts a physical table address and level into +/// a slice of u64 entries. This keeps the library independent of any +/// physical-to-virtual mapping strategy — the caller decides how to +/// access physical memory (identity map, kernel window, etc). +/// +/// For hierarchical page tables, this walks from root to leaf through +/// multiple levels. For hashed page tables (`A::HASHED`), use +/// `translate_hashed` instead. +/// +/// Returns `None` if the walk encounters an invalid entry at any level. +pub fn translate( + root_phys: PhysAddr, + vaddr: VirtAddr, + resolve: F, +) -> Option +where + A: TranslationArch, + F: Fn(PhysAddr, usize) -> Option<&'static [u64]>, +{ + let mut table_phys = root_phys; + + for level in 0..A::NUM_LEVELS { + let entries = resolve(table_phys, level)?; + let index = A::index_from_vaddr(vaddr, level); + + // Read the entry at this index, accounting for ENTRY_WIDTH. + let offset = index * A::ENTRY_WIDTH; + let entry_slice = entries.get(offset..offset + A::ENTRY_WIDTH)?; + let kind = A::decode_entry_wide(entry_slice, level); + + match kind { + EntryKind::Invalid => return None, + EntryKind::Table(next_phys) => { + table_phys = next_phys; + // Continue to next level. + } + EntryKind::Block(block_phys) => { + let caps = A::level_capabilities(level); + let page_offset = vaddr.as_usize() & (caps.block_size - 1); + let phys_addr = PhysAddr::new(block_phys.as_u64() + page_offset as u64); + return Some(TranslationResult { + phys_addr, + level, + block_size: caps.block_size, + }); + } + } + } + + // Reached past the last level without finding a block/page — should not happen + // with a well-formed architecture, but handle gracefully. + None +} + +/// Translate a virtual address using a hashed page table (e.g. PowerPC HPT). +/// +/// Unlike hierarchical translation, HPT lookup: +/// 1. Computes a primary hash from the VA and VSID to find a PTEG +/// 2. Searches all entries in the PTEG for a matching VA +/// 3. If not found, computes a secondary hash and searches that PTEG +/// +/// The `resolve` callback maps the HPT base physical address to a `&[u64]` +/// slice covering the entire hash table. The `vsid` is the Virtual Segment +/// ID from the SLB for this virtual address. +/// +/// `entries_per_pteg` is typically 8 for PPC64. +pub fn translate_hashed( + htab_phys: PhysAddr, + vaddr: VirtAddr, + vsid: u64, + htab_mask: u64, + entries_per_pteg: usize, + resolve: F, +) -> Option +where + A: TranslationArch, + F: Fn(PhysAddr, usize) -> Option<&'static [u64]>, +{ + let htab = resolve(htab_phys, 0)?; + let caps = A::level_capabilities(0); + + // Try primary hash. + let primary = A::hash_primary(vaddr, vsid, htab_mask); + if let Some(result) = search_pteg::(htab, primary, entries_per_pteg, vaddr, &caps) { + return Some(result); + } + + // Try secondary hash. + let secondary = A::hash_secondary(primary, htab_mask); + search_pteg::(htab, secondary, entries_per_pteg, vaddr, &caps) +} + +/// Search a single PTEG (Page Table Entry Group) for a matching entry. +fn search_pteg( + htab: &[u64], + pteg_index: usize, + entries_per_pteg: usize, + vaddr: VirtAddr, + caps: &crate::arch_trait::LevelCapabilities, +) -> Option { + let base = pteg_index * entries_per_pteg * A::ENTRY_WIDTH; + + for slot in 0..entries_per_pteg { + let offset = base + slot * A::ENTRY_WIDTH; + let entry_slice = htab.get(offset..offset + A::ENTRY_WIDTH)?; + if let EntryKind::Block(block_phys) = A::decode_entry_wide(entry_slice, 0) { + // For HPT, we need to verify the VA matches the entry's AVPN. + // The decode_entry_wide implementation should only return Block + // for entries matching the queried VA — this requires the arch + // implementation to encode the VA comparison in decode, or we + // expose a separate match check. For now, any valid Block is + // considered a hit since the PTEG was selected by hash. + let page_offset = vaddr.as_usize() & (caps.block_size - 1); + let phys_addr = PhysAddr::new(block_phys.as_u64() + page_offset as u64); + return Some(TranslationResult { + phys_addr, + level: 0, + block_size: caps.block_size, + }); + } + } + None +} diff --git a/libs/mmio/Cargo.toml b/libs/mmio/Cargo.toml new file mode 100644 index 000000000..f30ebed7a --- /dev/null +++ b/libs/mmio/Cargo.toml @@ -0,0 +1,30 @@ +[package] +name = "vesper-mmio" + +description = "Vesper nanokernel MMIO abstractions." + +authors = { workspace = true } +categories = { workspace = true } +documentation = { workspace = true } +edition = { workspace = true } +homepage = { workspace = true } +license = { workspace = true } +readme = { workspace = true } +repository = { workspace = true } +version = { workspace = true } +publish = false + +[badges] +maintenance = { status = "experimental" } + +[dependencies] +libaddress = { workspace = true } +# safe-mmio = { workspace = true } + +# Tests are offloaded to kernel integration tests binary. +[lib] +test = false +harness = false + +[lints] +workspace = true diff --git a/libs/mmio/README.md b/libs/mmio/README.md new file mode 100644 index 000000000..668aec85d --- /dev/null +++ b/libs/mmio/README.md @@ -0,0 +1,3 @@ +A safe wrapper for MMIO device access. + +Built on top of safe_mmio. diff --git a/libs/platform/src/platform/raspberrypi/device_driver/common.rs b/libs/mmio/src/lib.rs similarity index 92% rename from libs/platform/src/platform/raspberrypi/device_driver/common.rs rename to libs/mmio/src/lib.rs index 0d1bbe2fa..71031b499 100644 --- a/libs/platform/src/platform/raspberrypi/device_driver/common.rs +++ b/libs/mmio/src/lib.rs @@ -2,18 +2,20 @@ // // Copyright (c) 2020-2022 Andre Richter -//! Common device driver code. -// @todo: Move to libprimitive or libdriver or sth? +#![no_std] use { core::{marker::PhantomData, ops}, - libmemory::{Address, Virtual}, + libaddress::{Address, Virtual}, }; //-------------------------------------------------------------------------------------------------- // Public Definitions //-------------------------------------------------------------------------------------------------- +pub mod mmio_descriptor; +pub use mmio_descriptor::*; + #[allow(clippy::partial_pub_fields)] pub struct MMIODerefWrapper { pub base_addr: Address, // @todo unmake public, GPIO::Pin uses it diff --git a/libs/mmio/src/mmio_descriptor.rs b/libs/mmio/src/mmio_descriptor.rs new file mode 100644 index 000000000..12d1fb22e --- /dev/null +++ b/libs/mmio/src/mmio_descriptor.rs @@ -0,0 +1,31 @@ +use libaddress::{Address, Physical}; + +/// An MMIO descriptor for use in device drivers. +#[derive(Copy, Clone)] +pub struct MMIODescriptor { + start_addr: Address, + end_addr_exclusive: Address, +} + +impl MMIODescriptor { + /// Create an instance. + pub const fn new(start_addr: Address, size: usize) -> Self { + assert!(size > 0); + let end_addr_exclusive = Address::new(start_addr.as_u64() + size as u64); + + Self { + start_addr, + end_addr_exclusive, + } + } + + /// Return the start address. + pub const fn start_addr(&self) -> Address { + self.start_addr + } + + /// Return the exclusive end address. + pub fn end_addr_exclusive(&self) -> Address { + self.end_addr_exclusive + } +} diff --git a/libs/object/Cargo.toml b/libs/object/Cargo.toml new file mode 100644 index 000000000..939fe10f8 --- /dev/null +++ b/libs/object/Cargo.toml @@ -0,0 +1,33 @@ +[package] +name = "vesper-objects" + +description = "Vesper nanokernel objects interface." + +authors = { workspace = true } +categories = { workspace = true } +documentation = { workspace = true } +edition = { workspace = true } +homepage = { workspace = true } +license = { workspace = true } +readme = { workspace = true } +repository = { workspace = true } +version = { workspace = true } + +publish = false + +[badges] +maintenance = { status = "experimental" } + +[dependencies] +libaddress = { workspace = true } +libprint = { workspace = true } +libqemu = { workspace = true } +libsyscall = { workspace = true } + +# Tests are offloaded to kernel integration tests binary. +[lib] +test = false +harness = false + +[lints] +workspace = true diff --git a/libs/object/src/arch/asid_pool.rs b/libs/object/src/arch/asid_pool.rs new file mode 100644 index 000000000..71470c687 --- /dev/null +++ b/libs/object/src/arch/asid_pool.rs @@ -0,0 +1,5 @@ +#[repr(u8)] +pub enum ASIDPoolOp { + /// Allocate an ASID from this pool + Allocate = 0, +} diff --git a/libs/object/src/arch/frame.rs b/libs/object/src/arch/frame.rs new file mode 100644 index 000000000..99a930340 --- /dev/null +++ b/libs/object/src/arch/frame.rs @@ -0,0 +1,11 @@ +#[repr(u8)] +pub enum FrameOp { + /// Map frame into a VSpace at given virtual address + Map = 0, + /// Unmap frame from VSpace + Unmap = 1, + /// Get physical address (requires special rights) + GetAddress = 2, + /// Remap with different attributes + Remap = 3, +} diff --git a/libs/object/src/arch/page_table.rs b/libs/object/src/arch/page_table.rs new file mode 100644 index 000000000..ae0ad0a9e --- /dev/null +++ b/libs/object/src/arch/page_table.rs @@ -0,0 +1,7 @@ +#[repr(u8)] +pub enum PageTableOp { + /// Map this page table into parent table or VSpace + Map = 0, + /// Unmap from parent + Unmap = 1, +} diff --git a/libs/object/src/arch/vspace.rs b/libs/object/src/arch/vspace.rs new file mode 100644 index 000000000..1914c1d7a --- /dev/null +++ b/libs/object/src/arch/vspace.rs @@ -0,0 +1,11 @@ +#[repr(u8)] +pub enum VSpaceOp { + /// Assign root page table + SetRoot = 0, + /// Assign ASID + AssignASID = 1, + /// Activate (switch to this address space) + Activate = 2, + /// Get current ASID + GetASID = 3, +} diff --git a/libs/object/src/buffer.rs b/libs/object/src/buffer.rs new file mode 100644 index 000000000..0dfd34d65 --- /dev/null +++ b/libs/object/src/buffer.rs @@ -0,0 +1,191 @@ +// ================================================== +// == Public user interface, usable from userspace == +// ================================================== + +/// Buffer capability with permission tracking in type system +pub struct BufferKey { + key: Key, + size: usize, + _perm: PhantomData

, +} + +/// Buffer operations via protected_call +#[repr(u8)] +pub enum BufferOp { + Map = 0, // Map into caller's address space + Unmap = 1, // Remove mapping + Query = 2, // Get buffer info (size, flags, mapping status) +} + +pub trait Permission {} +pub struct ReadOnly; +pub struct ReadWrite; +impl Permission for ReadOnly {} +impl Permission for ReadWrite {} + +impl BufferKey

{ + /// Map buffer into current domain's address space. + /// + /// # Arguments + /// * `hint` - Optional preferred virtual address (kernel may ignore) + /// * `flags` - Mapping flags (caching behavior, etc.) + /// + /// # Returns + /// Virtual address where buffer is mapped + pub fn map(&self, hint: Option, flags: MapFlags) -> Result { + let hint_val = hint.map(|a| a.as_u64()).unwrap_or(0); + + let ret = unsafe { + protected_call2( + self.key.slot as u64, + BufferOp::Map as u64, + hint_val, + // self.size, ? + flags.bits() as u64, + ) + }; + + if ret == 0 { + Err(Error::MapFailed) + } else { + Ok(VirtAddr::new(ret)) + } + } + + /// Unmap buffer from current domain's address space. + pub fn unmap(&self) -> Result<(), Error> { + let ret = unsafe { protected_call1(self.key.slot as u64, BufferOp::Unmap as u64, 0) }; + Error::from_code(ret) + } + + /// Query buffer information (no mapping required) + pub fn query(&self) -> Result { + let mut info = BufferInfo::default(); + let ret = unsafe { + protected_call2( + self.key.slot as u64, + BufferOp::Query as u64, + &mut info as *mut _ as u64, + 0, + ) + }; + if ret == 0 { + Ok(info) + } else { + Err(Error::from_code(ret)) + } + } +} + +// Other ops +impl BufferKey

{ + /// Size of this buffer + #[inline] + pub fn size(&self) -> usize { + self.size + } +} + +impl BufferKey { + /// Derive read-only capability (like &T from &mut T) + pub fn derive_readonly(&self, dest_slot: u32) -> Result, Error> { + let ret = unsafe { + protected_call3( + CAPTBL_SELF, + KeyTableOp::CopyDerive, + self.key.slot as u64, + dest_slot as u64, + Rights::READ.bits() as u64, + ) + }; + if ret == 0 { + Ok(BufferKey { + key: Cap::new(dest_slot), + size: self.size, + _perm: PhantomData, + }) + } else { + Err(Error::from_code(ret)) + } + } + + /// Map and get mutable slice + pub fn map_slice_mut(&self) -> Result, Error> { + let addr = self.map(None, MapFlags::NONE)?; + Ok(MappedSliceMut { + cap: self, + ptr: addr.as_mut_ptr(), + len: self.size, + }) + } +} + +impl BufferKey { + /// Map and get immutable slice + pub fn map_slice(&self) -> Result, Error> { + let addr = self.map(None, MapFlags::NONE)?; + Ok(MappedSlice { + cap: self, + ptr: addr.as_ptr(), + len: self.size, + }) + } +} + +#[derive(Default)] +pub struct BufferInfo { + pub size: usize, + pub flags: BufferFlags, + pub is_mapped: bool, + pub mapped_addr: Option, +} + +bitflags::bitflags! { + pub struct MapFlags: u32 { + const NONE = 0; + const FIXED = 1 << 0; // Fail if hint can't be used + const POPULATE = 1 << 1; // Pre-fault all pages + const UNCACHED = 1 << 2; // Override to uncached + } +} + +/// RAII guard that unmaps on drop +pub struct MappedSlice<'a> { + cap: &'a BufferKey, + ptr: *const u8, + len: usize, +} + +impl<'a> MappedSlice<'a> { + pub fn as_slice(&self) -> &[u8] { + unsafe { core::slice::from_raw_parts(self.ptr, self.len) } + } +} + +impl Drop for MappedSlice<'_> { + fn drop(&mut self) { + let _ = self.cap.unmap(); + } +} + +pub struct MappedSliceMut<'a> { + cap: &'a BufferKey, + ptr: *mut u8, + len: usize, +} + +impl<'a> MappedSliceMut<'a> { + pub fn as_slice(&self) -> &[u8] { + unsafe { core::slice::from_raw_parts(self.ptr, self.len) } + } + + pub fn as_mut_slice(&mut self) -> &mut [u8] { + unsafe { core::slice::from_raw_parts_mut(self.ptr, self.len) } + } +} + +impl Drop for MappedSliceMut<'_> { + fn drop(&mut self) { + let _ = self.cap.unmap(); + } +} diff --git a/libs/object/src/debug_console.rs b/libs/object/src/debug_console.rs new file mode 100644 index 000000000..af2d15c61 --- /dev/null +++ b/libs/object/src/debug_console.rs @@ -0,0 +1,63 @@ +use crate::{CapError, Key, KeySlot}; + +// ================================================== +// == Public user interface, usable from userspace == +// ================================================== + +pub struct DebugConsoleKey { + key: Key, +} + +enum DebugConsoleType {} + +#[repr(u8)] +pub enum DebugConsoleOp { + /// Capability invocation to write a message to a debug console. + Write = 0, +} + +impl TryFrom for DebugConsoleOp { + type Error = CapError; + + fn try_from(value: u32) -> Result { + match value { + 0 => Ok(DebugConsoleOp::Write), + _ => Err(CapError::InvalidOperation), + } + } +} + +// Root domain gets a DebugConsoleCap, can delegate to others +impl DebugConsoleKey { + #[expect(clippy::new_without_default)] + pub const fn new() -> Self { + Self { + key: Key::new(KeySlot::DEBUG_CONSOLE), + } + } + + pub const fn new_slot(slot: KeySlot) -> Self { + Self { + key: Key::new(slot), + } + } + + pub fn write(&self, s: &str) -> Result<(), CapError> { + // SAFETY: Unsafe call. + let (ok, r1, r2) = unsafe { + libsyscall::protected_call2( + self.key.slot(), + DebugConsoleOp::Write as u32, + s.as_ptr() as u64, + s.len() as u64, + ) + }; + libqemu::semi_println!( + "Userspace return from DebugConsoleOp::Write with result ({}, {}, {})", + ok, + r1, + r2 + ); + Ok(()) + } +} diff --git a/libs/object/src/domain.rs b/libs/object/src/domain.rs new file mode 100644 index 000000000..96dc50ec1 --- /dev/null +++ b/libs/object/src/domain.rs @@ -0,0 +1,485 @@ +use { + crate::{CapError, Key, KeySlot}, + core::sync::atomic::{AtomicU32, AtomicU64, Ordering}, + libaddress::VirtAddr, + libsyscall::{protected_call0, protected_call2}, +}; + +// ┌─────────────────────────────────────────────────────────────────────┐ +// │ DCB SHARED PAGES ARCHITECTURE │ +// ├─────────────────────────────────────────────────────────────────────┤ +// │ │ +// │ The DCB pages are the Nemesis-inspired mechanism for zero-syscall │ +// │ domain state queries. The kernel maintains DCBs for all domains, │ +// │ mapped read-only into user space. │ +// │ │ +// │ KERNEL VIEW (RW) USER VIEW (RO) │ +// │ ──────────────── ─────────────── │ +// │ │ +// │ 0xFFFF_0000_xxxx_xxxx 0x7FFF_00xx_xxxx_xxxx │ +// │ (kernel linear map) (user mapping) │ +// │ │ │ │ +// │ │ ┌─────────────────────┐ │ │ +// │ └───►│ Physical DCB Page │◄─────────┘ │ +// │ │ ┌───────────────┐ │ │ +// │ │ │ DCB[0] 128B │ │ │ +// │ │ ├───────────────┤ │ │ +// │ │ │ DCB[1] 128B │ │ │ +// │ │ ├───────────────┤ │ │ +// │ │ │ DCB[2] 128B │ │ │ +// │ │ ├───────────────┤ │ │ +// │ │ │ ... │ │ │ +// │ │ ├───────────────┤ │ │ +// │ │ │ DCB[31] 128B │ │ │ +// │ │ └───────────────┘ │ │ +// │ └─────────────────────┘ │ +// │ 4KB page │ +// │ 32 DCBs per page │ +// │ │ +// └─────────────────────────────────────────────────────────────────────┘ + +// ================================================== +// == Public user interface, usable from userspace == +// ================================================== + +/// Domain identifier +#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)] +#[repr(transparent)] +pub struct DomainId(pub u32); + +impl DomainId { + pub const INVALID: Self = Self(u32::MAX); + + /// Get the page index for this domain's DCB + #[inline] + pub const fn page_index(&self) -> usize { + (self.0 / DcbPage::DCBS_PER_PAGE) as usize + } + + /// Get the slot within the page + #[inline] + pub const fn slot_in_page(&self) -> usize { + (self.0 % DcbPage::DCBS_PER_PAGE) as usize + } +} + +#[repr(u8)] +pub enum DomainOp { + Activate = 0, // Make domain runnable + Grant = 1, // Grant capability to domain + Suspend = 2, // Suspend domain + Resume = 3, // Resume suspended domain +} + +/// Domain capability - handle to a protection domain. +/// State queries use shared DCB (no syscall), mutations use `CapInvoke`. +pub struct DomainKey { + key: Key, + id: DomainId, +} + +enum DomainType {} + +impl DomainKey { + // Create a new domain from untyped memory. + // Convenience wrapper around UntypedRetype. + // pub fn create(untyped: &mut UntypedCap, dest_slot: KeySlot) -> Result { + // // Domains need ~4KB (12 bits) for kernel structures + // untyped.retype( + // untyped.split(12)?, // Carve off 4KB + // ObjectType::Domain, + // 12, + // dest_slot, + // )?; + // + // // Domain ID is returned in secondary return value + // // (or we query it from the newly created DCB) + // Ok(DomainKey { + // cap: Cap::new(dest_slot), + // id: DomainId(0), + // }) + // } + + /// Get domain state from shared DCB + #[inline] + pub fn state(&self) -> DomainState { + // SAFETY: Unsafe call. + let dcb_view = unsafe { DcbView::from_user_mapping() }; + let dcb = dcb_view.get(self.id).expect("oh well"); + DomainState::try_from(dcb.state.load(Ordering::Acquire)).unwrap_or(DomainState::Inactive) + } + + /// Get time used from shared DCB + #[inline] + pub fn time_used_ns(&self) -> u64 { + // SAFETY: Unsafe call. + let dcb_view = unsafe { DcbView::from_user_mapping() }; + let dcb = dcb_view.get(self.id).expect("oh well"); + dcb.time_consumed_ns.load(Ordering::Relaxed) + } + + /// Get pending notifications from shared DCB (NO SYSCALL!) + #[inline] + pub fn pending_notifications(&self) -> u64 { + // SAFETY: Unsafe call. + let dcb_view = unsafe { DcbView::from_user_mapping() }; + let dcb = dcb_view.get(self.id).expect("oh well"); + dcb.pending_notifications.load(Ordering::Relaxed) + } + + /// Activate domain (make runnable) - requires syscall + pub fn activate(&self) -> Result<(), CapError> { + // SAFETY: Unsafe call. + let (ok, _, _) = unsafe { protected_call0(self.key.slot(), DomainOp::Activate as u32) }; + match ok { + 0 => Ok(()), + _ => Err(CapError::Unknown), + } + } + + /// Grant a capability to this domain + pub fn grant(&self, key: &Key, dest_slot: KeySlot) -> Result<(), CapError> { + // SAFETY: Unsafe call. + let (ok, _, _) = unsafe { + protected_call2( + self.key.slot(), + DomainOp::Grant as u32, + u64::from(key.slot()), + u64::from(dest_slot.0), + ) + }; + match ok { + 0 => Ok(()), + _ => Err(CapError::Unknown), + } + } + + /// Suspend domain - requires syscall + pub fn suspend(&self) -> Result<(), CapError> { + // SAFETY: Unsafe call. + let (ok, _, _) = unsafe { protected_call0(self.key.slot(), DomainOp::Suspend as u32) }; + match ok { + 0 => Ok(()), + _ => Err(CapError::Unknown), + } + } + + /// Resume suspended domain - requires syscall + pub fn resume(&self) -> Result<(), CapError> { + // SAFETY: Unsafe call. + let (ok, _, _) = unsafe { protected_call0(self.key.slot(), DomainOp::Resume as u32) }; + match ok { + 0 => Ok(()), + _ => Err(CapError::Unknown), + } + } +} + +// ═══════════════════════════════════════════════════════════════════ +// DOMAIN CONTROL BLOCK (DCB) +// ═══════════════════════════════════════════════════════════════════ + +/// Domain execution state +#[repr(u32)] +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub enum DomainState { + /// Just created, never activated + Inactive = 0, + /// Ready to run, waiting for CPU time + Runnable = 1, + /// Currently executing on a CPU + Running = 2, + /// Blocked waiting on IPC/notification/event + Blocked = 3, + /// Explicitly suspended by parent + Suspended = 4, + /// Faulted, needs handler + Faulted = 5, + /// Being destroyed + Dying = 6, +} + +/// Why a domain is blocked +#[repr(u32)] +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub enum BlockReason { + None = 0, + Notification = 1, // Waiting on NotificationKey::wait() + EventCount = 2, // Waiting on EventCountKey::await_ge() + EndpointSend = 3, // Blocked on EndpointKey::call() send phase + EndpointRecv = 4, // Blocked on EndpointKey::recv() + Reply = 5, // Waiting for reply after call() + TimeDonation = 6, // Donated time, waiting for return +} + +// pub enum DeactivateReason { +// TimeExhausted, +// BlockedOnEvent(EndpointCap), <-- BlockReason::Endpoint +// Yielded, +// Faulted(fault), +// } + +// ═══════════════════════════════════════════════════════════ +// SHARED SECTION (read-only mapped to userspace) +// ═══════════════════════════════════════════════════════════ + +/// Domain Control Block - shared between kernel and userspace. +/// This is a user-visible half of domain state structure. +/// +/// This is the Nemesis-inspired design for zero-syscall state queries. +/// The kernel writes, userspace reads. +/// +/// Size: 128 bytes (cache-line aligned, 32 per 4KB page) +#[repr(C, align(128))] +pub struct DomainControlBlock { + // ─── Identity ─── + /// Domain ID + pub id: DomainId, + /// Domain name (for debugging) + pub name: [u8; 24], + + // ─── Execution State ─── + /// Current state (use Acquire ordering when reading) + pub state: AtomicU32, // DomainState + /// Why blocked (valid when state == Blocked) + pub block_reason: AtomicU32, // BlockReason (if blocked) + /// Key slot we're blocked on (e.g., which notification) + pub blocked_on_slot: AtomicU32, // Cap slot we're blocked on + /// CPU this domain is running on (or was last running on) + pub cpu: AtomicU32, + + // ─── Time Accounting (QoS) ─── + /// Total CPU time consumed (nanoseconds) + pub time_consumed_ns: AtomicU64, + /// Remaining time in current activation (nanoseconds) + pub time_remaining_ns: AtomicU64, + /// Number of times this domain has been activated + pub activation_count: AtomicU64, + /// Timestamp of last activation (for profiling) + pub last_activated_ns: AtomicU64, + + // ─── Scheduling Parameters ─── + /// Parent scheduler domain + pub scheduler_id: DomainId, + /// Priority (higher = more important) + pub priority: u32, + /// Scheduling flags + pub sched_flags: u32, + /// Period for periodic domains (nanoseconds, 0 = aperiodic) + pub period_ns: u64, + // CPU allocation per period + // pub budget_ns: u64, // slice_ns + // Scheduled deadline (absolute time) + // pub deadline: AtomicU64, + + // ─── Event State ─── + /// Bitmap of pending notification slots + pub pending_notifications: AtomicU64, + /// Count of event counts with pending events + pub pending_event_counts: AtomicU32, + /// Count of pending endpoint messages + pub pending_endpoints: AtomicU32, + // Endpoint that caused last wakeup + // pub last_event_ep: AtomicU32, + + // ─── Fault Information ─── + /// Fault type (valid when state == Faulted) + pub fault_type: AtomicU32, + /// Fault-specific code + pub fault_code: AtomicU32, + /// Fault address (for page faults) + pub fault_addr: AtomicU64, + /// Key slot that caused fault (for cap faults) + pub fault_slot: AtomicU32, +} + +// Compile-time size check +// TODO const _: () = assert!(core::mem::size_of::() == 128); +// TODO const _: () = assert!(core::mem::align_of::() == 128); + +impl DomainControlBlock { + /// Create a new DCB for a domain + pub const fn new(id: DomainId, scheduler_id: DomainId) -> Self { + Self { + id, + scheduler_id, + name: [0; 24], + state: AtomicU32::new(DomainState::Inactive as u32), + block_reason: AtomicU32::new(BlockReason::None as u32), + blocked_on_slot: AtomicU32::new(0), + cpu: AtomicU32::new(0), + time_consumed_ns: AtomicU64::new(0), + time_remaining_ns: AtomicU64::new(0), + activation_count: AtomicU64::new(0), + last_activated_ns: AtomicU64::new(0), + priority: 0, + sched_flags: 0, + period_ns: 0, + pending_notifications: AtomicU64::new(0), + pending_event_counts: AtomicU32::new(0), + pending_endpoints: AtomicU32::new(0), + fault_type: AtomicU32::new(0), + fault_code: AtomicU32::new(0), + fault_addr: AtomicU64::new(0), + fault_slot: AtomicU32::new(0), + } + } + + /// Set domain name (truncated to 8 bytes) + pub fn set_name(&mut self, name: &str) { + let bytes = name.as_bytes(); + let len = bytes.len().min(8); + self.name[..len].copy_from_slice(&bytes[..len]); + self.name[len..].fill(0); + } + + /// Get domain name as string slice + pub fn name_str(&self) -> &str { + let len = self.name.iter().position(|&b| b == 0).unwrap_or(8); + core::str::from_utf8(&self.name[..len]).unwrap_or("???") + } +} + +// ═══════════════════════════════════════════════════════════════════ +// DCB PAGE +// ═══════════════════════════════════════════════════════════════════ + +/// A single 4KB page containing 32 DCBs. +#[repr(C, align(4096))] +pub struct DcbPage { + dcbs: [DomainControlBlock; Self::DCBS_PER_PAGE as usize], +} + +impl DcbPage { + /// Number of DCBs per 4KB page (128 bytes each) + pub const DCBS_PER_PAGE: u32 = 4096 / 128; // = 32 + + /// Create a new page with uninitialized DCBs + #[expect(clippy::new_without_default)] + pub const fn new() -> Self { + // This is a bit ugly but works at const time + #[expect(clippy::declare_interior_mutable_const)] + const INIT_DCB: DomainControlBlock = + DomainControlBlock::new(DomainId::INVALID, DomainId::INVALID); + Self { + dcbs: [INIT_DCB; Self::DCBS_PER_PAGE as usize], + } + } + + /// Get a DCB by slot index + #[inline] + pub fn get(&self, slot: usize) -> Option<&DomainControlBlock> { + self.dcbs.get(slot) + } + + /// Get a mutable DCB by slot index + #[inline] + pub fn get_mut(&mut self, slot: usize) -> Option<&mut DomainControlBlock> { + self.dcbs.get_mut(slot) + } +} + +// TODO const _: () = assert!(core::mem::size_of::() == 4096); + +// ═══════════════════════════════════════════════════════════════════ +// DCB VIEW +// Userspace sees: const DCB_BASE: *const DomainControlBlock = DcbPages::USER_BASE; // FIXME: pervasives +// Access DCB n: &*DCB_BASE.add(n) +// ═══════════════════════════════════════════════════════════════════ + +/// User-space view of DCB pages (read-only). +/// +/// This is used by schedulers and other user-space code to query +/// domain state without syscalls. +pub struct DcbView { + base: *const DomainControlBlock, +} + +impl DcbView { + /// Create from the well-known user-space address + /// + /// # Safety + /// Must only be called after kernel has set up the mapping + pub const unsafe fn from_user_mapping() -> Self { + // FIXME: Duplicate DcbPages::USER_BASE const from nucleus/objects/domain.rs here, keep in sync! + // SAFETY: Unsafe call. + const USER_BASE: VirtAddr = unsafe { VirtAddr::new_unchecked(0x0000_7FFF_FE00_0000) }; + Self { + base: USER_BASE.as_ptr(), + } + } + + /// Get a DCB by domain ID (read-only) + /// + /// Returns None if the domain ID is invalid or not allocated. + /// Note: We can't actually check allocation status from user-space, + /// so we just check if the DCB looks valid. + #[inline] + pub fn get(&self, id: DomainId) -> Option<&DomainControlBlock> { + // FIXME: Duplicate DcbPages::MAX_DOMAINS const from nucleus/objects/domain.rs here, keep in sync! + const MAX_DOMAINS: u32 = 8192; + if id.0 >= MAX_DOMAINS { + return None; + } + + // SAFETY: unsafe + let dcb = unsafe { &*self.base.add(id.0 as usize) }; + + // Basic validity check + if dcb.id != id { + return None; + } + + Some(dcb) + } + + /// Get state of a domain (fast path for schedulers) + #[inline] + pub fn state(&self, id: DomainId) -> Option { + let dcb = self.get(id)?; + let raw = dcb.state.load(Ordering::Acquire); + DomainState::try_from(raw).ok() + } + + /// Check if a domain is runnable + #[inline] + pub fn is_runnable(&self, id: DomainId) -> bool { + self.state(id) == Some(DomainState::Runnable) + } + + /// Get time consumed by a domain + #[inline] + pub fn time_consumed(&self, id: DomainId) -> Option { + let dcb = self.get(id)?; + Some(dcb.time_consumed_ns.load(Ordering::Relaxed)) + } + + /// Get my own DCB + #[inline(always)] + pub fn myself(&self) -> &DomainControlBlock { + // Current domain ID stored in thread-local or well-known register + self.get(DomainId(1)) //current_domain_id() + .expect("Self-domain is always present") + } +} + +// Global user-space accessor (set up during domain init) - FIXME: ? +// In user-space code: +// static DCB: DcbView = unsafe { DcbView::from_user_mapping() }; + +impl TryFrom for DomainState { + type Error = (); + + fn try_from(value: u32) -> Result { + match value { + 0 => Ok(DomainState::Inactive), + 1 => Ok(DomainState::Runnable), + 2 => Ok(DomainState::Running), + 3 => Ok(DomainState::Blocked), + 4 => Ok(DomainState::Suspended), + 5 => Ok(DomainState::Faulted), + 6 => Ok(DomainState::Dying), + _ => Err(()), + } + } +} diff --git a/libs/object/src/endpoint.rs b/libs/object/src/endpoint.rs new file mode 100644 index 000000000..d8c8baaca --- /dev/null +++ b/libs/object/src/endpoint.rs @@ -0,0 +1,252 @@ +// Endpoint capability operations +// +// Endpoints enable synchronous call/return IPC with direct domain switch. +// Unlike Notifications (fire-and-forget) or EventCounts (streaming), +// Endpoints are for request/response patterns. + +use {crate::key::Key, libsyscall::protected_call4}; + +// ================================================== +// == Public user interface, usable from userspace == +// ================================================== + +/// Endpoint capability for synchronous IPC +/// +/// Two "views" of the same endpoint: +/// - Client holds EndpointKey (can Call/Send) +/// - Server holds EndpointKey with Recv rights (can Recv/Reply) +/// +/// Badges identify which client is calling (granted during cap derivation) +/// +/// With explicit Reply objects: +/// - recv() returns (badge, msg, ReplyCap) +/// - reply is done via ReplyCap::send(), not endpoint method +/// - reply_recv() takes a ReplyCap to consume +pub struct EndpointKey { + key: Key, +} + +enum EndpointType {} + +#[repr(u8)] +pub enum EndpointOp { + /// Send message and wait for reply (client operation) + /// Blocks until server receives, processes, and replies + Call = 0, + + /// Send message without waiting (non-blocking) + /// If no receiver waiting, message is dropped (returns error) + Send = 1, + + /// Wait for incoming message (server operation) + /// Blocks until a sender calls + Recv = 2, + + /// Reply to caller and wait for next message (server fast-path) + /// Combines Reply + Recv in single syscall (seL4 optimization) + ReplyRecv = 3, + + /// Reply to caller without waiting for next + Reply = 4, + + /// Forward call to another endpoint (proxy pattern) + /// Transfers the reply capability to the new endpoint + Forward = 5, +} + +impl EndpointKey { + // ═══════════════════════════════════════════════════════════════════ + // CLIENT OPERATIONS + // ═══════════════════════════════════════════════════════════════════ + + /// Call: send message and block waiting for reply + /// + /// This is the primary client→server operation. + /// Performs direct domain switch to receiver (fast path). + /// + /// Returns the reply message (label + data + optional cap) + pub fn call(&self, msg: &Message) -> Result { + let (ret0, ret1, ret2, ret3, ret4, ret5, ret6) = unsafe { + syscall_ipc( + self.cap.slot as u64, + EndpointOp::Call as u64, + msg.label, + msg.data[0], + msg.data[1], + msg.data[2], + msg.data[3], + msg.data[4], + msg.cap.map(|s| s as u64).unwrap_or(u64::MAX), + ) + }; + + if ret0 == 0 { + Ok(Message { + label: ret1, + data: [ret2, ret3, ret4, ret5, 0], + cap: if ret6 != u64::MAX { + Some(ret6 as CapSlot) + } else { + None + }, + }) + } else { + Err(IpcError::from_code(ret0)) + } + } + + /// Send: non-blocking send (no reply expected) + /// + /// If receiver is waiting → message delivered, returns Ok + /// If no receiver → message dropped, returns Err(WouldBlock) + pub fn send(&self, msg: &Message) -> Result<(), IpcError> { + let ret = unsafe { + protected_call4( + self.cap.slot as u64, + EndpointOp::Send as u64, + msg.label, + msg.data[0], + msg.data[1], + msg.data[2], + ) + }; + IpcError::from_code(ret.0) + } + + // ═══════════════════════════════════════════════════════════════════ + // SERVER OPERATIONS + // ═══════════════════════════════════════════════════════════════════ + + /// Recv: block waiting for incoming Call + /// + /// Returns (sender_badge, message, reply_cap) + /// Badge identifies which client called (set during cap derivation) + /// + /// The ReplyCap MUST be used to reply - it's consumed on use. + /// Dropping it without replying will send an error to the caller. + pub fn recv(&self, reply_slot: CapSlot) -> Result<(u64, Message, ReplyCap), IpcError> { + let (ret0, badge, label, d0, d1, d2, d3, d4, cap_slot) = unsafe { + syscall_ipc_recv( + self.cap.slot as u64, + EndpointOp::Recv as u64, + reply_slot as u64, // Where kernel places ReplyCap + ) + }; + + if ret0 == 0 { + let msg = Message { + label, + data: [d0, d1, d2, d3, d4], + cap: if cap_slot != u64::MAX { + Some(cap_slot as CapSlot) + } else { + None + }, + }; + + // Kernel placed ReplyCap in reply_slot + let reply_cap = ReplyCap { + cap: Cap::new(reply_slot), + }; + + Ok((badge, msg, reply_cap)) + } else { + Err(IpcError::from_code(ret0)) + } + } + + /// ReplyRecv: consume reply cap, send reply, AND wait for next message + /// + /// This is the server fast-path: one syscall does Reply + Recv. + /// + /// Takes the ReplyCap to consume (must reply to previous caller) + /// Returns new (badge, msg, reply_cap) for next caller + pub fn reply_recv( + &self, + reply_cap: ReplyCap, + reply_msg: &Message, + next_reply_slot: CapSlot, + ) -> Result<(u64, Message, ReplyCap), IpcError> { + let (ret0, badge, label, d0, d1, d2, d3, d4, cap_slot) = unsafe { + syscall_ipc_reply_recv( + self.cap.slot as u64, + EndpointOp::ReplyRecv as u64, + reply_cap.cap.slot as u64, // Reply cap to consume + next_reply_slot as u64, // Where to place next ReplyCap + reply_msg.label, + reply_msg.data[0], + reply_msg.data[1], + reply_msg.data[2], + reply_msg.data[3], + ) + }; + + // reply_cap consumed by kernel + core::mem::forget(reply_cap); + + if ret0 == 0 { + let msg = Message { + label, + data: [d0, d1, d2, d3, d4], + cap: if cap_slot != u64::MAX { + Some(cap_slot as CapSlot) + } else { + None + }, + }; + + let next_reply = ReplyCap { + cap: Cap::new(next_reply_slot), + }; + + Ok((badge, msg, next_reply)) + } else { + Err(IpcError::from_code(ret0)) + } + } + + /// Forward: pass this call to another endpoint (proxy pattern) + /// + /// Useful for: capability-filtered proxies, load balancers, etc. + /// The forwarded call appears to come from us (our badge) + pub fn forward(&self, target: &EndpointKey, msg: &Message) -> Result<(), IpcError> { + let ret = unsafe { + protected_call4( + self.cap.slot as u64, + EndpointOp::Forward as u64, + target.cap.slot as u64, + msg.label, + msg.data[0], + msg.data[1], + ) + }; + IpcError::from_code(ret.0) + } +} + +impl EndpointKey { + /// Derive a client capability with a specific badge + /// + /// The badge is returned to the server on recv(), identifying the caller. + /// This is how servers distinguish between clients. + pub fn derive_client(&self, badge: u64, dest_slot: CapSlot) -> Result { + let (ret, _, _) = unsafe { + protected_call4( + CAPTBL_SELF, // Support deriving directly into a client keytable? + KeyTableOp::CopyDerive, + self.key.slot() as u64, + dest_slot as u64, + Rights::CALL.bits() as u64, // Client can only Call, not Recv + badge, + ) + }; + + if ret == 0 { + Ok(EndpointKey { + cap: Cap::new(dest_slot), + }) + } else { + Err(Error::from_code(ret)) + } + } +} diff --git a/libs/object/src/event_count.rs b/libs/object/src/event_count.rs new file mode 100644 index 000000000..ff49fbdae --- /dev/null +++ b/libs/object/src/event_count.rs @@ -0,0 +1,73 @@ +// ================================================== +// == Public user interface, usable from userspace == +// ================================================== + +/// Event count capability - monotonic Reed-Kanodia counter for exact event tracking. +/// Best for: streaming, flow control, producer-consumer coordination. +/// +/// Unlike notifications, every advance() is counted - no coalescing. +/// This lets consumers know exactly how far behind they are. +pub struct EventCountKey { + key: Key, +} + +enum EventCountType {} + +#[repr(u8)] +pub enum EventCountOp { + Advance = 0, + Await = 1, + Read = 2, +} + +impl EventCountKey { + /// Advance: atomically increment counter by delta. + /// Returns new value. Never blocks. ~30 cycles. + #[inline] + pub fn advance(&self, delta: u64) -> u64 { + protected_call1(self.key.slot as u64, EventOp::Advance as u64, delta) + } + + /// Await: block until value >= target. + /// Returns current value (may be > target if producer is fast). + #[inline] + pub fn await_ge(&self, target: u64) -> u64 { + protected_call1(self.key.slot as u64, EventOp::Await as u64, target) + } + + /// Read: get current value without blocking. + #[inline] + pub fn read(&self) -> u64 { + protected_call0(self.key.slot as u64, EventOp::Read) + } +} + +/// Helper: tracks consumer position for a single reader +pub struct EventCountReader { + ec: EventCountKey, + last_seen: u64, +} + +impl EventCountReader { + pub fn new(ec: EventCountKey) -> Self { + let initial = ec.read(); + Self { + ec, + last_seen: initial, + } + } + + /// Wait for next event(s), returns count since last wait + pub fn wait_next(&mut self) -> u64 { + let target = self.last_seen + 1; + let current = self.ec.await_ge(target); + let delta = current - self.last_seen; + self.last_seen = current; + delta + } + + /// Check how many events pending without blocking + pub fn pending(&self) -> u64 { + self.ec.read() - self.last_seen + } +} diff --git a/libs/object/src/key.rs b/libs/object/src/key.rs new file mode 100644 index 000000000..df934a8cd --- /dev/null +++ b/libs/object/src/key.rs @@ -0,0 +1,26 @@ +use {crate::KeySlot, core::marker::PhantomData}; + +// ================================================== +// == Public user interface, usable from userspace == +// ================================================== + +/// Capability slot index - strongly typed +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +#[repr(transparent)] +pub struct Key { + slot: KeySlot, + _marker: PhantomData, +} + +impl Key { + pub const fn new(slot: KeySlot) -> Self { + Self { + slot, + _marker: PhantomData, + } + } + + pub const fn slot(&self) -> u32 { + self.slot.0 + } +} diff --git a/libs/object/src/key_table.rs b/libs/object/src/key_table.rs new file mode 100644 index 000000000..32d513971 --- /dev/null +++ b/libs/object/src/key_table.rs @@ -0,0 +1,124 @@ +use { + crate::{CapError, Key, Rights}, + libsyscall::{protected_call1, protected_call4}, +}; + +// ================================================== +// == Public user interface, usable from userspace == +// ================================================== + +/// Slot index in a `KeyTable` +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +#[repr(transparent)] +pub struct KeySlot(pub u32); + +impl KeySlot { + pub const NULL: KeySlot = KeySlot(0); + pub const SELF_DOMAIN: KeySlot = KeySlot(1); + pub const PARENT_DOMAIN: KeySlot = KeySlot(2); + // CSpace layout with self-reference + pub const CAPTBL_SELF: KeySlot = KeySlot(3); // Every domain has cap to own captbl here - or rather to KeyMaster + // ... other well-known slots + pub const DEBUG_CONSOLE: KeySlot = KeySlot(127); // FIXME: randomly chosen for now +} + +pub struct KeyTableKey { + key: Key, +} + +enum KeyTableType {} + +#[repr(u8)] +pub enum KeyTableOp { + CopyDerive = 0, // copy cap between slots or create derived cap with reduced rights + Move = 1, // move cap between slots + Delete = 2, // delete cap at slot + Revoke = 4, // revoke all children of cap +} + +// Userspace KeyMaster must track parent→child relationships, +// kernel only manages flat key tables. + +impl KeyTableKey { + // This naturally supports cross-domain derivation: + // "Create a read-only view of my buffer in their cspace" + // derive(&my_captbl, buffer_slot, &their_captbl, their_slot, Rights::READ)?; + /// Copy with derivation in single syscall + pub fn copy_derive( + &self, + src_slot: u32, + dst_captbl: &KeyTableKey, // Could be same or different! + dst_slot: u32, + rights: Rights, + ) -> Result<(), CapError> { + // SAFETY: Unsafe call. + let (ok, _, _) = unsafe { + protected_call4( + self.key.slot(), + KeyTableOp::CopyDerive as u32, + u64::from(src_slot), + u64::from(dst_captbl.key.slot()), + u64::from(dst_slot), + u64::from(rights.bits()), + ) + }; + match ok { + 0 => Ok(()), + _ => Err(CapError::Unknown), + } + } + + // fn activate(&self, slot: u32, object: NucleusObject) -> Result<()> { + // let captbl = self.get_captbl_mut()?; + // // SAFETY: User specifies slot, but kernel validates + // if slot >= captbl.len() { + // return Err(Error::SlotOutOfRange); + // } + // if captbl.slots[slot].is_valid() { + // return Err(Error::SlotOccupied); // User's bookkeeping was wrong + // } + // // Kernel creates the cap - user never touches this + // captbl.slots[slot] = Cap::new(object); + // Ok(()) + // } + + /// Move the key, named "transfer" to avoid clashing with Rust's reserved word. + pub fn transfer() {} + + pub fn delete(&mut self, slot: u32) -> Result<(), CapError> { + // TODO: Must invoke on self-captbl cap + // SAFETY: Unsafe call. + let (_ok, _, _) = + unsafe { protected_call1(self.key.slot(), KeyTableOp::Delete as u32, u64::from(slot)) }; + Ok(()) + } + + // Revoke all children of cap in slot + pub fn revoke(&self, _captbl: &KeyTableKey, slot: u32) -> Result<(), CapError> { + // SAFETY: Unsafe call. + let (_ok, _, _) = + unsafe { protected_call1(self.key.slot(), KeyTableOp::Revoke as u32, u64::from(slot)) }; + Ok(()) + } + + // User code to copy cap to another domain (if you have their captbl cap): + pub fn grant_to( + &self, + my_slot: u32, + their_captbl: &KeyTableKey, + their_slot: u32, + ) -> Result<(), CapError> { + // SAFETY: Unsafe call. + let (_ok, _, _) = unsafe { + protected_call4( + self.key.slot(), + KeyTableOp::CopyDerive as u32, + u64::from(my_slot), + u64::from(their_captbl.key.slot()), + u64::from(their_slot), + u64::from(Rights::all().bits()), + ) + }; + Ok(()) + } +} diff --git a/libs/object/src/lib.rs b/libs/object/src/lib.rs new file mode 100644 index 000000000..99e266824 --- /dev/null +++ b/libs/object/src/lib.rs @@ -0,0 +1,87 @@ +#![no_std] +#![no_main] +#![feature(format_args_nl)] + +pub mod debug_console; +pub mod domain; +pub mod key; +pub mod key_table; +pub mod object_type; +pub mod rights; + +pub use { + debug_console::DebugConsoleKey, + key::Key, + key_table::KeySlot, + object_type::{ArchType, CoreType, ObjectType}, + rights::Rights, +}; + +pub type SyscallResult = core::result::Result<(u64, u64), CapError>; + +pub enum CapError { + Unknown, + NullCapability, + InvalidDomain, + InvalidPointer, + InsufficientRights, + NotMapped, + AlreadyMapped, + InvalidOperation, + ASIDPoolExhausted, + NoASIDAssigned, + InvalidSlot(KeySlot), + EmptySlot(KeySlot), + SlotOccupied(KeySlot), + // Key types + NotCoreType(ObjectType), + UnknownCoreType(u8), + UnsupportedCoreType(CoreType), + NotArchType(ObjectType), + UnknownArchType(u8), + UnsupportedArchType(ArchType), + InvalidObjectType(ObjectType), + TypeMismatch { + expected: ObjectType, + found: ObjectType, + }, + // Object pools + InsufficientMemory, + PoolExhausted, + InvalidSize(usize), + InvalidFrameSize(usize), +} + +impl CapError { + pub fn code(self) -> (u64, u64, u64) { + match self { + CapError::Unknown => (1, 0, 0), + CapError::NullCapability => (2, 0, 0), + CapError::InvalidDomain => (3, 0, 0), + CapError::InvalidPointer => (4, 0, 0), + CapError::InsufficientRights => (5, 0, 0), + CapError::NotMapped => (6, 0, 0), + CapError::AlreadyMapped => (7, 0, 0), + CapError::InvalidOperation => (8, 0, 0), + CapError::ASIDPoolExhausted => (9, 0, 0), + CapError::NoASIDAssigned => (10, 0, 0), + CapError::InvalidSlot(s) => (11, u64::from(s.0), 0), + CapError::EmptySlot(s) => (12, u64::from(s.0), 0), + CapError::SlotOccupied(s) => (13, u64::from(s.0), 0), + CapError::NotCoreType(t) => (14, u64::from(t.as_u8()), 0), + CapError::UnknownCoreType(t) => (15, u64::from(t), 0), + CapError::UnsupportedCoreType(t) => (16, u64::from(t.as_u8()), 0), + CapError::NotArchType(t) => (17, u64::from(t.as_u8()), 0), + CapError::UnknownArchType(t) => (18, u64::from(t), 0), + CapError::UnsupportedArchType(t) => (19, u64::from(t.as_u8()), 0), + CapError::InvalidObjectType(t) => (20, u64::from(t.as_u8()), 0), + CapError::TypeMismatch { expected, found } => { + (21, u64::from(expected.as_u8()), u64::from(found.as_u8())) + } + CapError::InsufficientMemory => (22, 0, 0), + CapError::PoolExhausted => (23, 0, 0), + CapError::InvalidSize(s) => (24, s.try_into().unwrap(), 0), + CapError::InvalidFrameSize(s) => (25, s.try_into().unwrap(), 0), + } + } +} diff --git a/libs/object/src/notification.rs b/libs/object/src/notification.rs new file mode 100644 index 000000000..7a22e6199 --- /dev/null +++ b/libs/object/src/notification.rs @@ -0,0 +1,45 @@ +use { + crate::key::Key, + libsyscall::{protected_call0, protected_call1}, +}; + +// ================================================== +// == Public user interface, usable from userspace == +// ================================================== + +/// Notification capability - bitmap-based async signaling. +/// Best for: IRQs, completion events, wakeups. +pub struct NotificationKey { + key: Key, +} + +enum NotificationType {} + +#[repr(u8)] +pub enum NotifyOp { + Signal = 0, + Wait = 1, + Poll = 2, +} + +impl NotificationKey { + /// Signal: atomic OR into bitmap (always non-blocking) + /// Multiple signals to same bit coalesce. + #[inline] + pub fn signal(&self, bits: u64) -> Result<()> { + protected_call1(self.key.slot(), NotifyOp::Signal as u32, bits)?; + Ok(()) + } + + /// Wait: block until any bit set, returns + clears ALL bits + #[inline] + pub fn wait(&self) -> Result { + protected_call0(self.key.slot(), NotifyOp::Wait as u32) + } + + /// Poll: non-blocking check, returns + clears bits + #[inline] + pub fn poll(&self) -> u64 { + protected_call0(self.key.slot(), NotifyOp::Poll as u32) + } +} diff --git a/libs/object/src/object_type.rs b/libs/object/src/object_type.rs new file mode 100644 index 000000000..5c64981f3 --- /dev/null +++ b/libs/object/src/object_type.rs @@ -0,0 +1,187 @@ +use crate::CapError; + +/// Object type discriminant with architectural bit. +/// +/// Bit 7 (high bit) indicates architecture-specific type. +/// +/// Layout: +/// +/// Bit 7 Bits 6-0 +/// ───── ──────── +/// 0 Core type (0-127) +/// 1 Arch type (0-127) +/// +#[repr(transparent)] +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub struct ObjectType(u8); + +impl ObjectType { + /// Bit indicating architecture-specific capability + pub const ARCH_BIT: u8 = 0x80; + + // ─── Core Types (0x00 - 0x7F) ─── + pub const NULL: Self = Self(0); + pub const UNTYPED: Self = Self(1); + pub const DOMAIN: Self = Self(2); + pub const KEY_TABLE: Self = Self(3); + pub const NOTIFICATION: Self = Self(4); + pub const EVENT_COUNT: Self = Self(5); + pub const ENDPOINT: Self = Self(6); + pub const TIME: Self = Self(7); + pub const BUFFER: Self = Self(8); + pub const REPLY: Self = Self(9); + // Reserved: 10-126 + pub const DEBUG_CONSOLE: Self = Self(127); // only #cfg(debug) + + // ─── Arch Types (0x80 - 0xFF) ─── + pub const FRAME: Self = Self(Self::ARCH_BIT); + pub const PAGE_TABLE: Self = Self(Self::ARCH_BIT | 1); + pub const VSPACE: Self = Self(Self::ARCH_BIT | 2); + pub const ASID_POOL: Self = Self(Self::ARCH_BIT | 3); + pub const ASID: Self = Self(Self::ARCH_BIT | 4); + pub const IO_SPACE: Self = Self(Self::ARCH_BIT | 5); + pub const IO_PORT: Self = Self(Self::ARCH_BIT | 6); // x86 only + pub const IRQ_HANDLER: Self = Self(Self::ARCH_BIT | 7); + pub const IRQ_CONTROL: Self = Self(Self::ARCH_BIT | 8); + // Reserved: 0x89 - 0xFF + + /// Check if this is an architecture-specific type. + #[inline(always)] + pub const fn is_arch(&self) -> bool { + (self.0 & Self::ARCH_BIT) != 0 + } + + /// Check if this is a core type. + #[inline(always)] + pub const fn is_core(&self) -> bool { + (self.0 & Self::ARCH_BIT) == 0 + } + + /// Get the type index within its category (strips arch bit). + #[inline(always)] + pub const fn index(&self) -> u8 { + self.0 & !Self::ARCH_BIT + } + + /// Raw value + #[inline(always)] + pub const fn as_u8(self) -> u8 { + self.0 + } +} + +// ═══════════════════════════════════════════════════════════════════ +// CORE TYPE ENUM (FOR MATCH) +// ═══════════════════════════════════════════════════════════════════ + +/// Core object types - used for match dispatch after arch check +#[repr(u8)] +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub enum CoreType { + /// No capability + Null = 0, + /// Creates objects (including new key tables) + Untyped = 1, + /// Protection domain + Domain = 2, + /// capability table itself + KeyTable = 3, + /// Time capability + Time = 4, + /// Endpoint capability + Endpoint = 5, + /// Notification endpoint capability + Notification = 6, + /// Event count endpoint capability + EventCount = 7, + /// Shareable buffer capability + Buffer = 8, + Reply = 9, + DebugConsole = 127, +} + +impl CoreType { + /// Raw value + #[inline(always)] + pub const fn as_u8(self) -> u8 { + self as u8 + } +} + +impl TryFrom for CoreType { + type Error = CapError; + + #[inline] + fn try_from(ot: ObjectType) -> Result { + if ot.is_arch() { + return Err(CapError::NotCoreType(ot)); + } + match ot.index() { + 0 => Ok(CoreType::Null), + 1 => Ok(CoreType::Untyped), + 2 => Ok(CoreType::Domain), + 3 => Ok(CoreType::KeyTable), + 4 => Ok(CoreType::Notification), + 5 => Ok(CoreType::EventCount), + 6 => Ok(CoreType::Endpoint), + 7 => Ok(CoreType::Time), + 8 => Ok(CoreType::Buffer), + 9 => Ok(CoreType::Reply), + 127 => Ok(CoreType::DebugConsole), + _ => Err(CapError::UnknownCoreType(ot.index())), + } + } +} + +// ═══════════════════════════════════════════════════════════════════ +// ARCH TYPE ENUM (ARCHITECTURE-SPECIFIC) +// ═══════════════════════════════════════════════════════════════════ + +/// Architecture-specific object types +/// +/// This is defined per-architecture but the indices are the same. +/// The actual struct types differ per architecture. +#[repr(u8)] +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub enum ArchType { + Frame = 0, + PageTable = 1, + VSpace = 2, + ASIDPool = 3, + ASID = 4, + IOSpace = 5, + IOPort = 6, + IRQHandler = 7, + IRQControl = 8, +} + +impl ArchType { + /// Raw value + #[inline(always)] + pub const fn as_u8(self) -> u8 { + self as u8 + } +} + +impl TryFrom for ArchType { + type Error = CapError; + + #[inline] + fn try_from(ot: ObjectType) -> Result { + if !ot.is_arch() { + return Err(CapError::NotArchType(ot)); + } + match ot.index() { + 0 => Ok(ArchType::Frame), + 1 => Ok(ArchType::PageTable), + 2 => Ok(ArchType::VSpace), + 3 => Ok(ArchType::ASIDPool), + 4 => Ok(ArchType::ASID), + 5 => Ok(ArchType::IOSpace), + 6 => Ok(ArchType::IOPort), + 7 => Ok(ArchType::IRQHandler), + 8 => Ok(ArchType::IRQControl), + _ => Err(CapError::UnknownArchType(ot.index())), + } + } +} diff --git a/libs/object/src/reply.rs b/libs/object/src/reply.rs new file mode 100644 index 000000000..bdbb19122 --- /dev/null +++ b/libs/object/src/reply.rs @@ -0,0 +1,93 @@ +// ================================================== +// == Public user interface, usable from userspace == +// ================================================== + +/// Reply capability - one-shot reply to a blocked caller. +/// +/// Created by kernel when a Call arrives, consumed when reply is sent. +/// This is a LINEAR type - must be used exactly once (or explicitly dropped). +/// +/// Making reply explicit enables: +/// - Async reply (store reply cap, reply later) +/// - Delegation (pass reply cap to helper domain) +/// - Multiple outstanding calls (each has own reply cap) +pub struct ReplyKey { + key: Key, +} + +enum ReplyType {} + +#[repr(u8)] +pub enum ReplyOp { + Send = 0, // Send reply message + SendWithCap = 1, // Send reply with cap transfer + SendError = 2, // Send error (used by Drop) +} + +impl ReplyKey { + /// Send reply to the blocked caller + /// + /// Consumes self - reply cap is one-shot! + /// After this, the caller is unblocked with the response. + pub fn send(self, msg: &Message) -> Result<(), IpcError> { + let ret = unsafe { + protected_call4( + self.key.slot(), + ReplyOp::Send as u64, + msg.label, + msg.data[0], + msg.data[1], + msg.data[2], + ) + }; + + // Consumed - don't run Drop (kernel already invalidated) + core::mem::forget(self); + + IpcError::from_code(ret.0) + } + + /// Send reply with capability transfer + pub fn send_with_key(self, msg: &Message, key: KeySlot) -> Result<(), IpcError> { + let ret = unsafe { + syscall_ipc_reply( + self.key.slot(), + ReplyOp::SendWithCap as u64, + msg.label, + msg.data[0], + msg.data[1], + msg.data[2], + msg.data[3], + msg.data[4], + key as u64, + ) + }; + + // Consumed - don't run Drop (kernel already invalidated) + core::mem::forget(self); + + IpcError::from_code(ret.0) + } +} + +/// Dropping a ReplyKey without sending is an ERROR for the caller. +/// The caller would remains blocked forever (or until timeout/cancellation). +/// +/// In debug builds, we panic. In release, we send an error reply. +impl Drop for ReplyKey { + fn drop(&mut self) { + // Reply cap dropped without sending - this is usually a bug! + // Send error reply to unblock caller + unsafe { + protected_call1( + self.key.slot(), + ReplyOp::SendError as u64, + IpcError::ReplyDropped as u64, + ); + } + + // Panic in debug mode to detect misuse. + #[cfg(debug_assertions)] + panic!("ReplyKey dropped without sending reply!"); + } +} diff --git a/libs/object/src/rights.rs b/libs/object/src/rights.rs new file mode 100644 index 000000000..d679e5257 --- /dev/null +++ b/libs/object/src/rights.rs @@ -0,0 +1,22 @@ +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct Rights(pub u8); + +impl Rights { + pub const READ: u8 = 0x1; + pub const WRITE: u8 = 0x2; + pub const MAP: u8 = 0x2; + pub const SEND: u8 = 0x2; + pub const RECV: u8 = 0x1; + pub const CALL: u8 = 0x4; + pub const GRANT: u8 = 0x8; + + pub const fn empty() -> Rights { + Rights(0) + } + pub const fn all() -> Rights { + Rights(0xF) + } + pub fn bits(&self) -> u8 { + self.0 + } +} diff --git a/libs/object/src/time.rs b/libs/object/src/time.rs new file mode 100644 index 000000000..9ef88ca8e --- /dev/null +++ b/libs/object/src/time.rs @@ -0,0 +1,66 @@ +// ================================================== +// == Public user interface, usable from userspace == +// ================================================== + +/// Time capability — represents a slice of CPU time +pub struct TimeKey { + key: Key, +} + +enum TimeType {} + +#[repr(u8)] +pub enum TimeOp { + Donate = 0, + Split = 1, + Merge = 2, + Query = 3, +} + +impl TimeKey { + /// Donate this time slice to target domain. + /// Current domain blocks until target exhausts time or yields. + pub fn donate(self, target: DomainKey) -> Result<(), SchedError> { + let (ret, _, _) = + unsafe { protected_call1(self.key.slot(), TimeOp::Donate as u64, target.key.slot()) }; + // Note: self is consumed (moved) - FIXME + Error::from_code(ret) + } + + /// Split off 'amount_us' into a new TimeCap. + /// Self retains the remainder. + pub fn split(&mut self, amount_us: u64) -> Result { + let new_slot = self.alloc_slot()?; // FIXME: In-kernel memory allocation? + let (ret, _, _) = unsafe { + protected_call2( + self.key.slot(), + TimeOp::Split as u64, + amount_us, + new_slot as u64, + ) + }; + if ret == 0 { + Ok(TimeKey { + key: Key::new(new_slot), + }) + } else { + Err(SchedError::from_code(ret)) + } + } + + /// Query remaining time in microseconds. + pub fn remaining(&self) -> u64 { + let (_, us, _) = unsafe { protected_call0(self.key.slot(), TimeOp::Query as u64) }; + us + } +} + +// Yield is just dropping + returning to scheduler +impl Drop for TimeKey { + fn drop(&mut self) { + // Kernel: return remaining time to parent in derivation tree + unsafe { + protected_call1(KeySlot::CAPTBL_SELF, KeyTableOp::Delete, self.key.slot()); + } + } +} diff --git a/libs/object/src/untyped.rs b/libs/object/src/untyped.rs new file mode 100644 index 000000000..cce81659b --- /dev/null +++ b/libs/object/src/untyped.rs @@ -0,0 +1,85 @@ +// ================================================== +// == Public user interface, usable from userspace == +// ================================================== + +pub struct UntypedKey { + key: Key, +} + +enum UntypedType {} + +#[repr(u8)] +pub enum UntypedOp { + Retype = 0, +} + +// Errors that can occur during retype operation +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[repr(u32)] +pub enum RetypeError { + /// Invalid object type specified + InvalidObjectType = 1, + /// Requested size is invalid for object type + InvalidSize = 2, + /// Not enough memory remaining in untyped + InsufficientMemory = 3, + /// Alignment requirements not met + AlignmentError = 4, + /// Destination slot already occupied + SlotOccupied = 5, + /// Invalid destination slot + InvalidSlot = 6, + /// Untyped has already been retyped (has children) + AlreadyRetyped = 7, + /// Object type requires specific size_bits + SizeMismatch = 8, + /// Maximum number of objects reached + ObjectLimitReached = 9, + /// Internal kernel error + InternalError = 10, +} + +impl RetypeError { + pub fn code(self) -> u32 { + self as u32 + } +} + +// ┌────────────────────────────────────────────────────────────┐ +// │ ALLOWED OPERATIONS ON UNTYPED │ +// ├────────────────────────────────────────────────────────────┤ +// │ ✓ Untyped_Retype → Create children (objects/sub-untypeds)│ +// │ ✓ CNode_Revoke → Delete all children, reset watermark │ +// │ ✓ CNode_Delete → Delete this cap (if no children) │ +// │ ✓ CNode_Move → Move cap to different slot │ +// ├────────────────────────────────────────────────────────────┤ +// │ DISALLOWED │ +// ├────────────────────────────────────────────────────────────┤ +// │ ✗ CNode_Copy → Cannot duplicate │ +// │ ✗ CNode_Mint → Cannot derive with reduced rights │ +// │ ✗ CNode_Mutate → Cannot modify │ +// └────────────────────────────────────────────────────────────┘ + +impl UntypedKey { + /// Retype untyped memory into a typed nucleus object. + /// + /// This is how ALL nucleus objects are created. + /// The untyped capability is consumed/reduced by the operation. + pub fn retype( + &self, + object_type: ObjectType, + size_bits: u8, // log2 of size (for variable-size objects) + dest_slot: CapSlot, + ) -> Result<(), RetypeError> { + let ret = unsafe { + libsyscall::protected_call3( + self.key.slot(), + UntypedOp::Retype, + object_type as u64, + dest_slot as u64, + size_bits as u64, + ) + }; + Error::from_code(ret) + } +} diff --git a/libs/platform/Cargo.toml b/libs/platform/Cargo.toml index cc3142a69..69b5d66c8 100644 --- a/libs/platform/Cargo.toml +++ b/libs/platform/Cargo.toml @@ -1,6 +1,6 @@ [package] -name = "libplatform" -description = "Vesper nanokernel platform code." +name = "vesper-platforms" +description = "Vesper nanokernel platform abstraction code." authors = { workspace = true } categories = { workspace = true } documentation = { workspace = true } @@ -28,34 +28,29 @@ test_build = [] [dependencies] aarch64-cpu = { workspace = true } -bit_field = { workspace = true } -bitflags = { workspace = true } -buddy-alloc = { workspace = true } -cfg-if = { workspace = true } +libaddress = { workspace = true } libconsole = { workspace = true } libcpu = { workspace = true } +libdriver = { workspace = true } libexception = { workspace = true } libkernel-state = { workspace = true } -liblocal-irq = { workspace = true } liblocking = { workspace = true } liblog = { workspace = true } -libmemory = { workspace = true } -libdriver = { workspace = true } +libmapping = { workspace = true } +libmmio = { workspace = true } libprimitives = { workspace = true } libtime = { workspace = true } -once_cell = { workspace = true } -qemu-exit = { workspace = true } snafu = { workspace = true } tock-registers = { workspace = true } -usize_conversions = { workspace = true } -ux = { workspace = true } [dev-dependencies] -libtest = { workspace = true } +libqemu = { workspace = true } +# libtest = { workspace = true } -# Tests are offloaded to nucleus integration tests binary. +# Tests are offloaded to kernel integration tests binary. [lib] test = false +harness = false [lints] workspace = true diff --git a/libs/platform/README.md b/libs/platform/README.md new file mode 100644 index 000000000..04a966c95 --- /dev/null +++ b/libs/platform/README.md @@ -0,0 +1,7 @@ +# Board Support Packages + +This library contains support for specific Boards like RaspberryPi3 etc. + +---- + +For more information please re-read. diff --git a/libs/platform/src/lib.rs b/libs/platform/src/lib.rs index e6e57f94f..f88ca473c 100644 --- a/libs/platform/src/lib.rs +++ b/libs/platform/src/lib.rs @@ -1,10 +1,19 @@ +/* + * SPDX-License-Identifier: BlueOak-1.0.0 + * Copyright (c) Berkus Decker + */ + #![no_std] #![no_main] #![feature(decl_macro)] #![feature(allocator_api)] #![feature(format_args_nl)] -#![feature(custom_test_frameworks)] -#![test_runner(libtest::test_runner)] -#![reexport_test_harness_main = "test_main"] +// #![feature(custom_test_frameworks)] +// #![test_runner(libtest::test_runner)] +// #![reexport_test_harness_main = "test_main"] + +#[cfg(any(board_rpi3, board_rpi4, test, feature = "test_build"))] +pub mod raspberrypi; -pub mod platform; // @todo flatten! +#[cfg(any(board_rpi3, board_rpi4, test, feature = "test_build"))] +pub use raspberrypi::*; diff --git a/libs/platform/src/platform/README.md b/libs/platform/src/platform/README.md deleted file mode 100644 index 39247568b..000000000 --- a/libs/platform/src/platform/README.md +++ /dev/null @@ -1,7 +0,0 @@ -# Board Support Packages - -This directory contains support for specific Boards like RaspberryPi3 etc. - ----- - -For more information please re-read. diff --git a/libs/platform/src/platform/mod.rs b/libs/platform/src/platform/mod.rs deleted file mode 100644 index 5d0aacb8f..000000000 --- a/libs/platform/src/platform/mod.rs +++ /dev/null @@ -1,10 +0,0 @@ -/* - * SPDX-License-Identifier: BlueOak-1.0.0 - * Copyright (c) Berkus Decker - */ - -#[cfg(any(board_rpi3, board_rpi4))] -pub mod raspberrypi; - -#[cfg(any(board_rpi3, board_rpi4))] -pub use raspberrypi::*; diff --git a/libs/platform/src/platform/raspberrypi/linker/kernel.ld b/libs/platform/src/platform/raspberrypi/linker/kernel.ld deleted file mode 100644 index d59505e81..000000000 --- a/libs/platform/src/platform/raspberrypi/linker/kernel.ld +++ /dev/null @@ -1,152 +0,0 @@ -/* - * SPDX-License-Identifier: MIT OR BlueOak-1.0.0 - * Copyright (c) 2018 Andre Richter - * Copyright (c) Berkus Decker - * Original code distributed under MIT, additional changes are under BlueOak-1.0.0 - */ - -PAGE_SIZE = 64K; -PAGE_MASK = PAGE_SIZE - 1; - -__phys_mem_start = 0x0; - -__phys_load_addr = 0x80000; -ENTRY(__phys_load_addr); - -/* Flags: - * 4 == R - * 5 == RX - * 6 == RW - * - * Segments are marked PT_LOAD below so that the ELF file provides virtual and physical addresses. - * It doesn't mean all of them need actually be loaded. - */ -PHDRS -{ - segment_boot_core_stack PT_LOAD FLAGS(6); - segment_code PT_LOAD FLAGS(5); - segment_data PT_LOAD FLAGS(6); -} - -/* Symbols between __BOOT_START and __BOOT_END should be dropped after init is complete. - Symbols between __CODE_START and __CODE_END are the kernel code. - Symbols between __BSS_START and __BSS_END must be initialized to zero by startup code in the kernel. -*/ -SECTIONS -{ - . = __phys_mem_start; - - /*********************************************************************************************** - * Boot Core Stack - ***********************************************************************************************/ - .boot_core_stack (NOLOAD) : - { - __STACK_BOTTOM = .; /* ^ */ - /* | stack */ - . = __phys_load_addr; /* | growth AArch64 boot address is 0x80000, 4K-aligned */ - /* | direction */ - __STACK_TOP = .; /* | Stack grows from here towards 0x0. */ - } :segment_boot_core_stack - - ASSERT((__STACK_TOP & PAGE_MASK) == 0, "End of boot core stack is not page aligned") - - /*********************************************************************************************** - * Code + RO Data - ***********************************************************************************************/ - - .text : ALIGN(PAGE_SIZE) - { - /******************************************************************************************* - * Boot Code + Boot Data - *******************************************************************************************/ - - __BOOT_START = .; - - KEEP(*(.text.main.entry)) - *(.text.boot) - *(.data.boot) - - . = ALIGN(PAGE_SIZE); - - __BOOT_END = .; /* Here the boot code ends */ - ASSERT((__BOOT_END & PAGE_MASK) == 0, "End of boot code is not page aligned") - - /******************************************************************************************* - * Regular Kernel Code - *******************************************************************************************/ - - __CODE_START = .; - - *(.text*) - - . = ALIGN(2048); - } :segment_code - - .vectors : ALIGN(2048) - { - __EXCEPTION_VECTORS_START = .; - - KEEP(*(.vectors)) - - . = ALIGN(4); - } :segment_code - - .rodata : ALIGN(4) - { - *(.rodata*) - FILL(0x00) - - . = ALIGN(PAGE_SIZE); /* Fill up to page size */ - - __CODE_END = .; - ASSERT((__CODE_END & PAGE_MASK) == 0, "End of kernel code is not page aligned") - } :segment_code - - /*********************************************************************************************** - * Data + BSS - ***********************************************************************************************/ - - .data : ALIGN(PAGE_SIZE) - { - __DATA_START = .; - ASSERT((__DATA_START & PAGE_MASK) == 0, "Start of kernel data is not page aligned") - - *(.data*) - FILL(0x00) - - . = ALIGN(PAGE_SIZE); - } :segment_data - - .bss (NOLOAD) : ALIGN(PAGE_SIZE) - { - __BSS_START = .; - - *(.bss*) - - . = ALIGN(PAGE_SIZE); /* Align up to page size */ - __BSS_END = .; - __BSS_SIZE_U64S = (__BSS_END - __BSS_START) / 8; /* TODO: remove */ - } :segment_data - - __DATA_END = .; - - /*********************************************************************************************** - * MMIO Remap Reserved (8Mb) - ***********************************************************************************************/ - __MMIO_REMAP_START = .; - . += 8 * 1024 * 1024; - __MMIO_REMAP_END = .; - - ASSERT((. & PAGE_MASK) == 0, "MMIO remap reservation is not page aligned") - - /*********************************************************************************************** - * Misc - ***********************************************************************************************/ - - .got : { *(.got*) } - ASSERT(SIZEOF(.got) == 0, "Unexpected relocations in kernel binary.") - - /DISCARD/ : { *(.comment*) *(.gnu*) *(.note*) *(.eh_frame*) *(.text.chainboot*) } -} - -INCLUDE libs/exception/src/arch/aarch64/linker/aarch64-exceptions.ld diff --git a/libs/platform/src/platform/raspberrypi/cpu.rs b/libs/platform/src/raspberrypi/cpu.rs similarity index 100% rename from libs/platform/src/platform/raspberrypi/cpu.rs rename to libs/platform/src/raspberrypi/cpu.rs diff --git a/libs/platform/src/platform/raspberrypi/device_driver/arm/gicv2/gicc.rs b/libs/platform/src/raspberrypi/device_driver/arm/gicv2/gicc.rs similarity index 94% rename from libs/platform/src/platform/raspberrypi/device_driver/arm/gicv2/gicc.rs rename to libs/platform/src/raspberrypi/device_driver/arm/gicv2/gicc.rs index 61eee7700..add1f7253 100644 --- a/libs/platform/src/platform/raspberrypi/device_driver/arm/gicv2/gicc.rs +++ b/libs/platform/src/raspberrypi/device_driver/arm/gicv2/gicc.rs @@ -5,8 +5,8 @@ //! GICC Driver - GIC CPU interface. use { - crate::platform::device_driver::common::MMIODerefWrapper, - libmemory::{Address, Virtual}, + libaddress::{Address, Virtual}, + libmmio::MMIODerefWrapper, tock_registers::{ interfaces::{Readable, Writeable}, register_bitfields, register_structs, @@ -119,7 +119,7 @@ impl GICC { #[allow(clippy::trivially_copy_pass_by_ref)] pub fn pending_irq_number<'irq_context>( &self, - _ic: &libexception::exception::asynchronous::IRQContext<'irq_context>, + _ic: &libexception::asynchronous::IRQContext<'irq_context>, ) -> usize { self.registers.IAR.read(IAR::InterruptID) as usize } @@ -138,7 +138,7 @@ impl GICC { pub fn mark_completed<'irq_context>( &self, irq_number: u32, - _ic: &libexception::exception::asynchronous::IRQContext<'irq_context>, + _ic: &libexception::asynchronous::IRQContext<'irq_context>, ) { self.registers.EOIR.write(EOIR::EOIINTID.val(irq_number)); } diff --git a/libs/platform/src/platform/raspberrypi/device_driver/arm/gicv2/gicd.rs b/libs/platform/src/raspberrypi/device_driver/arm/gicv2/gicd.rs similarity index 89% rename from libs/platform/src/platform/raspberrypi/device_driver/arm/gicv2/gicd.rs rename to libs/platform/src/raspberrypi/device_driver/arm/gicv2/gicd.rs index 7d2b73ab6..b1f425685 100644 --- a/libs/platform/src/platform/raspberrypi/device_driver/arm/gicv2/gicd.rs +++ b/libs/platform/src/raspberrypi/device_driver/arm/gicv2/gicd.rs @@ -8,9 +8,9 @@ //! - SPI - Shared Peripheral Interrupt. use { - crate::platform::device_driver::common::MMIODerefWrapper, + libaddress::{Address, Virtual}, liblocking::IRQSafeNullLock, - libmemory::{Address, Virtual}, + libmmio::MMIODerefWrapper, tock_registers::{ interfaces::{Readable, Writeable}, register_bitfields, register_structs, @@ -69,10 +69,10 @@ register_structs! { } /// Abstraction for the non-banked parts of the associated MMIO registers. -type SharedRegisters = MMIODerefWrapper; +struct SharedRegisters(MMIODerefWrapper); /// Abstraction for the banked parts of the associated MMIO registers. -type BankedRegisters = MMIODerefWrapper; +struct BankedRegisters(MMIODerefWrapper); //-------------------------------------------------------------------------------------------------- // Public Definitions @@ -99,7 +99,7 @@ impl SharedRegisters { // Query number of implemented IRQs. // // Refer to GICv2 Architecture Specification, Section 4.3.2. - ((self.TYPER.read(TYPER::ITLinesNumber) as usize) + 1) * 32 + ((self.0.TYPER.read(TYPER::ITLinesNumber) as usize) + 1) * 32 } /// Return a slice of the implemented ITARGETSR. @@ -115,7 +115,7 @@ impl SharedRegisters { let spi_itargetsr_max_index = ((self.num_irqs() - 32) >> 2) - 1; // Rust automatically inserts slice range sanity check, i.e. max >= min. - &self.ITARGETSR[0..spi_itargetsr_max_index] + &self.0.ITARGETSR[0..spi_itargetsr_max_index] } } @@ -132,8 +132,10 @@ impl GICD { /// - The user must ensure to provide a correct MMIO start address. pub const unsafe fn new(mmio_start_addr: Address) -> Self { Self { - shared_registers: IRQSafeNullLock::new(SharedRegisters::new(mmio_start_addr)), - banked_registers: BankedRegisters::new(mmio_start_addr), + shared_registers: IRQSafeNullLock::new(SharedRegisters(MMIODerefWrapper::new( + mmio_start_addr, + ))), + banked_registers: BankedRegisters(MMIODerefWrapper::new(mmio_start_addr)), } } @@ -144,7 +146,7 @@ impl GICD { /// "`GICD_ITARGETSR0` to `GICD_ITARGETSR7` are read-only, and each field returns a value that /// corresponds only to the processor reading the register." fn local_gic_target_mask(&self) -> u32 { - self.banked_registers.ITARGETSR[0].read(ITARGETSR::Offset0) + self.banked_registers.0.ITARGETSR[0].read(ITARGETSR::Offset0) } /// Route all SPIs to the boot core and enable the distributor. @@ -167,7 +169,7 @@ impl GICD { ); } - regs.CTLR.write(CTLR::Enable::SET); + regs.0.CTLR.write(CTLR::Enable::SET); }); } @@ -184,7 +186,7 @@ impl GICD { match irq_num { // Private. 0..=31 => { - let enable_reg = &self.banked_registers.ISENABLER; + let enable_reg = &self.banked_registers.0.ISENABLER; enable_reg.set(enable_reg.get() | enable_bit); } // Shared. @@ -192,7 +194,7 @@ impl GICD { let enable_reg_index_shared = enable_reg_index - 1; self.shared_registers.lock(|regs| { - let enable_reg = ®s.ISENABLER[enable_reg_index_shared]; + let enable_reg = ®s.0.ISENABLER[enable_reg_index_shared]; enable_reg.set(enable_reg.get() | enable_bit); }); } diff --git a/libs/platform/src/platform/raspberrypi/device_driver/arm/gicv2/mod.rs b/libs/platform/src/raspberrypi/device_driver/arm/gicv2/mod.rs similarity index 97% rename from libs/platform/src/platform/raspberrypi/device_driver/arm/gicv2/mod.rs rename to libs/platform/src/raspberrypi/device_driver/arm/gicv2/mod.rs index 5796ee5d6..de7ce9bf1 100644 --- a/libs/platform/src/platform/raspberrypi/device_driver/arm/gicv2/mod.rs +++ b/libs/platform/src/raspberrypi/device_driver/arm/gicv2/mod.rs @@ -80,12 +80,10 @@ mod gicc; mod gicd; use { - crate::platform::cpu::BOOT_CORE_ID, - libexception::exception::asynchronous::{ - IRQContext, IRQHandlerDescriptor, interface::IRQManager, - }, + crate::cpu::BOOT_CORE_ID, + libaddress::{Address, Virtual}, + libexception::asynchronous::{IRQContext, IRQHandlerDescriptor, interface::IRQManager}, liblocking::{self, InitStateLock}, - libmemory::{Address, Virtual}, libprimitives::BoundedUsize, }; diff --git a/libs/platform/src/platform/raspberrypi/device_driver/arm/mod.rs b/libs/platform/src/raspberrypi/device_driver/arm/mod.rs similarity index 100% rename from libs/platform/src/platform/raspberrypi/device_driver/arm/mod.rs rename to libs/platform/src/raspberrypi/device_driver/arm/mod.rs diff --git a/libs/platform/src/platform/raspberrypi/device_driver/bcm/gpio.rs b/libs/platform/src/raspberrypi/device_driver/bcm/gpio.rs similarity index 99% rename from libs/platform/src/platform/raspberrypi/device_driver/bcm/gpio.rs rename to libs/platform/src/raspberrypi/device_driver/bcm/gpio.rs index 51400ada5..2753f4088 100644 --- a/libs/platform/src/platform/raspberrypi/device_driver/bcm/gpio.rs +++ b/libs/platform/src/raspberrypi/device_driver/bcm/gpio.rs @@ -6,10 +6,11 @@ */ use { - crate::platform::device_driver::{IRQNumber, common::MMIODerefWrapper}, + crate::device_driver::IRQNumber, core::marker::PhantomData, + libaddress::{Address, Virtual}, liblocking::{IRQSafeNullLock, interface::Mutex}, - libmemory::{Address, Virtual}, + libmmio::MMIODerefWrapper, tock_registers::{ fields::FieldValue, interfaces::{ReadWriteable, Readable, Writeable}, diff --git a/libs/platform/src/platform/raspberrypi/device_driver/bcm/interrupt_controller/mod.rs b/libs/platform/src/raspberrypi/device_driver/bcm/interrupt_controller/mod.rs similarity index 93% rename from libs/platform/src/platform/raspberrypi/device_driver/bcm/interrupt_controller/mod.rs rename to libs/platform/src/raspberrypi/device_driver/bcm/interrupt_controller/mod.rs index 20cc1535b..faf5554ab 100644 --- a/libs/platform/src/platform/raspberrypi/device_driver/bcm/interrupt_controller/mod.rs +++ b/libs/platform/src/raspberrypi/device_driver/bcm/interrupt_controller/mod.rs @@ -8,8 +8,8 @@ mod peripheral_ic; use { core::fmt, - libexception::exception::{self, asynchronous::IRQHandlerDescriptor}, - libmemory::{Address, Virtual}, + libaddress::{Address, Virtual}, + libexception::asynchronous::{IRQHandlerDescriptor, interface::IRQManager}, libprimitives::BoundedUsize, }; @@ -111,12 +111,12 @@ impl libdriver::drivers::interface::DeviceDriver for InterruptController { } } -impl exception::asynchronous::interface::IRQManager for InterruptController { +impl IRQManager for InterruptController { type IRQNumberType = IRQNumber; fn register_handler( &self, - irq_handler_descriptor: exception::asynchronous::IRQHandlerDescriptor, + irq_handler_descriptor: IRQHandlerDescriptor, ) -> Result<(), &'static str> { match irq_handler_descriptor.number() { IRQNumber::Local(_) => unimplemented!("Local IRQ controller not implemented."), @@ -142,7 +142,7 @@ impl exception::asynchronous::interface::IRQManager for InterruptController { fn handle_pending_irqs<'irq_context>( &'irq_context self, - ic: &exception::asynchronous::IRQContext<'irq_context>, + ic: &libexception::asynchronous::IRQContext<'irq_context>, ) { // It can only be a peripheral IRQ pending because enable() does not support local IRQs yet. self.periph.handle_pending_irqs(ic); diff --git a/libs/platform/src/platform/raspberrypi/device_driver/bcm/interrupt_controller/peripheral_ic.rs b/libs/platform/src/raspberrypi/device_driver/bcm/interrupt_controller/peripheral_ic.rs similarity index 96% rename from libs/platform/src/platform/raspberrypi/device_driver/bcm/interrupt_controller/peripheral_ic.rs rename to libs/platform/src/raspberrypi/device_driver/bcm/interrupt_controller/peripheral_ic.rs index 1c1cf70a9..e2892bc4d 100644 --- a/libs/platform/src/platform/raspberrypi/device_driver/bcm/interrupt_controller/peripheral_ic.rs +++ b/libs/platform/src/raspberrypi/device_driver/bcm/interrupt_controller/peripheral_ic.rs @@ -10,11 +10,9 @@ use { super::{PendingIRQs, PeripheralIRQ}, - crate::platform::device_driver::common::MMIODerefWrapper, - libexception::exception::asynchronous::{ - IRQContext, IRQHandlerDescriptor, interface::IRQManager, - }, + libexception::asynchronous::{IRQContext, IRQHandlerDescriptor, interface::IRQManager}, liblocking::{self, IRQSafeNullLock, InitStateLock}, + libmmio::MMIODerefWrapper, tock_registers::{ interfaces::{Readable, Writeable}, register_structs, @@ -101,8 +99,8 @@ impl PeripheralIC { // OS Interface Code //------------------------------------------------------------------------------ use { + libaddress::{Address, Virtual}, liblocking::interface::{Mutex, ReadWriteEx}, - libmemory::{Address, Virtual}, }; impl IRQManager for PeripheralIC { diff --git a/libs/platform/src/platform/raspberrypi/device_driver/bcm/mailbox.rs b/libs/platform/src/raspberrypi/device_driver/bcm/mailbox.rs similarity index 98% rename from libs/platform/src/platform/raspberrypi/device_driver/bcm/mailbox.rs rename to libs/platform/src/raspberrypi/device_driver/bcm/mailbox.rs index debe4fabe..a281db514 100644 --- a/libs/platform/src/platform/raspberrypi/device_driver/bcm/mailbox.rs +++ b/libs/platform/src/raspberrypi/device_driver/bcm/mailbox.rs @@ -12,7 +12,7 @@ #![allow(dead_code)] use { - crate::platform::{BcmHost, device_driver::common::MMIODerefWrapper}, + crate::BcmHost, aarch64_cpu::asm::barrier, core::{ // alloc::{AllocError, Allocator, Layout}, @@ -21,8 +21,9 @@ use { result::Result as CoreResult, sync::atomic::{Ordering, compiler_fence}, }, + libmmio::MMIODerefWrapper, // liblocking::IRQSafeNullLock, - // libmemory::{Address, Virtual}, + // libaddress::{Address, Virtual}, snafu::Snafu, tock_registers::{ interfaces::{Readable, Writeable}, @@ -150,7 +151,7 @@ impl MailboxStorage for LocalMailboxStorage { impl MailboxStorage for DmaBackedMailboxStorage { fn new() -> Result { - use libmemory::platform::memory::map::virt::DMA_HEAP_START; + use crate::memory::map::virt::DMA_HEAP_START; Ok(Self { storage: DMA_HEAP_START @@ -384,7 +385,7 @@ impl Mailbox< /// # Safety /// Caller is responsible for picking the correct MMIO register base address. pub unsafe fn new(base_addr: usize) -> Result> { - let base = libmemory::Address::new(base_addr); + let base = libaddress::Address::new(base_addr as u64); Ok(Mailbox { registers: Registers::new(base), buffer: Storage::new()?, diff --git a/libs/platform/src/platform/raspberrypi/device_driver/bcm/mailbox_refactor.rs b/libs/platform/src/raspberrypi/device_driver/bcm/mailbox_refactor.rs similarity index 94% rename from libs/platform/src/platform/raspberrypi/device_driver/bcm/mailbox_refactor.rs rename to libs/platform/src/raspberrypi/device_driver/bcm/mailbox_refactor.rs index a2dfee48b..80a78aee7 100644 --- a/libs/platform/src/platform/raspberrypi/device_driver/bcm/mailbox_refactor.rs +++ b/libs/platform/src/raspberrypi/device_driver/bcm/mailbox_refactor.rs @@ -26,7 +26,7 @@ use { mem, ptr::NonNull, result::Result as CoreResult, - sync::atomic::{compiler_fence, Ordering}, + sync::atomic::{Ordering, compiler_fence}, }, snafu::Snafu, tock_registers::{ @@ -123,9 +123,9 @@ pub trait MailboxOps { fn write(&self, channel: u32) -> Result<()>; fn read(&self, channel: u32) -> Result<()>; fn call(&self, channel: u32) -> Result<()>; //{ - // self.write(channel)?; - // self.read(channel) - // } + // self.write(channel)?; + // self.read(channel) + // } } pub trait MailboxStorage { @@ -671,30 +671,30 @@ impl MailboxS } } -#[cfg(test)] -mod tests { - use super::*; - - // Validate the buffer is filled correctly - // Validate the buffer is properly terminated when call()ed -- this invariant must be maintained - // by the end() fn. - #[test_case] - fn test_prepare_mailbox() { - let mut mailbox = Mailbox::<8>::default(); - let index = mailbox.request(); - let index = mailbox.set_led_on(index, true); - let mailbox = mailbox.end(index); - // Instead of calling just check the filled buffer format: - assert_eq!( - unsafe { mailbox.0.buffer.as_ref()[0] } as usize, - (index + 1) * 4 - ); - assert_eq!(unsafe { mailbox.0.buffer.as_ref()[1] }, REQUEST); - assert_eq!(unsafe { mailbox.0.buffer.as_ref()[2] }, tag::SetGpioState); - assert_eq!(unsafe { mailbox.0.buffer.as_ref()[3] }, 8); - assert_eq!(unsafe { mailbox.0.buffer.as_ref()[4] }, 0); - assert_eq!(unsafe { mailbox.0.buffer.as_ref()[5] }, 130); - assert_eq!(unsafe { mailbox.0.buffer.as_ref()[6] }, 1); - assert_eq!(unsafe { mailbox.0.buffer.as_ref()[7] }, tag::End); - } -} +// #[cfg(test)] +// mod tests { +// use super::*; + +// // Validate the buffer is filled correctly +// // Validate the buffer is properly terminated when call()ed -- this invariant must be maintained +// // by the end() fn. +// #[test_case] +// fn test_prepare_mailbox() { +// let mut mailbox = Mailbox::<8>::default(); +// let index = mailbox.request(); +// let index = mailbox.set_led_on(index, true); +// let mailbox = mailbox.end(index); +// // Instead of calling just check the filled buffer format: +// assert_eq!( +// unsafe { mailbox.0.buffer.as_ref()[0] } as usize, +// (index + 1) * 4 +// ); +// assert_eq!(unsafe { mailbox.0.buffer.as_ref()[1] }, REQUEST); +// assert_eq!(unsafe { mailbox.0.buffer.as_ref()[2] }, tag::SetGpioState); +// assert_eq!(unsafe { mailbox.0.buffer.as_ref()[3] }, 8); +// assert_eq!(unsafe { mailbox.0.buffer.as_ref()[4] }, 0); +// assert_eq!(unsafe { mailbox.0.buffer.as_ref()[5] }, 130); +// assert_eq!(unsafe { mailbox.0.buffer.as_ref()[6] }, 1); +// assert_eq!(unsafe { mailbox.0.buffer.as_ref()[7] }, tag::End); +// } +// } diff --git a/libs/platform/src/platform/raspberrypi/device_driver/bcm/mini_uart.rs b/libs/platform/src/raspberrypi/device_driver/bcm/mini_uart.rs similarity index 98% rename from libs/platform/src/platform/raspberrypi/device_driver/bcm/mini_uart.rs rename to libs/platform/src/raspberrypi/device_driver/bcm/mini_uart.rs index 1ad9f227a..7c92f0878 100644 --- a/libs/platform/src/platform/raspberrypi/device_driver/bcm/mini_uart.rs +++ b/libs/platform/src/raspberrypi/device_driver/bcm/mini_uart.rs @@ -8,18 +8,15 @@ // #[cfg(not(feature = "noserial"))] // use tock_registers::interfaces::{Readable, Writeable}; use { - crate::platform::{ - BcmHost, - device_driver::{common::MMIODerefWrapper, gpio}, - exception::asynchronous::IRQNumber, - }, + crate::{BcmHost, device_driver::gpio, exception::asynchronous::IRQNumber}, core::{ convert::From, fmt::{self, Arguments}, }, + libaddress::{Address, Virtual}, libconsole::{SerialOps, console::interface}, liblocking::{IRQSafeNullLock, interface::Mutex}, - libmemory::{Address, Virtual}, + libmmio::MMIODerefWrapper, tock_registers::{ interfaces::ReadWriteable, register_bitfields, register_structs, @@ -170,7 +167,7 @@ impl From for u32 { } // [temporary] Used in mmu.rs to set up local paging -pub const UART1_BASE: usize = BcmHost::get_peripheral_address() + 0x21_5000; +pub const UART1_BASE: usize = BcmHost::get_peripheral_address() + 0x21_5000; // DTB says 0x7e215040 impl libdriver::drivers::interface::DeviceDriver for MiniUart { type IRQNumberType = IRQNumber; diff --git a/libs/platform/src/platform/raspberrypi/device_driver/bcm/mod.rs b/libs/platform/src/raspberrypi/device_driver/bcm/mod.rs similarity index 100% rename from libs/platform/src/platform/raspberrypi/device_driver/bcm/mod.rs rename to libs/platform/src/raspberrypi/device_driver/bcm/mod.rs diff --git a/libs/platform/src/platform/raspberrypi/device_driver/bcm/pl011_uart.rs b/libs/platform/src/raspberrypi/device_driver/bcm/pl011_uart.rs similarity index 98% rename from libs/platform/src/platform/raspberrypi/device_driver/bcm/pl011_uart.rs rename to libs/platform/src/raspberrypi/device_driver/bcm/pl011_uart.rs index 58d1befc6..79e2d5855 100644 --- a/libs/platform/src/platform/raspberrypi/device_driver/bcm/pl011_uart.rs +++ b/libs/platform/src/raspberrypi/device_driver/bcm/pl011_uart.rs @@ -9,11 +9,12 @@ */ use { - crate::platform::device_driver::{IRQNumber, common::MMIODerefWrapper, gpio}, + crate::device_driver::{IRQNumber, gpio}, core::fmt::{self, Arguments}, + libaddress::{Address, Virtual}, libconsole::{SerialOps, console::interface}, liblocking::{IRQSafeNullLock, interface::Mutex}, - libmemory::{Address, Virtual}, + libmmio::MMIODerefWrapper, libprimitives::cpu::loop_while, // snafu::Snafu, tock_registers::{ @@ -520,8 +521,8 @@ impl libdriver::drivers::interface::DeviceDriver for PL011Uart { irq_number: Self::IRQNumberType, ) -> Result<(), &'static str> { use { - crate::platform::exception::asynchronous::irq_manager, - libexception::exception::asynchronous::IRQHandlerDescriptor, + crate::exception::asynchronous::irq_manager, + libexception::asynchronous::IRQHandlerDescriptor, }; let descriptor = IRQHandlerDescriptor::new(irq_number, Self::COMPATIBLE, self); @@ -567,7 +568,7 @@ impl interface::ConsoleOps for PL011Uart { impl interface::All for PL011Uart {} -impl libexception::exception::asynchronous::interface::IRQHandler for PL011Uart { +impl libexception::asynchronous::interface::IRQHandler for PL011Uart { fn handle(&self) -> Result<(), &'static str> { self.inner.lock(|inner| { let pending = inner.registers.MaskedInterruptStatus.extract(); diff --git a/libs/platform/src/platform/raspberrypi/device_driver/bcm/power.rs b/libs/platform/src/raspberrypi/device_driver/bcm/power.rs similarity index 96% rename from libs/platform/src/platform/raspberrypi/device_driver/bcm/power.rs rename to libs/platform/src/raspberrypi/device_driver/bcm/power.rs index 9d3bb17a2..e70797369 100644 --- a/libs/platform/src/platform/raspberrypi/device_driver/bcm/power.rs +++ b/libs/platform/src/raspberrypi/device_driver/bcm/power.rs @@ -6,12 +6,12 @@ */ use { - crate::platform::device_driver::{ - common::MMIODerefWrapper, + crate::device_driver::{ gpio, mailbox::{Mailbox, MailboxOps, channel}, }, - libmemory::{Address, Virtual}, + libaddress::{Address, Virtual}, + libmmio::MMIODerefWrapper, snafu::Snafu, tock_registers::{ interfaces::{Readable, Writeable}, diff --git a/libs/platform/src/platform/raspberrypi/device_driver/mod.rs b/libs/platform/src/raspberrypi/device_driver/mod.rs similarity index 94% rename from libs/platform/src/platform/raspberrypi/device_driver/mod.rs rename to libs/platform/src/raspberrypi/device_driver/mod.rs index 27fd6432d..a1b76e8d9 100644 --- a/libs/platform/src/platform/raspberrypi/device_driver/mod.rs +++ b/libs/platform/src/raspberrypi/device_driver/mod.rs @@ -9,8 +9,6 @@ pub mod arm; #[cfg(any(board_rpi3, board_rpi4))] pub mod bcm; -pub mod common; - #[cfg(board_rpi4)] pub use arm::*; #[cfg(any(board_rpi3, board_rpi4))] diff --git a/libs/platform/src/platform/raspberrypi/display.rs b/libs/platform/src/raspberrypi/display.rs similarity index 100% rename from libs/platform/src/platform/raspberrypi/display.rs rename to libs/platform/src/raspberrypi/display.rs diff --git a/libs/platform/src/platform/raspberrypi/drivers.rs b/libs/platform/src/raspberrypi/drivers.rs similarity index 92% rename from libs/platform/src/platform/raspberrypi/drivers.rs rename to libs/platform/src/raspberrypi/drivers.rs index 5efb93f25..7e1450d08 100644 --- a/libs/platform/src/platform/raspberrypi/drivers.rs +++ b/libs/platform/src/raspberrypi/drivers.rs @@ -1,12 +1,12 @@ use { super::exception, // @todo - crate::platform::{device_driver, exception::asynchronous::IRQNumber}, + crate::{device_driver, exception::asynchronous::IRQNumber, memory::map::mmio}, core::{ mem::MaybeUninit, sync::atomic::{AtomicBool, Ordering}, }, libdriver::drivers::DriverManager, - libmemory::{mmu::MMIODescriptor, platform::memory::map::mmio}, + libmmio::MMIODescriptor, }; //-------------------------------------------------------------------------------------------------- @@ -94,7 +94,7 @@ unsafe fn instantiate_uart() -> Result<(), &'static str> { let mmio_descriptor = MMIODescriptor::new(mmio::PL011_UART_BASE, mmio::PL011_UART_SIZE); // SAFETY: You may believe praying will save you. let virt_addr = unsafe { - libmemory::mmu::kernel_map_mmio(device_driver::PL011Uart::COMPATIBLE, &mmio_descriptor)? + libmapping::kernel_map_mmio(device_driver::PL011Uart::COMPATIBLE, &mmio_descriptor)? }; #[allow(static_mut_refs)] @@ -122,9 +122,8 @@ unsafe fn post_init_pl011_uart() -> Result<(), &'static str> { unsafe fn instantiate_gpio() -> Result<(), &'static str> { let mmio_descriptor = MMIODescriptor::new(mmio::GPIO_BASE, mmio::GPIO_SIZE); // SAFETY: You may believe praying will save you. - let virt_addr = unsafe { - libmemory::mmu::kernel_map_mmio(device_driver::GPIO::COMPATIBLE, &mmio_descriptor)? - }; + let virt_addr = + unsafe { libmapping::kernel_map_mmio(device_driver::GPIO::COMPATIBLE, &mmio_descriptor)? }; #[allow(static_mut_refs)] // SAFETY: You may believe praying will save you. @@ -153,7 +152,7 @@ unsafe fn instantiate_interrupt_controller() -> Result<(), &'static str> { MMIODescriptor::new(mmio::PERIPHERAL_IC_BASE, mmio::PERIPHERAL_IC_SIZE); // SAFETY: You may believe praying will save you. let periph_virt_addr = unsafe { - libmemory::mmu::kernel_map_mmio( + libmapping::kernel_map_mmio( device_driver::InterruptController::COMPATIBLE, &periph_mmio_descriptor, )? @@ -174,12 +173,12 @@ unsafe fn instantiate_interrupt_controller() -> Result<(), &'static str> { let gic_distr_mmio_descriptor = MMIODescriptor::new(mmio::GICD_BASE, mmio::GICD_SIZE); let gic_distr_virt_addr = // SAFETY: Not safe! - unsafe { libmemory::mmu::kernel_map_mmio("GICv2 GICD", &gic_distr_mmio_descriptor)? }; + unsafe { libmapping::kernel_map_mmio("GICv2 GICD", &gic_distr_mmio_descriptor)? }; let gic_ctrlr_mmio_descriptor = MMIODescriptor::new(mmio::GICC_BASE, mmio::GICC_SIZE); let gic_ctrlr_virt_addr = // SAFETY: Not safe! - unsafe { libmemory::mmu::kernel_map_mmio("GICV2 GICC", &gic_ctrlr_mmio_descriptor)? }; + unsafe { libmapping::kernel_map_mmio("GICV2 GICC", &gic_ctrlr_mmio_descriptor)? }; #[allow(static_mut_refs)] // SAFETY: Not safe! @@ -197,7 +196,7 @@ unsafe fn instantiate_interrupt_controller() -> Result<(), &'static str> { #[allow(clippy::unnecessary_wraps)] unsafe fn post_init_interrupt_controller() -> Result<(), &'static str> { #[allow(static_mut_refs)] - crate::platform::exception::asynchronous::register_irq_manager( + crate::exception::asynchronous::register_irq_manager( // SAFETY: You may believe praying will save you. unsafe { INTERRUPT_CONTROLLER.assume_init_ref() }, ); diff --git a/libs/platform/src/platform/raspberrypi/exception/asynchronous.rs b/libs/platform/src/raspberrypi/exception/asynchronous.rs similarity index 85% rename from libs/platform/src/platform/raspberrypi/exception/asynchronous.rs rename to libs/platform/src/raspberrypi/exception/asynchronous.rs index 73c7201ee..1b269a2d2 100644 --- a/libs/platform/src/platform/raspberrypi/exception/asynchronous.rs +++ b/libs/platform/src/raspberrypi/exception/asynchronous.rs @@ -5,9 +5,7 @@ //! Platform asynchronous exception handling. use { - libexception::exception::asynchronous::{ - IRQContext, IRQHandlerDescriptor, interface::IRQManager, - }, + libexception::asynchronous::{IRQContext, IRQHandlerDescriptor, interface::IRQManager}, liblocking::InitStateLock, }; @@ -16,18 +14,18 @@ use { //-------------------------------------------------------------------------------------------------- /// Export for reuse in generic asynchronous.rs. -pub use crate::platform::device_driver::IRQNumber; // @todo +pub use crate::device_driver::IRQNumber; // @todo #[cfg(board_rpi3)] -pub(in crate::platform) mod irq_map { - use crate::platform::device_driver::{IRQNumber, PeripheralIRQ}; +pub(crate) mod irq_map { + use crate::device_driver::{IRQNumber, PeripheralIRQ}; pub const PL011_UART: IRQNumber = IRQNumber::Peripheral(PeripheralIRQ::new(57)); } #[cfg(board_rpi4)] -pub(in crate::platform) mod irq_map { - use crate::platform::device_driver::IRQNumber; +pub(crate) mod irq_map { + use crate::device_driver::IRQNumber; pub const PL011_UART: IRQNumber = IRQNumber::new(153); } diff --git a/libs/platform/src/platform/raspberrypi/exception/mod.rs b/libs/platform/src/raspberrypi/exception/mod.rs similarity index 100% rename from libs/platform/src/platform/raspberrypi/exception/mod.rs rename to libs/platform/src/raspberrypi/exception/mod.rs diff --git a/libs/platform/src/platform/raspberrypi/fb.rs b/libs/platform/src/raspberrypi/fb.rs similarity index 96% rename from libs/platform/src/platform/raspberrypi/fb.rs rename to libs/platform/src/raspberrypi/fb.rs index 031f1b899..04c6d012a 100644 --- a/libs/platform/src/platform/raspberrypi/fb.rs +++ b/libs/platform/src/raspberrypi/fb.rs @@ -1,8 +1,8 @@ use { - crate::platform::raspberrypi::device_driver::bcm::mailbox::{ + crate::raspberrypi::device_driver::bcm::mailbox::{ self, LocalMailboxStorage, Mailbox, MailboxError, MailboxOps, }, - libmemory::{Address, Virtual}, + libaddress::{Address, Virtual}, }; /// `FrameBuffer` channel supported structure - use with `mailbox::channel::FrameBuffer` diff --git a/libs/platform/src/raspberrypi/linker/kickstart.ld b/libs/platform/src/raspberrypi/linker/kickstart.ld new file mode 100644 index 000000000..ddafec6ff --- /dev/null +++ b/libs/platform/src/raspberrypi/linker/kickstart.ld @@ -0,0 +1,61 @@ +/* + * Identity-mapped init thread. + * SPDX-License-Identifier: BlueOak-1.0.0 + * Copyright (c) Berkus Decker + */ + +ENTRY(_boot_cores) + +/* Physical load address - typical for RPi4 */ +__INIT_BASE = 0x80000; + +SECTIONS +{ + /* Stack for init thread (grows down) */ + . = 4K; + __STACK_BOTTOM = .; + . = __INIT_BASE; + __STACK_TOP = .; + + __INIT_START = .; + + .text : ALIGN(4K) + { + *(.text.main.entry) /* Entry point and early code */ + *(.text .text.*) + } + + .rodata : ALIGN(4K) + { + *(.rodata .rodata.*) + + /* Embedded kernel will be here via include_bytes! */ + } + + .data : ALIGN(4K) + { + *(.data .data.*) + } + + .bss : ALIGN(4K) + { + __BSS_START = .; + *(.bss .bss.*) + *(COMMON) + __BSS_END = .; + } + + . = ALIGN(4K); + __INIT_END = .; + + /* Free memory starts here - used for kernel placement */ + . = ALIGN(2M); /* 2MB aligned for huge pages if desired */ + __FREE_MEMORY_START = .; + + /DISCARD/ : + { + *(.comment) + *(.note*) + *(.eh_frame*) + } +} diff --git a/libs/platform/src/raspberrypi/linker/nucleus.ld b/libs/platform/src/raspberrypi/linker/nucleus.ld new file mode 100644 index 000000000..78fdad87b --- /dev/null +++ b/libs/platform/src/raspberrypi/linker/nucleus.ld @@ -0,0 +1,95 @@ +/* + * SPDX-License-Identifier: BlueOak-1.0.0 + * Copyright (c) Berkus Decker + */ + +__PAGE_SIZE = 64K; +__PAGE_MASK = __PAGE_SIZE - 1; + +__KERNEL_VIRT_ADDR_BASE = 0xffff800000000000; /* Nucleus is higher-mem mapped here */ + +/* Flags: + * 4 == R + * 5 == RX + * 6 == RW + * + * Segments are marked PT_LOAD below so that the ELF file provides virtual and physical addresses. + * It doesn't mean all of them need actually be loaded. + */ +PHDRS +{ + segment_code PT_LOAD FLAGS(5); + segment_data PT_LOAD FLAGS(6); +} + +SECTIONS +{ + . = __KERNEL_VIRT_ADDR_BASE; + + .text : ALIGN(__PAGE_SIZE) + { + /******************************************************************************************* + * Regular Kernel Code + *******************************************************************************************/ + *(.text*) + . = ALIGN(2K); + KEEP(*(.vectors)) + . = ALIGN(2K); + } :segment_code + + /* TODO: rodata is currently executable, but doesn't have to... also each individual section must be aligned to 4K at least for mappings to work */ + .rodata : ALIGN(__PAGE_SIZE) + { + *(.rodata*) + FILL(0x00) + . = ALIGN(__PAGE_SIZE); /* Fill up to page size */ + } :segment_code + + /*********************************************************************************************** + * Data + BSS + ***********************************************************************************************/ + + .data : ALIGN(__PAGE_SIZE) + { + *(.data*) + FILL(0x00) + + . = ALIGN(__PAGE_SIZE); + } :segment_data + + .bss (NOLOAD) : ALIGN(__PAGE_SIZE) + { + *(.bss*) + *(COMMON) + . = ALIGN(__PAGE_SIZE); /* Align up to page size */ + } :segment_data + + . += __PAGE_SIZE; /* add guard page between BSS and the stack! */ + + /* Kernel stack virt addr will be mapped beginning from here */ + __STACK_VIRT_BOTTOM = .; + + /*********************************************************************************************** + * MMIO Remap Reserved (8Mb) - TODO: we do not use, instead map it via bump allocator + ***********************************************************************************************/ + /* __MMIO_REMAP_START = .; FIXME: VMA? Must be LMA perhaps, the point is to remap this + . += 8 * 1024 * 1024; + __MMIO_REMAP_END = .; FIXME: VMA? Must be LMA perhaps, the point is to remap this */ + + /*********************************************************************************************** + * Misc + ***********************************************************************************************/ + + .got : { *(.got*) } + ASSERT(SIZEOF(.got) == 0, "Unexpected relocations in kernel binary.") + + /DISCARD/ : { + *(.comment*) + *(.gnu*) + *(.note*) + *(.eh_frame*) + *(.text.chainboot*) + } +} + +INCLUDE libs/exception/src/arch/aarch64/linker/aarch64-exceptions.ld diff --git a/libs/platform/src/raspberrypi/linker/test.ld b/libs/platform/src/raspberrypi/linker/test.ld new file mode 100644 index 000000000..b6c606ef4 --- /dev/null +++ b/libs/platform/src/raspberrypi/linker/test.ld @@ -0,0 +1,61 @@ +/* + * SPDX-License-Identifier: BlueOak-1.0.0 + * Copyright (c) Berkus Decker + * + * Linker script for device test binaries (RPi3, QEMU). + * + * Test binaries use libboot::entry! which puts the boot stub in .text.main.entry + * and expects __STACK_TOP, __STACK_BOTTOM, __BSS_START, __BSS_END symbols. + * Layout mirrors kickstart.ld: load at 0x80000, identity-mapped. + */ + +ENTRY(_boot_cores) + +__TEST_BASE = 0x80000; + +SECTIONS +{ + /* Stack for test binary (grows down toward 0x80000) */ + . = 4K; + __STACK_BOTTOM = .; + . = __TEST_BASE; + __STACK_TOP = .; + + __TEST_START = .; + + .text : ALIGN(4K) + { + *(.text.main.entry) /* Boot stub - must be first */ + *(.text .text.*) + } + + .rodata : ALIGN(4K) + { + *(.rodata .rodata.*) + } + + .data : ALIGN(4K) + { + *(.data .data.*) + } + + .bss : ALIGN(16) + { + __BSS_START = .; + *(.bss .bss.*) + *(COMMON) + . = ALIGN(16); + __BSS_END = .; + } + + . = ALIGN(4K); + __TEST_END = .; + + /DISCARD/ : + { + *(.comment) + *(.note*) + *(.eh_frame*) + *(.gnu*) + } +} diff --git a/libs/platform/src/raspberrypi/memory.rs b/libs/platform/src/raspberrypi/memory.rs new file mode 100644 index 000000000..ac4f5e399 --- /dev/null +++ b/libs/platform/src/raspberrypi/memory.rs @@ -0,0 +1,145 @@ +//! Platform memory Management. +//! +//! The physical memory layout. +//! +//! The Raspberry's firmware copies the kernel binary to `0x8_0000`. The preceding region will be used +//! as the boot core's stack. +//! + +use { + libaddress::PhysAddr, + // libmapping::{MemoryRegion, PageAddress}, + // libmemory::{AddressSpace, AssociatedTranslationTable, TranslationGranule}, +}; + +//-------------------------------------------------------------------------------------------------- +// Private Definitions +//-------------------------------------------------------------------------------------------------- + +// type KernelTranslationTable = +// ::TableStartFromBottom; + +//-------------------------------------------------------------------------------------------------- +// Public Definitions +//-------------------------------------------------------------------------------------------------- + +// The translation granule chosen by this platform. This will be used everywhere else +// in the kernel to derive respective data structures and their sizes. +// For example, the `crate::memory::mmu::Page`. +// pub type KernelGranule = TranslationGranule<{ 64 * 1024 }>; + +// The kernel's virtual address space defined by this platform. +// pub type KernelVirtAddrSpace = AddressSpace<{ 1024 * 1024 * 1024 }>; + +/// The board's physical memory map. +/// This is a fixed memory map for Raspberry Pi, +/// @todo we need to infer the memory map from the provided DTB instead. +#[rustfmt::skip] +pub mod map { + use super::*; + + /// Beginning of memory. + pub const START: u64 = 0x0000_0000; + /// End of memory - 8Gb `RPi4` + pub const END_INCLUSIVE: u64 = 0x1_FFFF_FFFF; + + /// Physical RAM addresses. + pub mod phys { + /// Base address of video (VC) memory. + pub const VIDEOMEM_BASE: u64 = 0x3e00_0000; + } + + pub const VIDEOCORE_MBOX_OFFSET: u64 = 0x0000_B880; + pub const POWER_OFFSET: u64 = 0x0010_0000; + pub const GPIO_OFFSET: u64 = 0x0020_0000; + pub const UART_OFFSET: u64 = 0x0020_1000; + pub const MINIUART_OFFSET: u64 = 0x0021_5000; + + /// Physical devices. + #[cfg(board_rpi3)] + pub mod mmio { + use super::*; + + /// Base address of MMIO register range. + pub const MMIO_BASE: u64 = 0x3F00_0000; + + /// Interrupt controller + pub const PERIPHERAL_IC_BASE: PhysAddr = PhysAddr::new(MMIO_BASE + 0x0000_B200); + pub const PERIPHERAL_IC_SIZE: usize = 0x24; + + /// Base address of ARM<->VC mailbox area. + pub const VIDEOCORE_MBOX_BASE: PhysAddr = PhysAddr::new(MMIO_BASE + VIDEOCORE_MBOX_OFFSET); + + /// Board power control. + pub const POWER_BASE: PhysAddr = PhysAddr::new(MMIO_BASE + POWER_OFFSET); + + /// Base address of GPIO registers. + pub const GPIO_BASE: PhysAddr = PhysAddr::new(MMIO_BASE + GPIO_OFFSET); + pub const GPIO_SIZE: usize = 0xA0; + + pub const PL011_UART_BASE: PhysAddr = PhysAddr::new(MMIO_BASE + UART_OFFSET); + pub const PL011_UART_SIZE: usize = 0x48; + + /// Base address of `MiniUART`. + pub const MINI_UART_BASE: PhysAddr = PhysAddr::new(MMIO_BASE + MINIUART_OFFSET); + + /// End of MMIO memory region. + pub const END: PhysAddr = PhysAddr::new(0x4001_0000); + } + + /// Physical devices. + #[cfg(board_rpi4)] + pub mod mmio { + use super::*; + + /// Base address of MMIO register range. + pub const MMIO_BASE: u64 = 0xFE00_0000; + + /// Base address of GPIO registers. + pub const GPIO_BASE: PhysAddr = PhysAddr::new(MMIO_BASE + GPIO_OFFSET); + pub const GPIO_SIZE: usize = 0xA0; + + /// Base address of regular UART. + pub const PL011_UART_BASE: PhysAddr = PhysAddr::new(MMIO_BASE + UART_OFFSET); + pub const PL011_UART_SIZE: usize = 0x48; + + /// Base address of `MiniUART`. + pub const MINI_UART_BASE: PhysAddr = PhysAddr::new(MMIO_BASE + MINIUART_OFFSET); + + /// Interrupt controller + pub const GICD_BASE: PhysAddr = PhysAddr::new(0xFF84_1000); + pub const GICD_SIZE: usize = 0x824; + + pub const GICC_BASE: PhysAddr = PhysAddr::new(0xFF84_2000); + pub const GICC_SIZE: usize = 0x14; + + /// Base address of ARM<->VC mailbox area. + pub const VIDEOCORE_MBOX_BASE: u64 = MMIO_BASE + VIDEOCORE_MBOX_OFFSET; + + /// End of MMIO memory region. + pub const END: PhysAddr = PhysAddr::new(0xFF85_0000); + } + + #[cfg(not(any(board_rpi3, board_rpi4)))] + compile_error!("No platform selected - specify TARGET_BOARD in configuration"); + + /// End address of mapped memory. + pub const END: PhysAddr = mmio::END; + + //---- + // Unused? + //---- + + /// Virtual (mapped) addresses. + pub mod virt { + /// Start (top) of kernel stack. + pub const KERN_STACK_START: u64 = super::START; + /// End (bottom) of kernel stack. SP starts at `KERN_STACK_END` + 1. + pub const KERN_STACK_END: u64 = 0x0007_FFFF; + + /// Location of DMA-able memory region (in the second 2 MiB block). + pub const DMA_HEAP_START: u64 = 0x0020_0000; + /// End of DMA-able memory region. + pub const DMA_HEAP_END: u64 = 0x005F_FFFF; + } +} diff --git a/libs/platform/src/platform/raspberrypi/mod.rs b/libs/platform/src/raspberrypi/mod.rs similarity index 99% rename from libs/platform/src/platform/raspberrypi/mod.rs rename to libs/platform/src/raspberrypi/mod.rs index 2f0d59f61..60175723b 100644 --- a/libs/platform/src/platform/raspberrypi/mod.rs +++ b/libs/platform/src/raspberrypi/mod.rs @@ -11,6 +11,7 @@ pub mod display; pub mod drivers; pub mod exception; pub mod fb; +pub mod memory; pub mod vc; /// See BCM2835-ARM-Peripherals.pdf diff --git a/libs/platform/src/platform/raspberrypi/vc.rs b/libs/platform/src/raspberrypi/vc.rs similarity index 84% rename from libs/platform/src/platform/raspberrypi/vc.rs rename to libs/platform/src/raspberrypi/vc.rs index d75442716..1b9ef808d 100644 --- a/libs/platform/src/platform/raspberrypi/vc.rs +++ b/libs/platform/src/raspberrypi/vc.rs @@ -3,7 +3,7 @@ * Copyright (c) Berkus Decker */ use { - crate::platform::raspberrypi::{ + crate::raspberrypi::{ BcmHost, device_driver::bcm::mailbox::{ self, Mailbox, MailboxOps, MailboxStorageRef, channel, response::VAL_LEN_FLAG, @@ -32,17 +32,17 @@ impl VC { // Use framebuffer mailbox interface to initialize // https://www.raspberrypi.org/forums/viewtopic.php?f=72&t=185116 pub fn init_fb(w: u32, h: u32, depth: u32) -> Result { - /* - * * All tags in the request are processed in one operation. - * * It is not valid to mix Test tags with Get/Set tags - * in the same operation and no tags will be returned. - * * Get tags will be processed after all Set tags. - * * If an allocate buffer tag is omitted when setting parameters, - * then no change occurs unless it can be accommodated without changing - * the buffer base or size. - * * When an allocate buffer response is returned, the old buffer area - * (if the base or size has changed) is implicitly freed. - */ + // + // - All tags in the request are processed in one operation. + // - It is not valid to mix Test tags with Get/Set tags + // in the same operation and no tags will be returned. + // - Get tags will be processed after all Set tags. + // - If an allocate buffer tag is omitted when setting parameters, + // then no change occurs unless it can be accommodated without changing + // the buffer base or size. + // - When an allocate buffer response is returned, the old buffer area + // (if the base or size has changed) is implicitly freed. + // // control: MailboxCommand<10, FrameBufferData> let mut mbox = Mailbox::<36>::default(); @@ -72,7 +72,7 @@ impl VC { // Apparently, QEMU doesn't care about intermixing Get/Set and Test tags either. let mut mbox = Mailbox::<36>::default(); let index = mbox.request(); - #[cfg(qemu)] + #[cfg(feature = "qemu")] let index = mbox.test_pixel_order(index, 1); #[cfg(not(qemu))] let index = mbox.set_pixel_order(index, 1); diff --git a/nucleus/tests/platform.rs b/libs/platform/tests/platform.rs similarity index 98% rename from nucleus/tests/platform.rs rename to libs/platform/tests/platform.rs index 3bd745dad..037117b88 100644 --- a/nucleus/tests/platform.rs +++ b/libs/platform/tests/platform.rs @@ -5,14 +5,14 @@ #![reexport_test_harness_main = "test_main"] use { - libmemory::{Address, Virtual}, + libaddress::{Address, Virtual}, libplatform::platform::device_driver::{ Function, GPIO, RateDivisors, mailbox::{self, Mailbox, tag}, }, }; -mod common; +// mod common; #[test_case] fn test_pin_transitions() { diff --git a/libs/primitives/Cargo.toml b/libs/primitives/Cargo.toml index 4314195bb..ef9b63fe2 100644 --- a/libs/primitives/Cargo.toml +++ b/libs/primitives/Cargo.toml @@ -1,7 +1,7 @@ [package] -name = "libprimitives" +name = "vesper-primitives" -description = "Kernel primitives" +description = "Vesper nanokernel primitives." authors = { workspace = true } categories = { workspace = true } @@ -20,5 +20,10 @@ maintenance = { status = "experimental" } [dependencies] +# Tests are offloaded to kernel integration tests binary. +[lib] +test = false +harness = false + [lints] workspace = true diff --git a/libs/print/Cargo.toml b/libs/print/Cargo.toml new file mode 100644 index 000000000..04b46929a --- /dev/null +++ b/libs/print/Cargo.toml @@ -0,0 +1,28 @@ +[package] +name = "vesper-print" + +description = "Vesper nanokernel buffered output functions for no_std." + +authors = { workspace = true } +categories = { workspace = true } +documentation = { workspace = true } +edition = { workspace = true } +homepage = { workspace = true } +license = { workspace = true } +readme = { workspace = true } +repository = { workspace = true } +version = { workspace = true } + +publish = false + +[badges] +maintenance = { status = "experimental" } + +[dependencies] + +[lib] +test = false +harness = false + +[lints] +workspace = true diff --git a/libs/console/src/write_to.rs b/libs/print/src/lib.rs similarity index 70% rename from libs/console/src/write_to.rs rename to libs/print/src/lib.rs index bc14d6165..0050c1b7a 100644 --- a/libs/console/src/write_to.rs +++ b/libs/print/src/lib.rs @@ -3,12 +3,16 @@ * Copyright (c) Berkus Decker */ -/// No-alloc write!() implementation from -/// Requires you to allocate a buffer somewhere manually. +#![no_std] + +// No-alloc write!() implementation from +// Requires you to allocate a buffer somewhere manually (usually, on stack). // @todo Try to use arrayvec::ArrayString here instead? -use core::{cmp::min, fmt}; +// @todo probably use defmt for comms with host? + +use core::{cmp::min, ffi::CStr, fmt}; -pub struct WriteTo<'a> { +struct WriteTo<'a> { buffer: &'a mut [u8], // on write error (i.e. not enough space in buffer) this grows beyond // `buffer.len()`. @@ -29,11 +33,12 @@ impl<'a> WriteTo<'a> { } #[allow(unused)] - pub fn into_cstr(self) -> Option<&'a str> { + pub fn into_cstr(self) -> Option<&'a CStr> { (self.used < self.buffer.len()).then(|| { - self.buffer[self.used] = 0; // Terminate the string + // Terminate the string + self.buffer[self.used] = 0; // SAFETY: only successful concats of str - must be a valid str. - unsafe { core::str::from_utf8_unchecked(&self.buffer[..=self.used]) } + unsafe { CStr::from_bytes_with_nul_unchecked(&self.buffer[..=self.used]) } }) } } @@ -56,16 +61,14 @@ impl fmt::Write for WriteTo<'_> { } } -#[allow(unused)] -pub fn show<'a>(buffer: &'a mut [u8], args: fmt::Arguments) -> Result<&'a str, fmt::Error> { +pub fn format_str<'a>(buffer: &'a mut [u8], args: fmt::Arguments) -> Result<&'a str, fmt::Error> { let mut w = WriteTo::new(buffer); fmt::write(&mut w, args)?; w.into_str().ok_or(fmt::Error) } // Return a zero-terminated str -#[allow(unused)] -pub fn c_show<'a>(buffer: &'a mut [u8], args: fmt::Arguments) -> Result<&'a str, fmt::Error> { +pub fn format_cstr<'a>(buffer: &'a mut [u8], args: fmt::Arguments) -> Result<&'a CStr, fmt::Error> { let mut w = WriteTo::new(buffer); fmt::write(&mut w, args)?; w.into_cstr().ok_or(fmt::Error) diff --git a/nucleus/tests/console.rs b/libs/print/tests/print.rs similarity index 98% rename from nucleus/tests/console.rs rename to libs/print/tests/print.rs index fa1d2440c..fdd2e316f 100644 --- a/nucleus/tests/console.rs +++ b/libs/print/tests/print.rs @@ -4,7 +4,7 @@ #![test_runner(libtest::test_runner)] #![reexport_test_harness_main = "test_main"] -mod common; +// mod common; #[test_case] pub fn write_to_works() { diff --git a/libs/qemu/Cargo.toml b/libs/qemu/Cargo.toml index 508d847a6..a0cc66bd5 100644 --- a/libs/qemu/Cargo.toml +++ b/libs/qemu/Cargo.toml @@ -1,7 +1,7 @@ [package] -name = "libqemu" +name = "vesper-qemu" -description = "QEMU implementations" +description = "Vesper nanokernel QEMU support." authors = { workspace = true } categories = { workspace = true } @@ -19,7 +19,15 @@ publish = false maintenance = { status = "experimental" } [dependencies] -qemu-exit = { workspace = true } +libprint = { workspace = true } + +# [dev-dependencies] +# libtest = { workspace = true } + +# Tests are offloaded to kernel integration tests binary. +[lib] +test = false +harness = false [lints] workspace = true diff --git a/libs/qemu/src/lib.rs b/libs/qemu/src/lib.rs index 138e1c50e..a1f05a950 100644 --- a/libs/qemu/src/lib.rs +++ b/libs/qemu/src/lib.rs @@ -5,38 +5,94 @@ #![no_std] #![no_main] -#![feature(custom_test_frameworks)] -#![test_runner(libtest::test_runner)] -#![reexport_test_harness_main = "test_main"] +#![feature(format_args_nl)] +// #![feature(custom_test_frameworks)] +// #![test_runner(libtest::test_runner)] +// #![reexport_test_harness_main = "test_main"] pub mod semihosting { - pub fn exit_success() -> ! { - use qemu_exit::QEMUExit; + #[repr(C)] + struct ExitParameters { + arg0: u64, + arg1: u64, + } - #[cfg(target_arch = "aarch64")] - let qemu_exit_handle = qemu_exit::AArch64::new(); + #[expect(non_upper_case_globals)] + const ADP_Stopped_ApplicationExit: u64 = 0x20026; - qemu_exit_handle.exit_success() + #[inline] + pub fn exit(code: u32) -> ! { + let params = ExitParameters { + arg0: ADP_Stopped_ApplicationExit, + arg1: u64::from(code), + }; + sys_exit_call(¶ms) } - pub fn exit_failure() -> ! { - use qemu_exit::QEMUExit; + #[inline] + pub fn exit_success() -> ! { + exit(0) + } - #[cfg(target_arch = "aarch64")] - let qemu_exit_handle = qemu_exit::AArch64::new(); + #[inline] + pub fn exit_failure() -> ! { + exit(1) + } - qemu_exit_handle.exit_failure() + fn sys_exit_call(params: &ExitParameters) -> ! { + // SAFETY: safe enough! + unsafe { + core::arch::asm!( + "hlt #0xF000", + in("w0") 0x20, // Sys_Exit_Extended + in("x1") core::ptr::from_ref(params) as u64, + options(nostack) + ); + loop { + core::arch::asm!("wfe", options(nomem, nostack)); + } + } } - pub fn sys_write0_call(text: &str) { - let cmd = 0x04; - // SAFETY: text must be \0-terminated! + #[inline] + pub fn sys_write0_call(text: &core::ffi::CStr) { + // SAFETY: text must be \0-terminated, which CStr above shall ensure. unsafe { core::arch::asm!( - "hlt #0xF000" - , in("w0") cmd - , in("x1") text.as_ptr() as u64 + "hlt #0xF000", + in("w0") 0x04, // Sys_Write0 + in("x1") text.as_ptr() as u64, + options(nostack) ); } } + + #[macro_export] + macro_rules! semi_print { + // semi_print!("a {} event", "log") + ($($arg:tt)+) => { + let mut buf = [0_u8; 4096]; // Increase this buffer size to allow dumping larger panic texts. + libqemu::semihosting::sys_write0_call( + libprint::format_cstr(&mut buf, core::format_args!($($arg)+)).unwrap(), + ); + }; + } + + #[macro_export] + macro_rules! semi_println { + // semi_println!() + () => { + let mut buf = [0_u8; 4096]; // Increase this buffer size to allow dumping larger panic texts. + libqemu::semihosting::sys_write0_call( + libprint::format_cstr(&mut buf, core::format_args_nl!("")).unwrap(), + ); + }; + // semi_println!("a {} event", "log") + ($($arg:tt)+) => { + let mut buf = [0_u8; 4096]; // Increase this buffer size to allow dumping larger panic texts. + libqemu::semihosting::sys_write0_call( + libprint::format_cstr(&mut buf, core::format_args_nl!($($arg)+)).unwrap(), + ); + }; + } } diff --git a/libs/syscall/Cargo.toml b/libs/syscall/Cargo.toml new file mode 100644 index 000000000..d2488a41c --- /dev/null +++ b/libs/syscall/Cargo.toml @@ -0,0 +1,29 @@ +[package] +name = "vesper-syscalls" + +description = "Vesper nanokernel syscall interface." + +authors = { workspace = true } +categories = { workspace = true } +documentation = { workspace = true } +edition = { workspace = true } +homepage = { workspace = true } +license = { workspace = true } +readme = { workspace = true } +repository = { workspace = true } +version = { workspace = true } + +publish = false + +[badges] +maintenance = { status = "experimental" } + +[dependencies] + +# Tests are offloaded to kernel integration tests binary. +[lib] +test = false +harness = false + +[lints] +workspace = true diff --git a/libs/syscall/src/lib.rs b/libs/syscall/src/lib.rs new file mode 100644 index 000000000..f07fc4164 --- /dev/null +++ b/libs/syscall/src/lib.rs @@ -0,0 +1,240 @@ +#![no_std] +#![no_main] + +// Syscall ABI: +// ┌───────────────────────────────────────────────────────────────────────┐ +// │ CAPTBL COPY (cross-domain grant): │ +// │ ┌─────────────────────────────────────────────────────────────────┐ │ +// │ │ x0 = src_captbl x3 = dst_captbl │ │ +// │ │ x1 = COPY op x4 = dst_slot │ │ +// │ │ x2 = src_slot x5 = rights_mask ← derive+copy in one! │ │ +// │ └─────────────────────────────────────────────────────────────────┘ │ +// │ │ +// │ BUFFER MAP (with full control): │ +// │ ┌─────────────────────────────────────────────────────────────────┐ │ +// │ │ x0 = buffer_cap x3 = size │ │ +// │ │ x1 = MAP op x4 = offset ← map partial buffer │ │ +// │ │ x2 = virt_addr x5 = flags ← cache policy, etc. │ │ +// │ └─────────────────────────────────────────────────────────────────┘ │ +// │ │ +// │ ENDPOINT CALL (with inline payload): │ +// │ ┌─────────────────────────────────────────────────────────────────┐ │ +// │ │ x0 = endpoint_cap x3 = msg_word_1 │ │ +// │ │ x1 = CALL op x4 = msg_word_2 │ │ +// │ │ x2 = msg_word_0 x5 = msg_word_3 ← 4 words inline! │ │ +// │ └─────────────────────────────────────────────────────────────────┘ │ +// │ │ +// │ UNTYPED RETYPE (batch creation): │ +// │ ┌─────────────────────────────────────────────────────────────────┐ │ +// │ │ x0 = untyped_cap x3 = dest_captbl │ │ +// │ │ x1 = RETYPE op x4 = dest_slot_start │ │ +// │ │ x2 = obj_type x5 = count ← create N objects at once! │ │ +// │ └─────────────────────────────────────────────────────────────────┘ │ +// └───────────────────────────────────────────────────────────────────────┘ + +/// Single syscall ABI +/// +/// Entry: SVC #0 +/// +/// Arguments: +/// x0 = capability slot +/// x1 = operation code +/// x2-x7 = operation arguments (6 args!) +/// x9-x15 are caller-saved, we don't use them +/// +/// Returns: +/// x0 = error code (0 = success) +/// x1 = return value 0 +/// x2 = return value 1 (if needed) +/// +/// # Safety +/// - Not safe. +#[inline(always)] +#[allow(clippy::too_many_arguments)] +pub unsafe fn protected_call6( + cap: u32, + op: u32, + a0: u64, + a1: u64, + a2: u64, + a3: u64, + a4: u64, + a5: u64, +) -> (u64, u64, u64) { + let r0: u64; + let r1: u64; + let r2: u64; + // # SAFETY: As safe as possible, duh! + unsafe { + core::arch::asm!( + "svc #0", + inlateout("x0") u64::from(cap) => r0, + inlateout("x1") u64::from(op) => r1, + inlateout("x2") a0 => r2, + in("x3") a1, + in("x4") a2, + in("x5") a3, + in("x6") a4, + in("x7") a5, + options(nostack), + ); + } + (r0, r1, r2) +} + +/// 5-arg invoke +/// +/// # Safety +/// - Not safe. +#[inline(always)] +pub unsafe fn protected_call5( + cap: u32, + op: u32, + a0: u64, + a1: u64, + a2: u64, + a3: u64, + a4: u64, +) -> (u64, u64, u64) { + let r0: u64; + let r1: u64; + let r2: u64; + // # SAFETY: As safe as possible, duh! + unsafe { + core::arch::asm!( + "svc #0", + inlateout("x0") u64::from(cap) => r0, + inlateout("x1") u64::from(op) => r1, + inlateout("x2") a0 => r2, + in("x3") a1, + in("x4") a2, + in("x5") a3, + in("x6") a4, + options(nostack), + ); + } + (r0, r1, r2) +} + +/// 4-arg invoke +/// +/// # Safety +/// - Not safe. +#[inline(always)] +pub unsafe fn protected_call4( + cap: u32, + op: u32, + a0: u64, + a1: u64, + a2: u64, + a3: u64, +) -> (u64, u64, u64) { + let r0: u64; + let r1: u64; + let r2: u64; + // # SAFETY: As safe as possible, duh! + unsafe { + core::arch::asm!( + "svc #0", + inlateout("x0") u64::from(cap) => r0, + inlateout("x1") u64::from(op) => r1, + inlateout("x2") a0 => r2, + in("x3") a1, + in("x4") a2, + in("x5") a3, + options(nostack), + ); + } + (r0, r1, r2) +} + +/// 3-arg invoke +/// +/// # Safety +/// - Not safe. +#[inline(always)] +pub unsafe fn protected_call3(cap: u32, op: u32, a0: u64, a1: u64, a2: u64) -> (u64, u64, u64) { + let r0: u64; + let r1: u64; + let r2: u64; + // # SAFETY: As safe as possible, duh! + unsafe { + core::arch::asm!( + "svc #0", + inlateout("x0") u64::from(cap) => r0, + inlateout("x1") u64::from(op) => r1, + inlateout("x2") a0 => r2, + in("x3") a1, + in("x4") a2, + options(nostack), + ); + } + (r0, r1, r2) +} + +/// 2-arg invoke +/// +/// # Safety +/// - Not safe. +#[inline(always)] +pub unsafe fn protected_call2(cap: u32, op: u32, a0: u64, a1: u64) -> (u64, u64, u64) { + let r0: u64; + let r1: u64; + let r2: u64; + // # SAFETY: As safe as possible, duh! + unsafe { + core::arch::asm!( + "svc #0", + inlateout("x0") u64::from(cap) => r0, + inlateout("x1") u64::from(op) => r1, + inlateout("x2") a0 => r2, + in("x3") a1, + options(nostack), + ); + } + (r0, r1, r2) +} + +/// 1-arg invoke +/// +/// # Safety +/// - Not safe. +#[inline(always)] +pub unsafe fn protected_call1(cap: u32, op: u32, a0: u64) -> (u64, u64, u64) { + let r0: u64; + let r1: u64; + let r2: u64; + // # SAFETY: As safe as possible, duh! + unsafe { + core::arch::asm!( + "svc #0", + inlateout("x0") u64::from(cap) => r0, + inlateout("x1") u64::from(op) => r1, + inlateout("x2") a0 => r2, + options(nostack), + ); + } + (r0, r1, r2) +} + +/// 0-arg invoke (cap + op only) +/// +/// # Safety +/// - Not safe. +#[inline(always)] +pub unsafe fn protected_call0(cap: u32, op: u32) -> (u64, u64, u64) { + let r0: u64; + let r1: u64; + let r2: u64; + // # SAFETY: As safe as possible, duh! + unsafe { + core::arch::asm!( + "svc #0", + inlateout("x0") u64::from(cap) => r0, + inlateout("x1") u64::from(op) => r1, + out("x2") r2, + options(nostack), + ); + } + (r0, r1, r2) +} diff --git a/libs/test/Cargo.toml b/libs/test/Cargo.toml index cc9b3795c..c7691bc79 100644 --- a/libs/test/Cargo.toml +++ b/libs/test/Cargo.toml @@ -1,7 +1,7 @@ [package] -name = "libtest" +name = "vesper-tests" -description = "Test runner" +description = "Vesper nanokernel test runner harness." authors = { workspace = true } categories = { workspace = true } @@ -22,5 +22,10 @@ maintenance = { status = "experimental" } liblog = { workspace = true } libqemu = { workspace = true } +# Tests are offloaded to kernel integration tests binary. +[lib] +test = false +harness = false + [lints] workspace = true diff --git a/libs/test/src/lib.rs b/libs/test/src/lib.rs index e474a826a..cba92e4d2 100644 --- a/libs/test/src/lib.rs +++ b/libs/test/src/lib.rs @@ -27,7 +27,7 @@ where } pub fn test_runner(tests: &[&dyn TestFn]) { - liblog::println!("Running {} tests", tests.len()); + liblog::println!("*TESTING* Running {} tests", tests.len()); for test in tests { test.run(); } diff --git a/libs/time/Cargo.toml b/libs/time/Cargo.toml index b8f0b3da8..f81d1bc30 100644 --- a/libs/time/Cargo.toml +++ b/libs/time/Cargo.toml @@ -1,7 +1,7 @@ [package] -name = "libtime" +name = "vesper-time" -description = "Time functions" +description = "Vesper nanokernel time functions." authors = { workspace = true } categories = { workspace = true } @@ -25,5 +25,13 @@ liblog = { workspace = true } once_cell = { workspace = true } tock-registers = { workspace = true } +# [dev-dependencies] +# libtest = { workspace = true } + +# Tests are offloaded to kernel integration tests binary. +[lib] +test = false +harness = false + [lints] workspace = true diff --git a/nucleus/Makefile.toml b/nucleus/Makefile.toml deleted file mode 100644 index ccba960af..000000000 --- a/nucleus/Makefile.toml +++ /dev/null @@ -1,102 +0,0 @@ -# -# SPDX-License-Identifier: BlueOak-1.0.0 -# -# Copyright (c) Berkus Decker -# -# Build nucleus -# -extend = "../Makefile.toml" - -[env] -KERNEL_LINK = "libs/platform/src/platform/raspberrypi/linker/kernel.ld" - -[tasks.build] -clear = true -dependencies = ["build-target-nucleus", "convert-kernel-binary"] -script_runner = "@duckscript" -script = ''' - echo - echo ✅ nucleus built for hw ${TARGET_BOARD} ${BOARD_RUSTFLAGS}. - echo -''' - -[tasks.build-target-nucleus] -env = { TARGET_BOARD = "rpi4", TARGET_LINK_RUSTFLAGS = "-C link-arg=--script=${KERNEL_LINK}", BOARD_RUSTFLAGS = "${BOARD_rpi4_RUSTFLAGS}" } -run_task = "build-target" - -# Generate a bootable binary file from KERNEL_ELF -[tasks.convert-kernel-binary] -workspace = false -env = { "BINARY_FILE" = "${KERNEL_ELF}" } -extend = "convert-custom-binary" - -[tasks.test-nucleus] -env = { TARGET_LINK_RUSTFLAGS = "-C link-arg=--script=${KERNEL_LINK}" } -run_task = "test-on-rpi3-qemu" - -[tasks.qemu] -clear = true -alias = "qemu-kernel" - -[tasks.qemu-kernel] -env = { "QEMU_RUNNER_OPTS" = "${QEMU_SERIAL_OPTS}", TARGET_LINK_RUSTFLAGS = "-C link-arg=--script=${KERNEL_LINK}", "TARGET_DTB" = "${CARGO_MAKE_WORKSPACE_WORKING_DIRECTORY}/targets/bcm2710-rpi-3-b-plus.dtb" } -run_task = "qemu-runner" - -[tasks.qemu-gdb] -env = { "QEMU_RUNNER_OPTS" = "${QEMU_SERIAL_OPTS} ${QEMU_DISASM_OPTS} ${QEMU_GDB_OPTS}", "TARGET_DTB" = "${CARGO_MAKE_WORKSPACE_WORKING_DIRECTORY}/targets/bcm2710-rpi-3-b-plus.dtb" } -extend = "qemu-runner" - -[tasks.zellij-nucleus] -alias = "zellij-nucleus-override" - -[tasks.zellij-nucleus-override] -env = { "KERNEL_BIN" = "${KERNEL_BIN}", "QEMU_RUNNER_OPTS" = "${QEMU_SERIAL_OPTS} ${QEMU_DISASM_OPTS} ${QEMU_GDB_OPTS}", "TARGET_DTB" = "${CARGO_MAKE_WORKSPACE_WORKING_DIRECTORY}/targets/bcm2710-rpi-3-b-plus.dtb" } -extend = "zellij-config" - -[tasks.gdb] -dependencies = ["nucleus", "gdb-config"] -env = { "RUST_GDB" = "${GDB}" } -script = ["exec < /dev/tty && pipx run gdbgui -g \"rust-gdb -x ${GDB_CONNECT_FILE} ${KERNEL_ELF}\""] - -[tasks.install-nm] -install_crate = { crate_name = "cargo-binutils", binary = "rust-nm", test_arg = ["--help"] } - -[tasks.install-rustfilt] -install_crate = { crate_name = "rustfilt", binary = "rustfilt", test_arg = ["--help"] } - -[tasks.xtool-nm] -dependencies = ["build", "install-nm", "install-rustfilt"] -script = ["${NM} ${KERNEL_ELF} | sort -k 1 | rustfilt"] - -[tasks.sdcard] # kernel-sdcard ? -dependencies = ["build"] -script_runner = "@duckscript" -script = [ - ''' - kernelImage = set "kernel8.img" - cp ${KERNEL_BIN} ${VOLUME}/${kernelImage} - echo 🔄 Copied nucleus to ${VOLUME}/${kernelImage} -''', -] - -[tasks.hopper] -clear = true -alias = "kernel-hopper" - -[tasks.kernel-hopper] -dependencies = ["build"] -# The cmd line below causes a bug in hopper, see https://www.dropbox.com/s/zyw5mfx0bepcjb1/hopperv4-RAW-bug.mov?dl=0 -# "hopper --loader RAW --base-address 0x80000 --entrypoint 0x80000 --file-offset 0 --plugin arm --cpu aarch64 --variant generic --contains-code true --executable ${KERNEL_BIN}" -script = ["hopper --loader ELF --executable ${KERNEL_ELF}"] - -[tasks.clippy] -clear = true -dependencies = ["clippy-rpi3", "clippy-rpi4"] - -[tasks.clippy-rpi3] -env = { TARGET_BOARD = "rpi3", TARGET_LINK_RUSTFLAGS = "-C link-arg=--script=${KERNEL_LINK}", BOARD_RUSTFLAGS = "${BOARD_rpi3_RUSTFLAGS}" } -run_task = "xtool-clippy" - -[tasks.clippy-rpi4] -env = { TARGET_BOARD = "rpi4", TARGET_LINK_RUSTFLAGS = "-C link-arg=--script=${KERNEL_LINK}", BOARD_RUSTFLAGS = "${BOARD_rpi4_RUSTFLAGS}" } -run_task = "xtool-clippy" diff --git a/nucleus/src/main.rs b/nucleus/src/main.rs deleted file mode 100644 index 2b78dc35f..000000000 --- a/nucleus/src/main.rs +++ /dev/null @@ -1,247 +0,0 @@ -/* - * SPDX-License-Identifier: BlueOak-1.0.0 - * Copyright (c) Berkus Decker - */ - -//! Vesper single-address-space nanokernel. -//! -//! This crate implements the kernel binary proper. - -#![no_std] -#![no_main] -#![feature(decl_macro)] -#![feature(allocator_api)] -#![feature(format_args_nl)] -#![feature(stmt_expr_attributes)] -#![feature(slice_ptr_get)] -#![deny(missing_docs)] -#![deny(warnings)] -#![allow(unused)] -#![allow(internal_features)] -#![allow(linker_messages)] -#![feature(ptr_internals)] -#![feature(core_intrinsics)] - -use core::panic::PanicInfo; -#[allow(unused_imports)] -use libconsole::{SerialOps, console::console}; -use { - cfg_if::cfg_if, - core::{cell::UnsafeCell, time::Duration}, - liblog::{info, println, warn}, - // machine::{arch, entry, memory}, - // exception, - // , time -}; - -libboot::entry!(kernel_init); - -/// Kernel early init code. -/// `arch` crate is responsible for calling it. -/// -/// # Safety -/// -/// - Only a single core must be active and running this function. -/// - The init calls in this function must appear in the correct order: -/// - MMU + Data caching must be activated at the earliest. Without it, any atomic operations, -/// e.g. the yet-to-be-introduced spinlocks in the device drivers (which currently employ -/// `IRQSafeNullLocks` instead of spinlocks), will fail to work (properly) on the `RPi` `SoCs`. -pub unsafe fn kernel_init() -> ! { - #[cfg(feature = "jtag")] - libmachine::debug::jtag::wait_debugger(); - - libexception::exception::handling_init(); - - // SAFETY: Not safe! - let phys_kernel_tables_base_addr = match unsafe { libmemory::mmu::kernel_map_binary() } { - Err(string) => panic!("Error mapping kernel binary: {}", string), - Ok(addr) => addr, - }; - - // SAFETY: Not safe! - if let Err(e) = unsafe { libmemory::mmu::enable_mmu_and_caching(phys_kernel_tables_base_addr) } - { - panic!("Enabling MMU failed: {}", e); - } - - libmemory::mmu::post_enable_init(); - - // SAFETY: Not safe! - if let Err(x) = unsafe { libplatform::platform::drivers::init() } { - panic!("Error initializing platform drivers: {}", x); - } - - // Initialize all device drivers. - // SAFETY: Not safe! - unsafe { - libplatform::platform::drivers::driver_manager().init_drivers_and_irqs(); - } - - // Unmask interrupts on the boot CPU core. - libexception::exception::asynchronous::local_irq_unmask(); - - // Announce conclusion of the kernel_init() phase. - libkernel_state::state_manager().transition_to_single_core_main(); - - libconsole::init_logger(); - - // Transition from unsafe to safe. - kernel_main() -} - -/// Safe kernel code. -// #[inline] -pub fn kernel_main() -> ! { - // info!("{}", libkernel::version()); - // info!("Booting on: {}", bsp::board_name()); - - info!( - "{} version {}", - env!("CARGO_PKG_NAME"), - env!("CARGO_PKG_VERSION") - ); - info!( - "Booting on: {}", - libplatform::platform::BcmHost::board_name() - ); - - // info!("MMU online. Special regions:"); - // machine::platform::memory::mmu::virt_mem_layout().print_layout(); - - let (_, privilege_level) = libexception::exception::current_privilege_level(); - info!("Current privilege level: {privilege_level}"); - - info!("Exception handling state:"); - libexception::exception::asynchronous::print_state(); - - info!( - "Architectural timer resolution: {} ns", - libtime::time::time_manager().resolution().as_nanos() - ); - - info!("Drivers loaded:"); - libplatform::platform::drivers::driver_manager().enumerate(); - - info!("Registered IRQ handlers:"); - libplatform::platform::exception::asynchronous::irq_manager().print_handler(); - - // Test a failing timer case. - libtime::time::time_manager().spin_for(Duration::from_nanos(1)); - - for _ in 0..3 { - info!("Spinning for 1 second"); - libtime::time::time_manager().spin_for(Duration::from_secs(1)); - } - - command_prompt(); - - reboot() -} - -#[panic_handler] -fn panicked(info: &PanicInfo) -> ! { - libmachine::panic::handler(info) -} - -fn print_mmu_state_and_features() { - // use machine::memory::mmu::interface::MMU; - // memory::mmu::mmu().print_features(); -} - -//------------------------------------------------------------ -// Start a command prompt -//------------------------------------------------------------ -fn command_prompt() { - 'cmd_loop: loop { - let mut buf = [0_u8; 64]; - - match libconsole::console::command_prompt(&mut buf) { - // b"mmu" => init_mmu(), - b"feats" => print_mmu_state_and_features(), - // b"disp" => check_display_init(), - // b"trap" => check_data_abort_trap(), - // b"map" => machine::platform::memory::mmu::virt_mem_layout().print_layout(), - // b"led on" => set_led(true), - // b"led off" => set_led(false), - b"help" => print_help(), - b"end" => break 'cmd_loop, - x => warn!("[!] Unknown command {x:?}, try 'help'"), - } - } -} - -fn print_help() { - println!("Supported console commands:"); - println!(" mmu - initialize MMU"); - println!(" feats - print MMU state and supported features"); - #[cfg(not(feature = "noserial"))] - println!(" uart - try to reinitialize UART serial"); - // println!(" disp - try to init VC framebuffer and draw some text"); - println!(" trap - trigger and recover from a data abort exception"); - println!(" map - show kernel memory layout"); - // println!(" led [on|off] - change RPi LED status"); - println!(" end - leave console and reset board"); -} - -// fn set_led(enable: bool) { -// let mut mbox = Mailbox::<8>::default(); -// let index = mbox.request(); -// let index = mbox.set_led_on(index, enable); -// let mbox = mbox.end(index); -// -// mbox.call(channel::PropertyTagsArmToVc) -// .map_err(|e| { -// warn!("Mailbox call returned error {}", e); -// warn!("Mailbox contents: {:?}", mbox); -// }) -// .ok(); -// } - -fn reboot() -> ! { - cfg_if! { - if #[cfg(feature = "qemu")] { - info!("Bye, shutting down QEMU"); - libqemu::semihosting::exit_success() - } else { - // use machine::platform::raspberrypi::power::Power; - - info!("Bye, going to reset now"); - // Power::default().reset() - libcpu::endless_sleep() - } - } -} - -// fn check_display_init() { -// display_graphics() -// .map_err(|e| { -// warn!("Error in display: {}", e); -// }) -// .ok(); -// } -// -// fn display_graphics() -> Result<(), DrawError> { -// if let Ok(mut display) = VC::init_fb(800, 600, 32) { -// info!("Display created"); -// -// display.clear(Color::black()); -// info!("Display cleared"); -// -// display.rect(10, 10, 250, 250, Color::rgb(32, 96, 64)); -// display.draw_text(50, 50, "Hello there!", Color::rgb(128, 192, 255))?; -// -// let mut buf = [0u8; 64]; -// let s = machine::write_to::show(&mut buf, format_args!("Display width {}", display.width)); -// -// if s.is_err() { -// display.draw_text(50, 150, "Error displaying", Color::red())? -// } else { -// display.draw_text(50, 150, s.unwrap(), Color::white())? -// } -// -// display.draw_text(150, 50, "RED", Color::red())?; -// display.draw_text(160, 60, "GREEN", Color::green())?; -// display.draw_text(170, 70, "BLUE", Color::blue())?; -// } -// Ok(()) -// } diff --git a/nucleus/tests/memory.rs b/nucleus/tests/memory.rs deleted file mode 100644 index 1dbf8dfd1..000000000 --- a/nucleus/tests/memory.rs +++ /dev/null @@ -1,303 +0,0 @@ -#![no_std] -#![no_main] -#![feature(allocator_api)] -#![feature(step_trait)] -#![feature(format_args_nl)] -#![feature(custom_test_frameworks)] -#![test_runner(libtest::test_runner)] -#![reexport_test_harness_main = "test_main"] -#![allow(unused_imports)] // commented-out tests - -use { - core::{ - alloc::{Allocator, Layout}, - cell::UnsafeCell, - iter::Step, - num::NonZeroUsize, - ops::Range, - }, - liblocking::interface::Mutex, - liblog::println, - libmemory::{ - Address, - Physical, - Virtual, - arch::mmu::translation_table::{PageDescriptor, TableDescriptor}, - mm::{BumpAllocator, align_up}, - mmu::{ - AccessPermissions, AttributeFields, MemAttributes, MemoryRegion, PageAddress, - kernel_map_at, page_alloc, - translation_table::{FixedSizeTranslationTable, interface::TranslationTable}, - }, - phys_addr::{PhysAddr, PhysAddrNotValid}, - platform::KernelGranule, //memory::mmu::KernelGranule}, - platform::memory::mmu::{ - KERNEL_TABLES, virt_boot_core_stack_region, virt_code_region, virt_data_region, - }, - }, -}; - -mod common; - -pub type MinSizeTranslationTable = FixedSizeTranslationTable<1>; - -/// Check if the size of `struct TableDescriptor` is as expected. -#[test_case] -fn size_of_tabledescriptor_equals_64_bit() { - assert_eq!( - core::mem::size_of::(), - core::mem::size_of::() - ); -} - -/// Check if the size of `struct PageDescriptor` is as expected. -#[test_case] -fn size_of_pagedescriptor_equals_64_bit() { - assert_eq!( - core::mem::size_of::(), - core::mem::size_of::() - ); -} - -/// Sanity of [Address] methods. -#[test_case] -fn address_type_method_sanity() { - let addr = Address::::new(KernelGranule::SIZE + 100); - - assert_eq!(addr.align_down_page().as_usize(), KernelGranule::SIZE); - - assert_eq!(addr.align_up_page().as_usize(), KernelGranule::SIZE * 2); - - assert!(!addr.is_page_aligned()); - - assert_eq!(addr.offset_into_page(), 100); -} - -// Validate allocator allocates from the provided address range -// Validate allocation fails when range is exhausted -#[test_case] -fn test_allocates_within_init_range() { - let allocator = BumpAllocator::new(256, 512, "Test allocator 1"); - let result1 = allocator.allocate(unsafe { Layout::from_size_align_unchecked(128, 1) }); - assert!(result1.is_ok()); - let result2 = allocator.allocate(unsafe { Layout::from_size_align_unchecked(128, 32) }); - println!("{:?}", result2); - assert!(result2.is_ok()); - let result3 = allocator.allocate(unsafe { Layout::from_size_align_unchecked(1, 1) }); - assert!(result3.is_err()); -} - -// Creating with end <= start sshould fail -// @todo return Result<> from new? -#[test_case] -fn test_bad_allocator() { - let bad_allocator = BumpAllocator::new(512, 256, "Test allocator 2"); - let result1 = bad_allocator.allocate(unsafe { Layout::from_size_align_unchecked(1, 1) }); - assert!(result1.is_err()); -} - -#[test_case] -pub fn test_align_up() { - // align 1 - assert_eq!(align_up(0, 1), 0); - assert_eq!(align_up(1234, 1), 1234); - assert_eq!(align_up(0xffff_ffff_ffff_ffff, 1), 0xffff_ffff_ffff_ffff); - // align 2 - assert_eq!(align_up(0, 2), 0); - assert_eq!(align_up(1233, 2), 1234); - assert_eq!(align_up(0xffff_ffff_ffff_fffe, 2), 0xffff_ffff_ffff_fffe); - // address 0 - assert_eq!(align_up(0, 128), 0); - assert_eq!(align_up(0, 1), 0); - assert_eq!(align_up(0, 2), 0); - assert_eq!(align_up(0, 0x8000_0000_0000_0000), 0); -} - -//============================================================================== -//============================================================================== -//============================================================================== -/// Check that you cannot map into the MMIO VA range from kernel_map_at(). -/*#[test_case] -fn no_manual_mmio_map() { - let allocator_region = - MemoryRegion::new(PageAddress::from(0xab0000), PageAddress::from(0xab00000)); - page_alloc::kernel_mmio_va_allocator().lock(|allocator| allocator.init(allocator_region)); - - let phys_start_page_addr: PageAddress = PageAddress::from(0); - let phys_end_exclusive_page_addr: PageAddress = - phys_start_page_addr.checked_offset(5).unwrap(); - let phys_region = MemoryRegion::new(phys_start_page_addr, phys_end_exclusive_page_addr); - - let num_pages = NonZeroUsize::new(phys_region.num_pages()).unwrap(); - let virt_region = page_alloc::kernel_mmio_va_allocator() - .lock(|allocator| allocator.alloc(num_pages)) - .unwrap(); - - let attr = AttributeFields { - mem_attributes: MemAttributes::CacheableDRAM, - acc_perms: AccessPermissions::ReadWrite, - execute_never: true, - }; - - unsafe { - assert_eq!( - kernel_map_at("test", &virt_region, &phys_region, &attr), - Err("Attempt to manually map into MMIO region") - ) - }; -}*/ -//============================================================================== -//============================================================================== -//============================================================================== - -/// Sanity checks for the TranslationTable implementation. -#[test_case] -fn translation_table_implementation_sanity() { - // This will occupy a lot of space on the stack. - let mut tables = MinSizeTranslationTable::new(); - - tables.init().unwrap(); - - let virt_start_page_addr: PageAddress = PageAddress::from(0); - let virt_end_exclusive_page_addr: PageAddress = - virt_start_page_addr.checked_offset(5).unwrap(); - - let phys_start_page_addr: PageAddress = PageAddress::from(0); - let phys_end_exclusive_page_addr: PageAddress = - phys_start_page_addr.checked_offset(5).unwrap(); - - let virt_region = MemoryRegion::new(virt_start_page_addr, virt_end_exclusive_page_addr); - let phys_region = MemoryRegion::new(phys_start_page_addr, phys_end_exclusive_page_addr); - - let attr = AttributeFields { - mem_attributes: MemAttributes::CacheableDRAM, - acc_perms: AccessPermissions::ReadWrite, - execute_never: true, - }; - - unsafe { assert_eq!(tables.map_at(&virt_region, &phys_region, attr), Ok(())) }; -} - -/// Sanity of [PageAddress] methods. -#[test_case] -fn pageaddress_type_method_sanity() { - let page_addr: PageAddress = PageAddress::from(KernelGranule::SIZE * 2); - - assert_eq!( - page_addr.checked_offset(-2), - Some(PageAddress::::from(0)) - ); - - assert_eq!( - page_addr.checked_offset(2), - Some(PageAddress::::from(KernelGranule::SIZE * 4)) - ); - - assert_eq!( - PageAddress::::from(0).checked_offset(0), - Some(PageAddress::::from(0)) - ); - assert_eq!(PageAddress::::from(0).checked_offset(-1), None); - - let max_page_addr = Address::::new(usize::MAX).align_down_page(); - assert_eq!( - PageAddress::::from(max_page_addr).checked_offset(1), - None - ); - - let zero = PageAddress::::from(0); - let three = PageAddress::::from(KernelGranule::SIZE * 3); - assert_eq!(PageAddress::steps_between(&zero, &three), (3, Some(3))); -} - -/// Sanity of [MemoryRegion] methods. -#[test_case] -fn memoryregion_type_method_sanity() { - let zero = PageAddress::::from(0); - let zero_region = MemoryRegion::new(zero, zero); - assert_eq!(zero_region.num_pages(), 0); - assert_eq!(zero_region.size(), 0); - - let one = PageAddress::::from(KernelGranule::SIZE); - let one_region = MemoryRegion::new(zero, one); - assert_eq!(one_region.num_pages(), 1); - assert_eq!(one_region.size(), KernelGranule::SIZE); - - let three = PageAddress::::from(KernelGranule::SIZE * 3); - let mut three_region = MemoryRegion::new(zero, three); - assert!(three_region.contains(zero.into_inner())); - assert!(!three_region.contains(three.into_inner())); - assert!(three_region.overlaps(&one_region)); - - let allocation = three_region - .take_first_n_pages(NonZeroUsize::new(2).unwrap()) - .unwrap(); - assert_eq!(allocation.num_pages(), 2); - assert_eq!(three_region.num_pages(), 1); - - for (i, alloc) in allocation.into_iter().enumerate() { - assert_eq!(alloc.into_inner().as_usize(), i * KernelGranule::SIZE); - } -} - -#[test_case] -pub fn test_invalid_phys_addr() { - let result = PhysAddr::try_new(0xfafa_0123_3210_3210); - if let Err(e) = result { - assert_eq!(e, PhysAddrNotValid(0xfafa_0123_3210_3210)); - } else { - assert!(false) - } -} - -/// Check alignment of the kernel's virtual memory layout sections. -#[test_case] -fn virt_mem_layout_sections_are_64kib_aligned() { - for i in [ - virt_boot_core_stack_region, - virt_code_region, - virt_data_region, - ] - .iter() - { - let start = i().start_page_addr().into_inner(); - let end_exclusive = i().end_exclusive_page_addr().into_inner(); - - assert!(start.is_page_aligned()); - assert!(end_exclusive.is_page_aligned()); - assert!(end_exclusive >= start); - } -} - -/// Ensure the kernel's virtual memory layout is free of overlaps. -#[test_case] -fn virt_mem_layout_has_no_overlaps() { - let layout = [ - virt_boot_core_stack_region(), - virt_code_region(), - virt_data_region(), - ]; - - for (i, first_range) in layout.iter().enumerate() { - for second_range in layout.iter().skip(i + 1) { - assert!(!first_range.overlaps(second_range)) - } - } -} - -/// Check if KERNEL_TABLES is in .bss. -#[test_case] -fn kernel_tables_in_bss() { - unsafe extern "Rust" { - static __BSS_START: UnsafeCell; - static __BSS_END: UnsafeCell; - } - - let bss_range = Range { - start: unsafe { __BSS_START.get() }, - end: unsafe { __BSS_END.get() }, - }; - let kernel_tables_addr = &KERNEL_TABLES as *const _ as usize as *mut u64; - - assert!(bss_range.contains(&kernel_tables_addr)); -} diff --git a/nucleus/tests/nucleus.rs b/nucleus/tests/nucleus.rs deleted file mode 100644 index 95b7b7053..000000000 --- a/nucleus/tests/nucleus.rs +++ /dev/null @@ -1,40 +0,0 @@ -#![no_std] -#![no_main] -#![feature(custom_test_frameworks)] -#![test_runner(libtest::test_runner)] -#![reexport_test_harness_main = "test_main"] - -use liblog::info; - -mod common; - -fn check_data_abort_trap() { - // Cause an exception by accessing a virtual address for which no - // address translations have been set up. - // - // This line of code accesses the address 2 GiB, but page tables are - // only set up for the range [0..1) GiB. - let big_addr: u64 = 2 * 1024 * 1024 * 1024; - unsafe { core::ptr::read_volatile(big_addr as *mut u64) }; - - info!("[i] Whoa! We recovered from an exception."); -} - -#[test_case] -fn test_data_abort_trap() { - libexception::exception::handling_init(); - - let phys_kernel_tables_base_addr = match unsafe { libmemory::mmu::kernel_map_binary() } { - Err(string) => panic!("Error mapping kernel binary: {}", string), - Ok(addr) => addr, - }; - - if let Err(e) = unsafe { libmemory::mmu::enable_mmu_and_caching(phys_kernel_tables_base_addr) } - { - panic!("Enabling MMU failed: {}", e); - } - - libmemory::mmu::post_enable_init(); - - check_data_abort_trap() //-- this needs setup and properly configured mmu to recover ("test mode" in libmemory) -} diff --git a/targets/aarch64-metta-none-eabi.json b/targets/aarch64-metta-none-eabi.json index a7393a7d9..da8592636 100644 --- a/targets/aarch64-metta-none-eabi.json +++ b/targets/aarch64-metta-none-eabi.json @@ -17,9 +17,6 @@ }, "os": "vesper", "panic-strategy": "abort", - "pre-link-args": { - "ld.lld": ["--print-gc-sections"] - }, "relocation-model": "static", "stack-probes": { "kind": "inline" diff --git a/targets/bcm2708-rpi-zero-w.dtb b/targets/bcm2708-rpi-zero-w.dtb new file mode 100644 index 000000000..0b63b9e9c Binary files /dev/null and b/targets/bcm2708-rpi-zero-w.dtb differ diff --git a/targets/bcm2710-rpi-3-b-plus.dtb b/targets/bcm2710-rpi-3-b-plus.dtb index bdc7e43f0..c842951c7 100644 Binary files a/targets/bcm2710-rpi-3-b-plus.dtb and b/targets/bcm2710-rpi-3-b-plus.dtb differ diff --git a/tests/allocator.rs b/tests/allocator.rs new file mode 100644 index 000000000..9bf9c27f9 --- /dev/null +++ b/tests/allocator.rs @@ -0,0 +1,22 @@ +// Validate allocator allocates from the provided address range +// Validate allocation fails when range is exhausted +#[test_case] +fn test_allocates_within_init_range() { + let allocator = BumpAllocator::new(256, 512, "Test allocator 1"); + let result1 = allocator.allocate(unsafe { Layout::from_size_align_unchecked(128, 1) }); + assert!(result1.is_ok()); + let result2 = allocator.allocate(unsafe { Layout::from_size_align_unchecked(128, 32) }); + println!("{:?}", result2); + assert!(result2.is_ok()); + let result3 = allocator.allocate(unsafe { Layout::from_size_align_unchecked(1, 1) }); + assert!(result3.is_err()); +} + +// Creating with end <= start sshould fail +// @todo return Result<> from new? +#[test_case] +fn test_bad_allocator() { + let bad_allocator = BumpAllocator::new(512, 256, "Test allocator 2"); + let result1 = bad_allocator.allocate(unsafe { Layout::from_size_align_unchecked(1, 1) }); + assert!(result1.is_err()); +} diff --git a/nucleus/tests/common/mod.rs b/tests/common/mod.rs similarity index 90% rename from nucleus/tests/common/mod.rs rename to tests/common/mod.rs index 351ba8da1..2fe74bc69 100644 --- a/nucleus/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -8,7 +8,7 @@ fn panicked(info: &PanicInfo) -> ! { libboot::entry!(test_run_helper); -pub fn test_run_helper() -> ! { +pub fn test_run_helper(_dtb: u32) -> ! { libconsole::init_logger().unwrap(); liblog::set_max_level(liblog::Level::Trace); // Allow everything in tests crate::test_main(); diff --git a/tests/mapping.rs b/tests/mapping.rs new file mode 100644 index 000000000..187facff2 --- /dev/null +++ b/tests/mapping.rs @@ -0,0 +1,64 @@ +/// Sanity of [PageAddress] methods. +#[test_case] +fn pageaddress_type_method_sanity() { + let page_addr: PageAddress = PageAddress::from(KernelGranule::SIZE * 2); + + assert_eq!( + page_addr.checked_page_offset(-2), + Some(PageAddress::::from(0)) + ); + + assert_eq!( + page_addr.checked_page_offset(2), + Some(PageAddress::::from(KernelGranule::SIZE * 4)) + ); + + assert_eq!( + PageAddress::::from(0).checked_page_offset(0), + Some(PageAddress::::from(0)) + ); + assert_eq!( + PageAddress::::from(0).checked_page_offset(-1), + None + ); + + let max_page_addr = Address::::new(usize::MAX).align_down_page(); + assert_eq!( + PageAddress::::from(max_page_addr).checked_page_offset(1), + None + ); + + let zero = PageAddress::::from(0); + let three = PageAddress::::from(KernelGranule::SIZE * 3); + assert_eq!(PageAddress::steps_between(&zero, &three), (3, Some(3))); +} + +/// Sanity of [MemoryRegion] methods. +#[test_case] +fn memoryregion_type_method_sanity() { + let zero = PageAddress::::from(0); + let zero_region = MemoryRegion::new(zero, zero); + assert_eq!(zero_region.num_pages(), 0); + assert_eq!(zero_region.size(), 0); + + let one = PageAddress::::from(KernelGranule::SIZE); + let one_region = MemoryRegion::new(zero, one); + assert_eq!(one_region.num_pages(), 1); + assert_eq!(one_region.size(), KernelGranule::SIZE); + + let three = PageAddress::::from(KernelGranule::SIZE * 3); + let mut three_region = MemoryRegion::new(zero, three); + assert!(three_region.contains(zero.into_inner())); + assert!(!three_region.contains(three.into_inner())); + assert!(three_region.overlaps(&one_region)); + + let allocation = three_region + .take_first_n_pages(NonZeroUsize::new(2).unwrap()) + .unwrap(); + assert_eq!(allocation.num_pages(), 2); + assert_eq!(three_region.num_pages(), 1); + + for (i, alloc) in allocation.into_iter().enumerate() { + assert_eq!(alloc.into_inner().as_usize(), i * KernelGranule::SIZE); + } +} diff --git a/tests/memory.rs b/tests/memory.rs new file mode 100644 index 000000000..5f04588e4 --- /dev/null +++ b/tests/memory.rs @@ -0,0 +1,150 @@ +#![no_std] +#![no_main] +#![feature(allocator_api)] +#![feature(step_trait)] +#![feature(format_args_nl)] +#![feature(custom_test_frameworks)] +#![test_runner(libtest::test_runner)] +#![reexport_test_harness_main = "test_main"] +#![allow(unused_imports)] // commented-out tests + +use { + core::{ + alloc::{Allocator, Layout}, + cell::UnsafeCell, + iter::Step, + num::NonZeroUsize, + ops::Range, + }, + libaddress::{align_up, Address, PhysAddr, PhysAddrNotValid, Physical, Virtual}, + liballoc::BumpAllocator, + liblocking::interface::Mutex, + liblog::println, + libmemory::{ + arch::mmu::translation_table::{PageDescriptor, TableDescriptor}, + mmu::{ + kernel_map_at, page_alloc, + translation_table::{interface::TranslationTable, FixedSizeTranslationTable}, + AccessPermissions, AttributeFields, MemAttributes, MemoryRegion, PageAddress, + }, + platform::memory::mmu::{ + virt_boot_core_stack_region, virt_code_region, virt_data_region, KERNEL_TABLES, + }, + }, + libplatform::KernelGranule, +}; + +// mod common; + +// pub type MinSizeTranslationTable = FixedSizeTranslationTable<1>; + +// /// Check if the size of `struct TableDescriptor` is as expected. +// #[test_case] +// fn size_of_tabledescriptor_equals_64_bit() { +// assert_eq!( +// core::mem::size_of::(), +// core::mem::size_of::() +// ); +// } + +// /// Check if the size of `struct PageDescriptor` is as expected. +// #[test_case] +// fn size_of_pagedescriptor_equals_64_bit() { +// assert_eq!( +// core::mem::size_of::(), +// core::mem::size_of::() +// ); +// } + +// /// Check that you cannot map into the MMIO VA range from kernel_map_at(). +// /*#[test_case] +// fn no_manual_mmio_map() { +// let allocator_region = +// MemoryRegion::new(PageAddress::from(0xab0000), PageAddress::from(0xab00000)); +// page_alloc::kernel_mmio_va_allocator().lock(|allocator| allocator.init(allocator_region)); + +// let phys_start_page_addr: PageAddress = PageAddress::from(0); +// let phys_end_exclusive_page_addr: PageAddress = +// phys_start_page_addr.checked_page_offset(5).unwrap(); +// let phys_region = MemoryRegion::new(phys_start_page_addr, phys_end_exclusive_page_addr); + +// let num_pages = NonZeroUsize::new(phys_region.num_pages()).unwrap(); +// let virt_region = page_alloc::kernel_mmio_va_allocator() +// .lock(|allocator| allocator.alloc(num_pages)) +// .unwrap(); + +// let attr = AttributeFields { +// mem_attributes: MemAttributes::CacheableDRAM, +// acc_perms: AccessPermissions::ReadWrite, +// execute_never: true, +// }; + +// unsafe { +// assert_eq!( +// kernel_map_at("test", &virt_region, &phys_region, &attr), +// Err("Attempt to manually map into MMIO region") +// ) +// }; +// }*/ +// /// Sanity checks for the TranslationTable implementation. +// #[test_case] +// fn translation_table_implementation_sanity() { +// // This will occupy a lot of space on the stack. +// let mut tables = MinSizeTranslationTable::new(); + +// tables.init().unwrap(); + +// let virt_start_page_addr: PageAddress = PageAddress::from(0); +// let virt_end_exclusive_page_addr: PageAddress = +// virt_start_page_addr.checked_page_offset(5).unwrap(); + +// let phys_start_page_addr: PageAddress = PageAddress::from(0); +// let phys_end_exclusive_page_addr: PageAddress = +// phys_start_page_addr.checked_page_offset(5).unwrap(); + +// let virt_region = MemoryRegion::new(virt_start_page_addr, virt_end_exclusive_page_addr); +// let phys_region = MemoryRegion::new(phys_start_page_addr, phys_end_exclusive_page_addr); + +// let attr = AttributeFields { +// mem_attributes: MemAttributes::CacheableDRAM, +// acc_perms: AccessPermissions::ReadWrite, +// execute_never: true, +// }; + +// unsafe { assert_eq!(tables.map_at(&virt_region, &phys_region, attr), Ok(())) }; +// } + +// /// Check alignment of the kernel's virtual memory layout sections. +// #[test_case] +// fn virt_mem_layout_sections_are_64kib_aligned() { +// for i in [ +// virt_boot_core_stack_region, +// virt_code_region, +// virt_data_region, +// ] +// .iter() +// { +// let start = i().start_page_addr().into_inner(); +// let end_exclusive = i().end_exclusive_page_addr().into_inner(); + +// assert!(start.is_page_aligned()); +// assert!(end_exclusive.is_page_aligned()); +// assert!(end_exclusive >= start); +// } +// } + +// /// Ensure the kernel's virtual memory layout is free of overlaps. +// #[test_case] +// fn virt_mem_layout_has_no_overlaps() { +// let layout = [ +// virt_boot_core_stack_region(), +// virt_code_region(), +// virt_data_region(), +// ]; + +// for (i, first_range) in layout.iter().enumerate() { +// for second_range in layout.iter().skip(i + 1) { +// assert!(!first_range.overlaps(second_range)) +// } +// } +// } diff --git a/tests/nucleus.rs b/tests/nucleus.rs new file mode 100644 index 000000000..e44fdb377 --- /dev/null +++ b/tests/nucleus.rs @@ -0,0 +1,40 @@ +#![no_std] +#![no_main] +#![feature(custom_test_frameworks)] +#![test_runner(libtest::test_runner)] +#![reexport_test_harness_main = "test_main"] + +use liblog::info; + +// mod common; + +// fn check_data_abort_trap() { +// // Cause an exception by accessing a virtual address for which no +// // address translations have been set up. +// // +// // This line of code accesses the address 2 GiB, but page tables are +// // only set up for the range [0..1) GiB. +// let big_addr: u64 = 2 * 1024 * 1024 * 1024; +// unsafe { core::ptr::read_volatile(big_addr as *mut u64) }; + +// info!("[i] Whoa! We recovered from an exception."); +// } + +// #[test_case] +// fn test_data_abort_trap() { +// libexception::exception::handling_init(); + +// let phys_kernel_tables_base_addr = match unsafe { libmemory::mmu::kernel_map_binary() } { +// Err(string) => panic!("Error mapping kernel binary: {}", string), +// Ok(addr) => addr, +// }; + +// if let Err(e) = unsafe { libmemory::mmu::enable_mmu_and_caching(phys_kernel_tables_base_addr) } +// { +// panic!("Enabling MMU failed: {}", e); +// } + +// libmemory::mmu::post_enable_init(); + +// check_data_abort_trap() //-- this needs setup and properly configured mmu to recover ("test mode" in libmemory) +// } diff --git a/userspace/README.md b/userspace/README.md new file mode 100644 index 000000000..0b69010e1 --- /dev/null +++ b/userspace/README.md @@ -0,0 +1 @@ +This should move outside of vesper repo completely, as well as many other userspace drivers. diff --git a/userspace/graphics.rs b/userspace/graphics.rs new file mode 100644 index 000000000..0831ba289 --- /dev/null +++ b/userspace/graphics.rs @@ -0,0 +1,33 @@ +// fn check_display_init() { +// display_graphics() +// .map_err(|e| { +// warn!("Error in display: {}", e); +// }) +// .ok(); +// } +// +// fn display_graphics() -> Result<(), DrawError> { +// if let Ok(mut display) = VC::init_fb(800, 600, 32) { +// info!("Display created"); +// +// display.clear(Color::black()); +// info!("Display cleared"); +// +// display.rect(10, 10, 250, 250, Color::rgb(32, 96, 64)); +// display.draw_text(50, 50, "Hello there!", Color::rgb(128, 192, 255))?; +// +// let mut buf = [0u8; 64]; +// let s = machine::write_to::show(&mut buf, format_args!("Display width {}", display.width)); +// +// if s.is_err() { +// display.draw_text(50, 150, "Error displaying", Color::red())? +// } else { +// display.draw_text(50, 150, s.unwrap(), Color::white())? +// } +// +// display.draw_text(150, 50, "RED", Color::red())?; +// display.draw_text(160, 60, "GREEN", Color::green())?; +// display.draw_text(170, 70, "BLUE", Color::blue())?; +// } +// Ok(()) +// } diff --git a/userspace/privileged/process.rs b/userspace/privileged/process.rs new file mode 100644 index 000000000..8b5593df2 --- /dev/null +++ b/userspace/privileged/process.rs @@ -0,0 +1,2 @@ +// create process +// diff --git a/userspace/privileged/scheduler.rs b/userspace/privileged/scheduler.rs new file mode 100644 index 000000000..dcaaa73ed --- /dev/null +++ b/userspace/privileged/scheduler.rs @@ -0,0 +1 @@ +// schedule processes by selecting next process to run and invoking its upcall key diff --git a/userspace/shell.rs b/userspace/shell.rs new file mode 100644 index 000000000..51d6856c3 --- /dev/null +++ b/userspace/shell.rs @@ -0,0 +1,70 @@ +fn print_mmu_state_and_features() { + // use machine::memory::mmu::interface::MMU; + features::print_mmu_features(); +} + +// TODO: AFTER KICKSTART, one of the userspace processes +// +//------------------------------------------------------------ +// Start a command prompt +//------------------------------------------------------------ +fn command_prompt() { + 'cmd_loop: loop { + let mut buf = [0_u8; 64]; + + match libconsole::console::command_prompt(&mut buf) { + // b"mmu" => init_mmu(), + b"feats" => print_mmu_state_and_features(), + // b"disp" => check_display_init(), + // b"trap" => check_data_abort_trap(), + // b"map" => machine::platform::memory::mmu::virt_mem_layout().print_layout(), + // b"led on" => set_led(true), + // b"led off" => set_led(false), + b"help" => print_help(), + b"end" => break 'cmd_loop, + x => warn!("[!] Unknown command {x:?}, try 'help'"), + } + } +} + +fn print_help() { + println!("Supported console commands:"); + println!(" mmu - initialize MMU"); + println!(" feats - print MMU state and supported features"); + #[cfg(not(feature = "noserial"))] + println!(" uart - try to reinitialize UART serial"); + // println!(" disp - try to init VC framebuffer and draw some text"); + println!(" trap - trigger and recover from a data abort exception"); + println!(" map - show kernel memory layout"); + // println!(" led [on|off] - change RPi LED status"); + println!(" end - leave console and reset board"); +} + +// fn set_led(enable: bool) { +// let mut mbox = Mailbox::<8>::default(); +// let index = mbox.request(); +// let index = mbox.set_led_on(index, enable); +// let mbox = mbox.end(index); +// +// mbox.call(channel::PropertyTagsArmToVc) +// .map_err(|e| { +// warn!("Mailbox call returned error {}", e); +// warn!("Mailbox contents: {:?}", mbox); +// }) +// .ok(); +// } + +fn reboot() -> ! { + cfg_if! { + if #[cfg(feature = "qemu")] { + info!("Bye, shutting down QEMU"); + libqemu::semihosting::exit_success() + } else { + // use machine::platform::raspberrypi::power::Power; + + info!("Bye, going to reset now"); + // Power::default().reset() + libcpu::endless_sleep() + } + } +}