diff --git a/.github/actions/install-nix-action/action.yaml b/.github/actions/install-nix-action/action.yaml deleted file mode 100644 index 535ae9d08fd9..000000000000 --- a/.github/actions/install-nix-action/action.yaml +++ /dev/null @@ -1,124 +0,0 @@ -name: "Install Nix" -description: "Helper action for installing Nix with support for dogfooding from master" -inputs: - dogfood: - description: "Whether to use Nix installed from the latest artifact from master branch" - required: true # Be explicit about the fact that we are using unreleased artifacts - experimental-installer: - description: "Whether to use the experimental installer to install Nix" - default: false - experimental-installer-version: - description: "Version of the experimental installer to use. If `latest`, the newest artifact from the default branch is used." - # TODO: This should probably be pinned to a release after https://github.com/NixOS/experimental-nix-installer/pull/49 lands in one - default: "latest" - extra_nix_config: - description: "Gets appended to `/etc/nix/nix.conf` if passed." - install_url: - description: "URL of the Nix installer" - required: false - default: "https://releases.nixos.org/nix/nix-2.32.1/install" - tarball_url: - description: "URL of the Nix tarball to use with the experimental installer" - required: false - github_token: - description: "Github token" - required: true - use_cache: - description: "Whether to setup github actions cache (not implemented currently)" - default: false - required: false -runs: - using: "composite" - steps: - - name: "Download nix install artifact from master" - shell: bash - id: download-nix-installer - if: inputs.dogfood == 'true' - run: | - RUN_ID=$(gh run list --repo "$DOGFOOD_REPO" --workflow ci.yml --branch master --status success --json databaseId --jq ".[0].databaseId") - - if [ "$RUNNER_OS" == "Linux" ]; then - INSTALLER_ARTIFACT="installer-linux" - elif [ "$RUNNER_OS" == "macOS" ]; then - INSTALLER_ARTIFACT="installer-darwin" - else - echo "::error ::Unsupported RUNNER_OS: $RUNNER_OS" - exit 1 - fi - - INSTALLER_DOWNLOAD_DIR="$GITHUB_WORKSPACE/$INSTALLER_ARTIFACT" - mkdir -p "$INSTALLER_DOWNLOAD_DIR" - - gh run download "$RUN_ID" --repo "$DOGFOOD_REPO" -n "$INSTALLER_ARTIFACT" -D "$INSTALLER_DOWNLOAD_DIR" - echo "installer-path=file://$INSTALLER_DOWNLOAD_DIR" >> "$GITHUB_OUTPUT" - TARBALL_PATH="$(find "$INSTALLER_DOWNLOAD_DIR" -name 'nix*.tar.xz' -print | head -n 1)" - echo "tarball-path=file://$TARBALL_PATH" >> "$GITHUB_OUTPUT" - - echo "::notice ::Dogfooding Nix installer from master (https://github.com/$DOGFOOD_REPO/actions/runs/$RUN_ID)" - env: - GH_TOKEN: ${{ inputs.github_token }} - DOGFOOD_REPO: "NixOS/nix" - - name: "Gather system info for experimental installer" - shell: bash - if: ${{ inputs.experimental-installer == 'true' }} - run: | - echo "::notice Using experimental installer from $EXPERIMENTAL_INSTALLER_REPO (https://github.com/$EXPERIMENTAL_INSTALLER_REPO)" - - if [ "$RUNNER_OS" == "Linux" ]; then - EXPERIMENTAL_INSTALLER_SYSTEM="linux" - echo "EXPERIMENTAL_INSTALLER_SYSTEM=$EXPERIMENTAL_INSTALLER_SYSTEM" >> "$GITHUB_ENV" - elif [ "$RUNNER_OS" == "macOS" ]; then - EXPERIMENTAL_INSTALLER_SYSTEM="darwin" - echo "EXPERIMENTAL_INSTALLER_SYSTEM=$EXPERIMENTAL_INSTALLER_SYSTEM" >> "$GITHUB_ENV" - else - echo "::error ::Unsupported RUNNER_OS: $RUNNER_OS" - exit 1 - fi - - if [ "$RUNNER_ARCH" == "X64" ]; then - EXPERIMENTAL_INSTALLER_ARCH=x86_64 - echo "EXPERIMENTAL_INSTALLER_ARCH=$EXPERIMENTAL_INSTALLER_ARCH" >> "$GITHUB_ENV" - elif [ "$RUNNER_ARCH" == "ARM64" ]; then - EXPERIMENTAL_INSTALLER_ARCH=aarch64 - echo "EXPERIMENTAL_INSTALLER_ARCH=$EXPERIMENTAL_INSTALLER_ARCH" >> "$GITHUB_ENV" - else - echo "::error ::Unsupported RUNNER_ARCH: $RUNNER_ARCH" - exit 1 - fi - - echo "EXPERIMENTAL_INSTALLER_ARTIFACT=nix-installer-$EXPERIMENTAL_INSTALLER_ARCH-$EXPERIMENTAL_INSTALLER_SYSTEM" >> "$GITHUB_ENV" - env: - EXPERIMENTAL_INSTALLER_REPO: "NixOS/experimental-nix-installer" - - name: "Download latest experimental installer" - shell: bash - id: download-latest-experimental-installer - if: ${{ inputs.experimental-installer == 'true' && inputs.experimental-installer-version == 'latest' }} - run: | - RUN_ID=$(gh run list --repo "$EXPERIMENTAL_INSTALLER_REPO" --workflow ci.yml --branch main --status success --json databaseId --jq ".[0].databaseId") - - EXPERIMENTAL_INSTALLER_DOWNLOAD_DIR="$GITHUB_WORKSPACE/$EXPERIMENTAL_INSTALLER_ARTIFACT" - mkdir -p "$EXPERIMENTAL_INSTALLER_DOWNLOAD_DIR" - - gh run download "$RUN_ID" --repo "$EXPERIMENTAL_INSTALLER_REPO" -n "$EXPERIMENTAL_INSTALLER_ARTIFACT" -D "$EXPERIMENTAL_INSTALLER_DOWNLOAD_DIR" - # Executable permissions are lost in artifacts - find $EXPERIMENTAL_INSTALLER_DOWNLOAD_DIR -type f -exec chmod +x {} + - echo "installer-path=$EXPERIMENTAL_INSTALLER_DOWNLOAD_DIR" >> "$GITHUB_OUTPUT" - env: - GH_TOKEN: ${{ inputs.github_token }} - EXPERIMENTAL_INSTALLER_REPO: "NixOS/experimental-nix-installer" - - uses: cachix/install-nix-action@c134e4c9e34bac6cab09cf239815f9339aaaf84e # v31.5.1 - if: ${{ inputs.experimental-installer != 'true' }} - with: - # Ternary operator in GHA: https://www.github.com/actions/runner/issues/409#issuecomment-752775072 - install_url: ${{ inputs.dogfood == 'true' && format('{0}/install', steps.download-nix-installer.outputs.installer-path) || inputs.install_url }} - install_options: ${{ inputs.dogfood == 'true' && format('--tarball-url-prefix {0}', steps.download-nix-installer.outputs.installer-path) || '' }} - extra_nix_config: ${{ inputs.extra_nix_config }} - - uses: DeterminateSystems/nix-installer-action@786fff0690178f1234e4e1fe9b536e94f5433196 # v20 - if: ${{ inputs.experimental-installer == 'true' }} - with: - diagnostic-endpoint: "" - # TODO: It'd be nice to use `artifacts.nixos.org` for both of these, maybe through an `/experimental-installer/latest` endpoint? or `/commit/`? - local-root: ${{ inputs.experimental-installer-version == 'latest' && steps.download-latest-experimental-installer.outputs.installer-path || '' }} - source-url: ${{ inputs.experimental-installer-version != 'latest' && 'https://artifacts.nixos.org/experimental-installer/tag/${{ inputs.experimental-installer-version }}/${{ env.EXPERIMENTAL_INSTALLER_ARTIFACT }}' || '' }} - nix-package-url: ${{ inputs.dogfood == 'true' && steps.download-nix-installer.outputs.tarball-path || (inputs.tarball_url || '') }} - extra-conf: ${{ inputs.extra_nix_config }} diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 665b1c0a12ba..0fa07991d353 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -53,7 +53,7 @@ jobs: - run: nix build .#packages.${{ inputs.system }}.default .#packages.${{ inputs.system }}.binaryTarball --no-link -L - run: nix build .#packages.${{ inputs.system }}.binaryTarball --out-link tarball - run: nix build .#packages.${{ inputs.system }}.nix-cli-static --no-link -L - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@v6 with: name: ${{ inputs.system }} path: ./tarball/*.xz diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 08000ac4c871..684a48323895 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -98,7 +98,7 @@ jobs: run: mkdir -p ./artifacts - name: Fetch artifacts - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v7 with: path: downloaded - name: Move downloaded artifacts to artifacts directory diff --git a/.gitignore b/.gitignore index 4782bfbafd27..96c134335232 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,7 @@ # Default meson build dir /build +# Meson creates this file too +src/.wraplock # /tests/functional/ /tests/functional/common/subst-vars.sh @@ -14,6 +16,10 @@ /tests/functional/lang/*.err /tests/functional/lang/*.ast +# /tests/functional/cli-characterisation/ +/tests/functional/cli-characterisation/*.out +/tests/functional/cli-characterisation/*.err + /outputs *~ diff --git a/.version b/.version index dc148c42f45d..138ac199ffd4 100644 --- a/.version +++ b/.version @@ -1 +1 @@ -2.33.3 +2.34.4 diff --git a/ci/gha/tests/default.nix b/ci/gha/tests/default.nix index 4a30a96a475d..7d206750fcfe 100644 --- a/ci/gha/tests/default.nix +++ b/ci/gha/tests/default.nix @@ -77,6 +77,7 @@ rec { topLevel = { installerScriptForGHA = hydraJobs.installerScriptForGHA.${system}; nixpkgsLibTests = hydraJobs.tests.nixpkgsLibTests.${system}; + nixpkgsLibTestsLazy = hydraJobs.tests.nixpkgsLibTestsLazy.${system}; rl-next = pkgs.buildPackages.runCommand "test-rl-next-release-notes" { } '' LANG=C.UTF-8 ${pkgs.changelog-d}/bin/changelog-d ${../../../doc/manual/rl-next} >$out ''; diff --git a/ci/gha/tests/windows.nix b/ci/gha/tests/windows.nix new file mode 100644 index 000000000000..5f13f51dfe35 --- /dev/null +++ b/ci/gha/tests/windows.nix @@ -0,0 +1,30 @@ +{ + nixFlake ? builtins.getFlake ("git+file://" + toString ../../..), + system ? builtins.currentSystem, + pkgs ? nixFlake.inputs.nixpkgs.legacyPackages.${system}, +}: + +let + packages = nixFlake.packages.${system}; + + fixOutput = + test: + test.overrideAttrs (prev: { + nativeBuildInputs = prev.nativeBuildInputs or [ ] ++ [ pkgs.colorized-logs ]; + env.GTEST_COLOR = "no"; + # Wine's console emulation wraps every character in ANSI cursor + # hide/show sequences, making logs unreadable in GitHub Actions. + buildCommand = '' + set -o pipefail + { + ${prev.buildCommand} + } 2>&1 | ansi2txt + ''; + }); +in + +{ + unitTests = { + "nix-util-tests" = fixOutput packages."nix-util-tests-x86_64-w64-mingw32".passthru.tests.run; + }; +} diff --git a/doc/manual/meson.build b/doc/manual/meson.build index 1b9a325df2ac..4f9a55b515bf 100644 --- a/doc/manual/meson.build +++ b/doc/manual/meson.build @@ -16,7 +16,6 @@ bash = find_program('bash', native : true) # HTML manual dependencies (conditional) if get_option('html-manual') mdbook = find_program('mdbook', native : true) - rsync = find_program('rsync', required : true, native : true) endif pymod = import('python') @@ -116,7 +115,12 @@ if get_option('html-manual') @0@ @INPUT0@ @CURRENT_SOURCE_DIR@ > @DEPFILE@ @0@ @INPUT1@ summary @2@ < @CURRENT_SOURCE_DIR@/source/SUMMARY.md.in > @2@/source/SUMMARY.md sed -e 's|@version@|@3@|g' < @INPUT2@ > @2@/book.toml - @4@ -r -L --exclude='*.drv' --include='*.md' @CURRENT_SOURCE_DIR@/ @2@/ + # Copy source to build directory, excluding the build directory itself + # (which is present when built as an individual component). + # Use tar with --dereference to copy symlink targets (e.g., JSON examples from tests). + (cd @CURRENT_SOURCE_DIR@ && find . -mindepth 1 -maxdepth 1 ! -name build | tar -c --dereference -T - -f -) | (cd @2@ && tar -xf -) + chmod -R u+w @2@ + find @2@ -name '*.drv' -delete (cd @2@; RUST_LOG=warn @1@ build -d @2@ 3>&2 2>&1 1>&3) | { grep -Fv "because fragment resolution isn't implemented" || :; } 3>&2 2>&1 1>&3 rm -rf @2@/manual mv @2@/html @2@/manual @@ -128,7 +132,6 @@ if get_option('html-manual') mdbook.full_path(), meson.current_build_dir(), fs.read('../../.version-determinate').strip(), - rsync.full_path(), ), ], input : [ diff --git a/doc/manual/package.nix b/doc/manual/package.nix index 0b3d8ca940a2..9e69156ac0ca 100644 --- a/doc/manual/package.nix +++ b/doc/manual/package.nix @@ -1,5 +1,6 @@ { lib, + stdenv, callPackage, mkMesonDerivation, runCommand, @@ -10,7 +11,6 @@ mdbook, jq, python3, - rsync, nix-cli, changelog-d, json-schema-for-humans, @@ -55,6 +55,8 @@ mkMesonDerivation (finalAttrs: { ../../src/libstore-tests/data/nar-info ../../src/libstore-tests/data/build-result ../../src/libstore-tests/data/dummy-store + # For derivation examples referenced by symlinks in doc/manual/source/protocols/json/schema/ + ../../tests/functional/derivation # Too many different types of files to filter for now ../../doc/manual ./. @@ -91,13 +93,13 @@ mkMesonDerivation (finalAttrs: { ] ++ lib.optionals buildHtmlManual [ mdbook - rsync json-schema-for-humans ] - ++ lib.optionals (!officialRelease && buildHtmlManual) [ + ++ lib.optionals (!officialRelease && buildHtmlManual && !stdenv.hostPlatform.isi686) [ # When not an official release, we likely have changelog entries that have # yet to be rendered. # When released, these are rendered into a committed file to save a dependency. + # Broken on i686. changelog-d ]; diff --git a/doc/manual/rl-next/c-api-new-store-methods.md b/doc/manual/rl-next/c-api-new-store-methods.md deleted file mode 100644 index 28792e7cc42d..000000000000 --- a/doc/manual/rl-next/c-api-new-store-methods.md +++ /dev/null @@ -1,9 +0,0 @@ ---- -synopsis: "C API: New store API methods" -prs: [14766] ---- - -The C API now includes additional methods: - -- `nix_store_query_path_from_hash_part()` - Get the full store path given its hash part -- `nix_store_copy_path()` - Copy a single store path between two stores, allows repairs and configuring signature checking diff --git a/doc/manual/source/SUMMARY.md.in b/doc/manual/source/SUMMARY.md.in index df3637e48a39..c58ed5a6b1aa 100644 --- a/doc/manual/source/SUMMARY.md.in +++ b/doc/manual/source/SUMMARY.md.in @@ -128,7 +128,9 @@ - [Serving Tarball Flakes](protocols/tarball-fetcher.md) - [Store Path Specification](protocols/store-path.md) - [Nix Archive (NAR) Format](protocols/nix-archive/index.md) + - [Nix Cache Info Format](protocols/nix-cache-info.md) - [Derivation "ATerm" file format](protocols/derivation-aterm.md) + - [Nix32 Encoding](protocols/nix32.md) - [`builtins.wasm` Host Interface](protocols/wasm.md) - [Flake Schemas](protocols/flake-schemas.md) - [C API](c-api.md) @@ -195,6 +197,7 @@ - [Release 3.0.0 (2025-03-04)](release-notes-determinate/rl-3.0.0.md) - [Nix Release Notes](release-notes/index.md) {{#include ./SUMMARY-rl-next.md}} + - [Release 2.34 (2026-02-27)](release-notes/rl-2.34.md) - [Release 2.33 (2025-12-09)](release-notes/rl-2.33.md) - [Release 2.32 (2025-10-06)](release-notes/rl-2.32.md) - [Release 2.31 (2025-08-21)](release-notes/rl-2.31.md) diff --git a/doc/manual/source/command-ref/env-common.md b/doc/manual/source/command-ref/env-common.md index fe6e822ff16a..35d682949ba8 100644 --- a/doc/manual/source/command-ref/env-common.md +++ b/doc/manual/source/command-ref/env-common.md @@ -57,11 +57,6 @@ Most Nix commands interpret the following environment variables: Overrides the location of the Nix store (default `prefix/store`). -- [`NIX_DATA_DIR`](#env-NIX_DATA_DIR) - - Overrides the location of the Nix static data directory (default - `prefix/share`). - - [`NIX_LOG_DIR`](#env-NIX_LOG_DIR) Overrides the location of the Nix log directory (default diff --git a/doc/manual/source/command-ref/files/default-nix-expression.md b/doc/manual/source/command-ref/files/default-nix-expression.md index e886e3ff4991..66ce84b48f8b 100644 --- a/doc/manual/source/command-ref/files/default-nix-expression.md +++ b/doc/manual/source/command-ref/files/default-nix-expression.md @@ -39,11 +39,11 @@ This makes all subscribed channels available as attributes in the default expres A symlink that ensures that [`nix-env`] can find the current user's channels: - `~/.nix-defexpr/channels` -- `$XDG_STATE_HOME/defexpr/channels` if [`use-xdg-base-directories`] is set to `true`. +- `$XDG_STATE_HOME/nix/defexpr/channels` if [`use-xdg-base-directories`] is set to `true`. This symlink points to: -- `$XDG_STATE_HOME/profiles/channels` for regular users +- `$XDG_STATE_HOME/nix/profiles/channels` for regular users - `$NIX_STATE_DIR/profiles/per-user/root/channels` for `root` In a multi-user installation, you may also have `~/.nix-defexpr/channels_root`, which links to the channels of the root user. diff --git a/doc/manual/source/development/building.md b/doc/manual/source/development/building.md index 9694183ba82e..917e39e1ca5f 100644 --- a/doc/manual/source/development/building.md +++ b/doc/manual/source/development/building.md @@ -1,5 +1,9 @@ # Building Nix +> **Note** +> +> When checking out the repo on Windows, make sure you have the git setting `core.symlinks` enabled, before cloning, as there are symlinks in the repo. + To build all dependencies and start a shell in which all environment variables are set up so that those dependencies can be found: ```console diff --git a/doc/manual/source/installation/building-source.md b/doc/manual/source/installation/building-source.md index d35cc18c21d9..6c409364a943 100644 --- a/doc/manual/source/installation/building-source.md +++ b/doc/manual/source/installation/building-source.md @@ -6,14 +6,23 @@ It is broken up into multiple Meson packages, which are optionally combined in a There are no mandatory extra steps to the building process: generic Meson installation instructions like [this](https://mesonbuild.com/Quick-guide.html#using-meson-as-a-distro-packager) should work. -The installation path can be specified by passing the `-Dprefix=prefix` -to `configure`. The default installation directory is `/usr/local`. You +```bash +git clone https://github.com/NixOS/nix.git +cd nix +meson setup build +cd build +ninja +(sudo) ninja install +``` + +The installation path can be specified by passing `-Dprefix=prefix` +to `meson setup build`. The default installation directory is `/usr/local`. You can change this to any location you like. You must have write permission to the *prefix* path. Nix keeps its *store* (the place where packages are stored) in `/nix/store` by default. This can be changed using -`-Dstore-dir=path`. +`-Dlibstore:store-dir=path`. > **Warning** > diff --git a/doc/manual/source/language/advanced-attributes.md b/doc/manual/source/language/advanced-attributes.md index f0b1a4c730e7..67612029c8a4 100644 --- a/doc/manual/source/language/advanced-attributes.md +++ b/doc/manual/source/language/advanced-attributes.md @@ -338,7 +338,7 @@ Here is more information on the `output*` attributes, and what values they may b This will specify the output hash of the single output of a [fixed-output derivation]. The `outputHash` attribute must be a string containing the hash in either hexadecimal or "nix32" encoding, or following the format for integrity metadata as defined by [SRI](https://www.w3.org/TR/SRI/). - The "nix32" encoding is an adaptation of base-32 encoding. + The ["nix32" encoding](@docroot@/protocols/nix32.md) is Nix's variant of base-32 encoding. > **Note** > diff --git a/doc/manual/source/language/syntax.md b/doc/manual/source/language/syntax.md index b127aca14c12..80046d4845b9 100644 --- a/doc/manual/source/language/syntax.md +++ b/doc/manual/source/language/syntax.md @@ -51,6 +51,7 @@ See [String literals](string-literals.md). Path literals can also include [string interpolation], besides being [interpolated into other expressions]. + [string interpolation]: ./string-interpolation.md [interpolated into other expressions]: ./string-interpolation.md#interpolated-expression At least one slash (`/`) must appear *before* any interpolated expression for the result to be recognized as a path. @@ -272,7 +273,7 @@ will crash with an `infinite recursion encountered` error message. A let-expression allows you to define local variables for an expression. -> *let-in* = `let` [ *identifier* = *expr* ]... `in` *expr* +> *let-in* = `let` [ *identifier* = *expr* `;` ]... `in` *expr* Example: @@ -285,6 +286,27 @@ in x + y This evaluates to `"foobar"`. +There is also another, older, syntax for let expressions that should not be used in new code: + +> *let* = `let` `{` *identifier* = *expr* `;` [ *identifier* = *expr* `;`]... `}` + +In this form, the attribute set between the `{` `}` is recursive. + +One of the attributes must have the special name `body`, +which is the result of the expression. + +Example: + +```nix +let { + foo = bar; + bar = "baz"; + body = foo; +} +``` + +This evaluates to "baz". + ## Inheriting attributes When defining an [attribute set](./types.md#type-attrs) or in a [let-expression](#let-expressions) it is often convenient to copy variables from the surrounding lexical scope (e.g., when you want to propagate attributes). diff --git a/doc/manual/source/package-management/binary-cache-substituter.md b/doc/manual/source/package-management/binary-cache-substituter.md index 855eaf4705ad..e6a772213d6d 100644 --- a/doc/manual/source/package-management/binary-cache-substituter.md +++ b/doc/manual/source/package-management/binary-cache-substituter.md @@ -19,17 +19,16 @@ whatever port you like: $ nix-serve -p 8080 ``` -To check whether it works, try the following on the client: +To check whether it works, try fetching the [`nix-cache-info`](@docroot@/protocols/nix-cache-info.md) file on the client: ```console $ curl http://avalon:8080/nix-cache-info +StoreDir: /nix/store +WantMassQuery: 1 +Priority: 30 ``` -which should print something like: - - StoreDir: /nix/store - WantMassQuery: 1 - Priority: 30 +When writing to a binary cache (e.g., with [`nix copy`](@docroot@/command-ref/new-cli/nix3-copy.md)), Nix creates [`nix-cache-info`](@docroot@/protocols/nix-cache-info.md) automatically if it doesn't exist. On the client side, you can tell Nix to use your binary cache using `--substituters`, e.g.: diff --git a/doc/manual/source/protocols/json/build-trace-entry.md b/doc/manual/source/protocols/json/build-trace-entry.md index 8050a2840bfa..9eea93712601 100644 --- a/doc/manual/source/protocols/json/build-trace-entry.md +++ b/doc/manual/source/protocols/json/build-trace-entry.md @@ -1,27 +1,21 @@ -{{#include build-trace-entry-v1-fixed.md}} +{{#include build-trace-entry-v2-fixed.md}} ## Examples ### Simple build trace entry ```json -{{#include schema/build-trace-entry-v1/simple.json}} -``` - -### Build trace entry with dependencies - -```json -{{#include schema/build-trace-entry-v1/with-dependent-realisations.json}} +{{#include schema/build-trace-entry-v2/simple.json}} ``` ### Build trace entry with signature ```json -{{#include schema/build-trace-entry-v1/with-signature.json}} +{{#include schema/build-trace-entry-v2/with-signature.json}} ``` \ No newline at end of file +[JSON Schema for Build Trace Entry v1](schema/build-trace-entry-v2.json) +--> diff --git a/doc/manual/source/protocols/json/meson.build b/doc/manual/source/protocols/json/meson.build index e32cf06408bf..c7c48c4ed6e2 100644 --- a/doc/manual/source/protocols/json/meson.build +++ b/doc/manual/source/protocols/json/meson.build @@ -17,7 +17,7 @@ schemas = [ 'derivation-v4', 'derivation-options-v1', 'deriving-path-v1', - 'build-trace-entry-v1', + 'build-trace-entry-v2', 'build-result-v1', 'store-v1', ] diff --git a/doc/manual/source/protocols/json/schema/build-result-v1.yaml b/doc/manual/source/protocols/json/schema/build-result-v1.yaml index 31f59a44ddac..55e55d3e5cbe 100644 --- a/doc/manual/source/protocols/json/schema/build-result-v1.yaml +++ b/doc/manual/source/protocols/json/schema/build-result-v1.yaml @@ -83,7 +83,7 @@ properties: description: | A mapping from output names to their build trace entries. additionalProperties: - "$ref": "build-trace-entry-v1.yaml" + "$ref": "build-trace-entry-v2.yaml" failure: type: object diff --git a/doc/manual/source/protocols/json/schema/build-trace-entry-v1 b/doc/manual/source/protocols/json/schema/build-trace-entry-v2 similarity index 100% rename from doc/manual/source/protocols/json/schema/build-trace-entry-v1 rename to doc/manual/source/protocols/json/schema/build-trace-entry-v2 diff --git a/doc/manual/source/protocols/json/schema/build-trace-entry-v1.yaml b/doc/manual/source/protocols/json/schema/build-trace-entry-v2.yaml similarity index 74% rename from doc/manual/source/protocols/json/schema/build-trace-entry-v1.yaml rename to doc/manual/source/protocols/json/schema/build-trace-entry-v2.yaml index a85738b50b98..4340f82388c2 100644 --- a/doc/manual/source/protocols/json/schema/build-trace-entry-v1.yaml +++ b/doc/manual/source/protocols/json/schema/build-trace-entry-v2.yaml @@ -1,5 +1,5 @@ "$schema": "http://json-schema.org/draft-04/schema" -"$id": "https://nix.dev/manual/nix/latest/protocols/json/schema/build-trace-entry-v1.json" +"$id": "https://nix.dev/manual/nix/latest/protocols/json/schema/build-trace-entry-v2.json" title: Build Trace Entry description: | A record of a successful build outcome for a specific derivation output. @@ -11,10 +11,17 @@ description: | > This JSON format is currently > [**experimental**](@docroot@/development/experimental-features.md#xp-feature-ca-derivations) > and subject to change. + + Verision history: + + - Version 1: Original format + + - Version 2: Remove `dependentRealisations` + +type: object required: - id - outPath - - dependentRealisations - signatures allOf: - "$ref": "#/$defs/key" @@ -22,9 +29,11 @@ allOf: properties: id: {} outPath: {} - dependentRealisations: {} signatures: {} -additionalProperties: false +additionalProperties: + dependentRealisations: + description: deprecated field + type: object "$defs": key: @@ -60,7 +69,6 @@ additionalProperties: false type: object required: - outPath - - dependentRealisations - signatures properties: outPath: @@ -69,19 +77,6 @@ additionalProperties: false description: | The path to the store object that resulted from building this derivation for the given output name. - dependentRealisations: - type: object - title: Underlying Base Build Trace - description: | - This is for [*derived*](@docroot@/store/build-trace.md#derived) build trace entries to ensure coherence. - - Keys are derivation output IDs (same format as the main `id` field). - Values are the store paths that those dependencies resolved to. - - As described in the linked section on derived build trace traces, derived build trace entries must be kept in addition and not instead of the underlying base build entries. - This is the set of base build trace entries that this derived build trace is derived from. - (The set is also a map since this miniature base build trace must be coherent, mapping each key to a single value.) - patternProperties: "^sha256:[0-9a-f]{64}![a-zA-Z_][a-zA-Z0-9_-]*$": "$ref": "store-path-v1.yaml" diff --git a/doc/manual/source/protocols/json/schema/store-path-v1.yaml b/doc/manual/source/protocols/json/schema/store-path-v1.yaml index 61653d60e214..3cd7c56fcf79 100644 --- a/doc/manual/source/protocols/json/schema/store-path-v1.yaml +++ b/doc/manual/source/protocols/json/schema/store-path-v1.yaml @@ -18,7 +18,7 @@ description: | The format follows this pattern: `${digest}-${name}` - - **hash**: Digest rendered in a custom variant of [Base32](https://en.wikipedia.org/wiki/Base32) (20 arbitrary bytes become 32 ASCII characters) + - **hash**: Digest rendered in [Nix32](@docroot@/protocols/nix32.md), a variant of base-32 (20 hash bytes become 32 ASCII characters) - **name**: The package name and optional version/suffix information type: string diff --git a/doc/manual/source/protocols/json/schema/store-v1.yaml b/doc/manual/source/protocols/json/schema/store-v1.yaml index 31aa10c41476..e3e09c699621 100644 --- a/doc/manual/source/protocols/json/schema/store-v1.yaml +++ b/doc/manual/source/protocols/json/schema/store-v1.yaml @@ -70,7 +70,7 @@ properties: "^[A-Za-z0-9+/]{43}=$": type: object additionalProperties: - "$ref": "./build-trace-entry-v1.yaml#/$defs/value" + "$ref": "./build-trace-entry-v2.yaml#/$defs/value" additionalProperties: false "$defs": diff --git a/doc/manual/source/protocols/nix-cache-info.md b/doc/manual/source/protocols/nix-cache-info.md new file mode 100644 index 000000000000..e8351e1cebe8 --- /dev/null +++ b/doc/manual/source/protocols/nix-cache-info.md @@ -0,0 +1,55 @@ +# Nix Cache Info Format + +The `nix-cache-info` file is a metadata file at the root of a [binary cache](@docroot@/package-management/binary-cache-substituter.md) (e.g., `https://cache.example.com/nix-cache-info`). + +MIME type: `text/x-nix-cache-info` + +## Format + +Line-based key-value format: + +``` +Key: value +``` + +Leading and trailing whitespace is trimmed from values. +Lines without a colon are ignored. +Unknown keys are silently ignored. + +## Fields + +### `StoreDir` + +The Nix store directory path that this cache was built for (e.g., `/nix/store`). + +If present, Nix verifies that this matches the client's store directory: + +``` +error: binary cache 'https://example.com' is for Nix stores with prefix '/nix/store', not '/home/user/nix/store' +``` + +### `WantMassQuery` + +`1` or `0`. Sets the default for [`want-mass-query`](@docroot@/store/types/http-binary-cache-store.md#store-http-binary-cache-store-want-mass-query). + +### `Priority` + +Integer. Sets the default for [`priority`](@docroot@/store/types/http-binary-cache-store.md#store-http-binary-cache-store-priority). + +## Example + +``` +StoreDir: /nix/store +WantMassQuery: 1 +Priority: 30 +``` + +## Caching Behavior + +Nix caches `nix-cache-info` in the [cache directory](@docroot@/command-ref/env-common.md#env-NIX_CACHE_HOME) with a 7-day TTL. + +## See Also + +- [HTTP Binary Cache Store](@docroot@/store/types/http-binary-cache-store.md) +- [Serving a Nix store via HTTP](@docroot@/package-management/binary-cache-substituter.md) +- [`substituters`](@docroot@/command-ref/conf-file.md#conf-substituters) diff --git a/doc/manual/source/protocols/nix32.md b/doc/manual/source/protocols/nix32.md new file mode 100644 index 000000000000..72afe893ea24 --- /dev/null +++ b/doc/manual/source/protocols/nix32.md @@ -0,0 +1,19 @@ +# Nix32 Encoding + +Nix32 is Nix's variant of base-32 encoding, used for [store path digests](@docroot@/protocols/store-path.md), hash output via [`nix hash`](@docroot@/command-ref/new-cli/nix3-hash.md), and the [`outputHash`](@docroot@/language/advanced-attributes.md#adv-attr-outputHash) derivation attribute. + +## Alphabet + +The Nix32 alphabet consists of these 32 characters: + +``` +0 1 2 3 4 5 6 7 8 9 a b c d f g h i j k l m n p q r s v w x y z +``` + +The letters `e`, `o`, `u`, and `t` are omitted. + +## Byte Order + +Nix32 encoding processes the hash bytes from the end (last byte first), while base-16 encoding processes from the beginning (first byte first). + +Consequently, the string sort order is determined primarily by the first bytes for base-16, and by the last bytes for Nix32. diff --git a/doc/manual/source/protocols/store-path.md b/doc/manual/source/protocols/store-path.md index 5be2355015f6..1aa79615d1c8 100644 --- a/doc/manual/source/protocols/store-path.md +++ b/doc/manual/source/protocols/store-path.md @@ -20,12 +20,11 @@ where - `store-dir` = the [store directory](@docroot@/store/store-path.md#store-directory) -- `digest` = base-32 representation of the compressed to 160 bits [SHA-256] hash of `fingerprint` +- `digest` = base-32 representation of the compressed to 160 bits [SHA-256] hash of `fingerprint`. -For the definition of the hash compression algorithm, please refer to the section 5.1 of -the [Nix thesis](https://edolstra.github.io/pubs/phd-thesis.pdf), which also defines the -specifics of base-32 encoding. Note that base-32 encoding processes the hash bytestring from -the end, while base-16 processes in from the beginning. + Nix uses a custom base-32 encoding called [Nix32](@docroot@/protocols/nix32.md). + + For the definition of the hash compression algorithm, please refer to section 5.1 of the [Nix thesis](https://edolstra.github.io/pubs/phd-thesis.pdf). ## Fingerprint diff --git a/doc/manual/source/release-notes/rl-2.34.md b/doc/manual/source/release-notes/rl-2.34.md new file mode 100644 index 000000000000..89017f1767cf --- /dev/null +++ b/doc/manual/source/release-notes/rl-2.34.md @@ -0,0 +1,419 @@ +# Release 2.34.0 (2026-02-27) + +## Highlights + +- Rust nix-installer in beta + + The Rust-based rewrite of the Nix installer is now in beta. + We'd love help testing it out! + + To test out the new installer, run: + ``` + curl -sSfL https://artifacts.nixos.org/nix-installer | sh -s -- install + ``` + + This installer can be run even when you have an existing, script-based Nix installation without any adjustments. + + This new installer also comes with the ability to uninstall your Nix installation; run: + ``` + /nix/nix-installer uninstall + ``` + + This will get rid of your entire Nix installation (even if you installed over an existing, script-based installation). + + This installer is a modified version of the [Determinate Nix Installer](https://github.com/DeterminateSystems/nix-installer) by Determinate Systems. + Thanks to Determinate Systems for all the investment they've put into the installer. + + Source for the installer is in . + Report any issues in that repo. + + For CI usage, a GitHub Action to install Nix using this installer is available at . + +- Stabilisation of `no-url-literals` experimental feature and new diagnostics infrastructure, with `lint-url-literals`, `lint-short-path-literals`, and `lint-absolute-path-literals` settings [#8738](https://github.com/NixOS/nix/issues/8738) [#10048](https://github.com/NixOS/nix/issues/10048) [#10281](https://github.com/NixOS/nix/issues/10281) [#15326](https://github.com/NixOS/nix/pull/15326) + + Experimental feature `no-url-literals` has been stabilised and is now controlled by the `lint-url-literals` option. + New diagnostics infrastructure has been added for linting discouraged language features. + + ### New lint infrastructure + + #### [`lint-url-literals`](@docroot@/command-ref/conf-file.md#conf-lint-url-literals) + + The `no-url-literals` experimental feature has been stabilised and replaced with a new [`lint-url-literals`](@docroot@/command-ref/conf-file.md#conf-lint-url-literals) setting. + + To migrate from the experimental feature, replace: + ``` + experimental-features = no-url-literals + ``` + with: + ``` + lint-url-literals = fatal + ``` + + #### [`lint-short-path-literals`](@docroot@/command-ref/conf-file.md#conf-lint-short-path-literals) + + The [`warn-short-path-literals`](@docroot@/command-ref/conf-file.md#conf-warn-short-path-literals) boolean setting has been deprecated and replaced with [`lint-short-path-literals`](@docroot@/command-ref/conf-file.md#conf-lint-short-path-literals). + + To migrate, replace: + ``` + warn-short-path-literals = true + ``` + with: + ``` + lint-short-path-literals = warn + ``` + + #### [`lint-absolute-path-literals`](@docroot@/command-ref/conf-file.md#conf-lint-absolute-path-literals) + + A new [`lint-absolute-path-literals`](@docroot@/command-ref/conf-file.md#conf-lint-absolute-path-literals) setting has been added to control handling of absolute path literals (paths starting with `/`) and home path literals (paths starting with `~/`). + + #### Setting values + + All three settings accept three values: + - `ignore`: Allow the feature without emitting any diagnostic (default) + - `warn`: Emit a warning when the feature is used + - `fatal`: Treat the feature as a parse error + + The defaults may change in future versions. + +- Improved parser error messages [#15092](https://github.com/NixOS/nix/pull/15092) + + Parser error messages now use legible strings for tokens instead of internal names. For example, malformed expression `a ++ ++ b` now produces the following error: + ``` + error: syntax error, unexpected '++' + at «string»:1:6: + 1| a ++ ++ b + | ^ + ``` + + Instead of: + ``` + error: syntax error, unexpected CONCAT + at «string»:1:6: + 1| a ++ ++ b + | ^ + ``` + +## New features + +- `nix repl` now supports `inherit` and multiple bindings [#15082](https://github.com/NixOS/nix/pull/15082) + + The `nix repl` now supports `inherit` statements and multiple bindings per line: + + ``` + nix-repl> a = { x = 1; y = 2; } + nix-repl> inherit (a) x y + nix-repl> x + y + 3 + + nix-repl> p = 1; q = 2; + nix-repl> p + q + 3 + + nix-repl> foo.bar.baz = 1; + nix-repl> foo.bar + { baz = 1; } + ``` + +- New command `nix store roots-daemon` for serving GC roots [#15143](https://github.com/NixOS/nix/pull/15143) + + New command [`nix store roots-daemon`](@docroot@/command-ref/new-cli/nix3-store-roots-daemon.md) runs a daemon that serves garbage collector roots over a Unix domain socket. + It enables the garbage collector to discover runtime roots when the main Nix daemon doesn't have `CAP_SYS_PTRACE` capability and therefore cannot scan `/proc`. + + The garbage collector can be configured to use this daemon via the [`use-roots-daemon`](@docroot@/store/types/local-store.md#store-experimental-option-use-roots-daemon) store setting. + + This feature requires the [`local-overlay-store` experimental feature](@docroot@/development/experimental-features.md#xp-feature-local-overlay-store). + +- New command `nix-nswrapper` in `libexec` [#15183](https://github.com/NixOS/nix/pull/15183) + + The new command `libexec/nix-nswrapper` is used to run the Nix daemon in an unprivileged user namespace on Linux. In order to use this command, build user UIDs and GIDs must be allocated in `/etc/subuid` and `/etc/subgid`. + + It can be used to run the Nix daemon with full sandboxing without executing as root. Support has been added to Nixpkgs with the new `nix.daemonUser` and `nix.daemonGroup` settings. + +- New setting `ignore-gc-delete-failure` for local stores [#15054](https://github.com/NixOS/nix/pull/15054) + + A new local store setting [`ignore-gc-delete-failure`](@docroot@/store/types/local-store.md#store-local-store-ignore-gc-delete-failure) has been added. + When enabled, garbage collection will log warnings instead of failing when it cannot delete store paths. + This is useful when running Nix as an unprivileged user that may not have write access to all paths in the store. + + This setting is experimental and requires the [`local-overlay-store`](@docroot@/development/experimental-features.md#xp-feature-local-overlay-store) experimental feature. + +- New setting `narinfo-cache-meta-ttl` [#15287](https://github.com/NixOS/nix/pull/15287) + + The new setting `narinfo-cache-meta-ttl` controls how long binary cache metadata (i.e. `/nix-cache-info`) is cached locally, in seconds. This was previously hard-coded to 7 days, which is still the default. As a result, you can now use `nix store info --refresh` to check whether a binary cache is still valid. + +- Support HTTPS binary caches using mTLS (client certificate) authentication [#13002](https://github.com/NixOS/nix/issues/13002) [#13030](https://github.com/NixOS/nix/pull/13030) + + Added support for `tls-certificate` and `tls-private-key` options in substituter URLs. + + Example: + + ``` + https://substituter.invalid?tls-certificate=/path/to/cert.pem&tls-private-key=/path/to/key.pem + ``` + + When these options are configured, Nix will use this certificate/private key pair to authenticate to the server. + +- `nix store gc --dry-run` and `nix-collect-garbage --dry-run` now report the number of paths that would be freed [#15229](https://github.com/NixOS/nix/pull/15229) [#5704](https://github.com/NixOS/nix/issues/5704) + +## Performance improvements + +- Unpacking tarballs to `~/.cache/nix/tarball-cache-v2` is now multithreaded [#12087](https://github.com/NixOS/nix/pull/12087) + + Content-addressed cache for `builtins.fetchTarball` and tarball-based flake inputs (e.g. `github:NixOS/nixpkgs`, `https://channels.nixos.org/nixos-25.11/nixexprs.tar.xz`) now writes git blobs (files) to the `tarball-cache-v2` repository concurrently, which significantly reduces the wall time for tarball unpacking (up to ~1.8x faster unpacking for `https://channels.nixos.org/nixos-25.11/nixexprs.tar.xz` in our testing). + + Currently, Nix doesn't perform any maintenance on the `~/.cache/nix/tarball-cache-v2` repository, which will be addressed in future versions. Users that wish to reclaim disk space used by the tarball cache may want to run: + + ``` + rm -rf ~/.cache/nix/tarball-cache # Historical tarball-cache, not used by Nix >= 2.33 + cd ~/.cache/nix/tarball-cache-v2 && git multi-pack-index write && git multi-pack-index repack && git multi-pack-index expire + ``` + +- `nix nar ls` and other NAR listing operations have been optimised further [#15163](https://github.com/NixOS/nix/pull/15163) + +- Evaluator hot-path optimizations [#15270](https://github.com/NixOS/nix/pull/15270) [#15271](https://github.com/NixOS/nix/pull/15271) + +## C API Changes + +- New store API methods [#14766](https://github.com/NixOS/nix/pull/14766) [#14768](https://github.com/NixOS/nix/pull/14768) + + The C API now includes additional methods: + + - `nix_store_query_path_from_hash_part()` - Get the full store path given its hash part + - `nix_store_copy_path()` - Copy a single store path between two stores, allows repairs and configuring signature checking + +- Errors returned from your primops are not treated as recoverable by default [#13930](https://github.com/NixOS/nix/pull/13930) [#15286](https://github.com/NixOS/nix/pull/15286) + + Nix 2.34 by default remembers the error in the thunk that triggered it. + + Previously the following sequence of events worked: + + 1. Have a thunk that invokes a primop that's defined through the C API + 2. The primop returns an error + 3. Force the thunk again + 4. The primop returns a value + 5. The thunk evaluated successfully + + **Resolution** + + C API consumers that rely on this must change their recoverable error calls: + + ```diff + -nix_set_err_msg(context, NIX_ERR_*, msg); + +nix_set_err_msg(context, NIX_ERR_RECOVERABLE, msg); + ``` + +## Bug fixes + +- Avoid dropping ssh connections with `ssh-ng://` stores for store path copying [#14998](https://github.com/NixOS/nix/pull/14998) [#6950](https://github.com/NixOS/nix/issues/6950) + + Due to a bug in how Nix handled Boost.Coroutine2 suspension and resumption, copying from `ssh-ng://` stores would drop the SSH connection for each copied path. This issue has been fixed, which improves performance by avoiding multiple SSH/Nix Worker Protocol handshakes. + +- S3 binary caches now use virtual-hosted-style addressing by default [#15208](https://github.com/NixOS/nix/issues/15208) [#15216](https://github.com/NixOS/nix/pull/15216) + + S3 binary caches now use virtual-hosted-style URLs + (`https://bucket.s3.region.amazonaws.com/key`) instead of path-style URLs + (`https://s3.region.amazonaws.com/bucket/key`) when connecting to standard AWS + S3 endpoints. This enables HTTP/2 multiplexing and fixes TCP connection + exhaustion (TIME_WAIT socket accumulation) under high-concurrency workloads. + + A new `addressing-style` store option controls this behavior: + + - `auto` (default): virtual-hosted-style for standard AWS endpoints, path-style + for custom endpoints. + - `path`: forces path-style addressing (deprecated by AWS). + - `virtual`: forces virtual-hosted-style addressing (bucket names must not + contain dots). + + Bucket names containing dots (e.g., `my.bucket.name`) automatically fall back + to path-style addressing in `auto` mode, because dotted names create + multi-level subdomains that break TLS wildcard certificate validation. + + Example using path-style for backwards compatibility: + + ``` + s3://my-bucket/key?region=us-east-1&addressing-style=path + ``` + + Additionally, TCP keep-alive is now enabled on all HTTP connections, preventing + idle connections from being silently dropped by intermediate network devices + (NATs, firewalls, load balancers). + +- `nix-prefetch-url --unpack` now properly checks for empty archives [#15242](https://github.com/NixOS/nix/pull/15242) + + Prior versions failed to check for empty archives and would crash with a `nullptr` dereference when unpacking empty archives. + This is now fixed. + +- Prevent runaway processes when Nix is killed with `SIGKILL` when building in a local store with build users [#15193](https://github.com/NixOS/nix/pull/15193) + + When run as root, Nix doesn't run builds via the daemon and is a parent of the forked build processes. Prior versions of Nix failed to preserve the `PR_SET_PDEATHSIG` parent-death signal across `setuid` calls. This could lead to build processes being reparented and continue running in the background. This has been fixed. + +- Fix crash when interrupting `--log-format internal-json` [#15335](https://github.com/NixOS/nix/pull/15335) + + Pressing Ctrl-C during `--log-format internal-json` (used by [nix-output-monitor](https://github.com/maralorn/nix-output-monitor)) no longer causes a spurious "Nix crashed. This is a bug." report. + +- Fix percent-encoding in `file://` and `local://` store URIs [#15280](https://github.com/NixOS/nix/pull/15280) + + Store URIs with special characters like `+` in the path (e.g. `file:///tmp/a+b`) no longer incorrectly create percent-encoded directories (e.g. `/tmp/a%2Bb`). + +- Fix crash during tab completion in `nix repl` [#15255](https://github.com/NixOS/nix/pull/15255) + +- Fix "Too many open files" on macOS [#15205](https://github.com/NixOS/nix/pull/15205) + + Nix now raises the open file soft limit to the hard limit at startup, fixing "Too many open files" errors on macOS where the default soft limit is low. + +- `nix develop` no longer fails when `inputs.nixpkgs` has `flake = false` [#15175](https://github.com/NixOS/nix/pull/15175) + +- `builtins.flakeRefToString` no longer fails with "attribute is a thunk" [#15160](https://github.com/NixOS/nix/pull/15160) + +- Fix `QueryPathInfo` throwing on invalid paths in the daemon [#15134](https://github.com/NixOS/nix/pull/15134) + +- `nix-store --generate-binary-cache-key` now fsyncs key files to prevent corruption [#15107](https://github.com/NixOS/nix/pull/15107) + +- Fix `build-hook` setting in `nix.conf` being ignored [#15083](https://github.com/NixOS/nix/pull/15083) + +- Fix empty error messages when builds are cancelled due to a dependency failure [#14972](https://github.com/NixOS/nix/pull/14972) + + When a build fails without `--keep-going`, other in-progress builds are cancelled. Previously, these cancelled builds were incorrectly reported as failed with empty error messages. This affected `buildPathsWithResults` callers such as `nix flake check`. + +## Miscellaneous changes + +- Content-Encoding decompression is now handled by libcurl [#14324](https://github.com/NixOS/nix/issues/14324) [#15336](https://github.com/NixOS/nix/pull/15336) + + Transparent decompression of HTTP downloads specifying `Content-Encoding` header now uses libcurl. This adds support for previously advertised, but not supported `deflate` encoding as well as deprecated `x-gzip` alias. + Non-standard `xz`, `bzip2` encodings that were previously advertised are no longer supported, as they do not commonly appear in the wild and should not be sent by compliant servers. + + `br`, `zstd`, `gzip` continue to be supported. Distro packaging should ensure that the `libcurl` dependency is linked against required libraries to support these encodings. By default, the build system now requires libcurl >= 8.17.0, which is not known to have issues around [pausing and decompression](https://github.com/curl/curl/issues/16280). + +- Static builds now support S3 features (`libstore:s3-aws-auth` meson option) [#15076](https://github.com/NixOS/nix/pull/15076) + +- Improved package-related error messages [#15349](https://github.com/NixOS/nix/pull/15349) + + Store path context is now rendered in the user-facing `hash^out` format instead of the internal `!out!hash` format. + A misleading error message in `nix-env` that incorrectly blamed content-addressed derivations has been fixed. + +- Improved error message for empty derivation files [#15298](https://github.com/NixOS/nix/pull/15298) + + Parsing an empty `.drv` file (e.g. due to store corruption after an unclean shutdown) now produces a clear error message instead of the cryptic `expected string 'D'`. + +- Relative `file:` paths for tarballs are now rejected with a clear error [#14983](https://github.com/NixOS/nix/pull/14983) + +- Continued progress on the Windows port, including build fixes, CI improvements, and platform abstractions. + +- Nix docker images are now uploaded to [GHCR](https://github.com/NixOS/nix/pkgs/container/nix) as part of the release process + + Historically, only pre-release builds of `amd64` docker images have been uploaded to ghcr.io with the `latest` tag pointing to the last built image from `master` branch. This has been fixed and going forward, will include the same images as that are built by [Hydra](https://hydra.nixos.org/project/nix) for [arm64](https://hydra.nixos.org/job/nix/maintenance-2.34/dockerImage.aarch64-linux) and [amd64](https://hydra.nixos.org/job/nix/maintenance-2.34/dockerImage.x86_64-linux). Pre-release versions are no longer pushed to the registry. + +## Contributors + +This release was made possible by the following 43 contributors: + +- Taeer Bar-Yam [**(@Radvendii)**](https://github.com/Radvendii) +- Sergei Zimmerman [**(@xokdvium)**](https://github.com/xokdvium) +- Jörg Thalheim [**(@Mic92)**](https://github.com/Mic92) +- Graham Dennis [**(@GrahamDennis)**](https://github.com/GrahamDennis) +- Damien Diederen [**(@ztzg)**](https://github.com/ztzg) +- koberbe-jh [**(@koberbe-jh)**](https://github.com/koberbe-jh) +- Robert Hensing [**(@roberth)**](https://github.com/roberth) +- Bouke van der Bijl [**(@bouk)**](https://github.com/bouk) +- Lisanna Dettwyler [**(@lisanna-dettwyler)**](https://github.com/lisanna-dettwyler) +- kiara [**(@KiaraGrouwstra)**](https://github.com/KiaraGrouwstra) +- Side Effect [**(@YawKar)**](https://github.com/YawKar) +- dram [**(@dramforever)**](https://github.com/dramforever) +- tomf [**(@tomfitzhenry)**](https://github.com/tomfitzhenry) +- Kamil Monicz [**(@Zaczero)**](https://github.com/Zaczero) +- Cosima Neidahl [**(@OPNA2608)**](https://github.com/OPNA2608) +- Siddhant Kumar [**(@siddhantk232)**](https://github.com/siddhantk232) +- Jens Petersen [**(@juhp)**](https://github.com/juhp) +- Johannes Kirschbauer [**(@hsjobeki)**](https://github.com/hsjobeki) +- tomberek [**(@tomberek)**](https://github.com/tomberek) +- Eelco Dolstra [**(@edolstra)**](https://github.com/edolstra) +- Artemis Tosini [**(@artemist)**](https://github.com/artemist) +- David McFarland [**(@corngood)**](https://github.com/corngood) +- Tucker Shea [**(@NoRePercussions)**](https://github.com/NoRePercussions) +- Connor Baker [**(@ConnorBaker)**](https://github.com/ConnorBaker) +- Cole Helbling [**(@cole-h)**](https://github.com/cole-h) +- Eveeifyeve [**(@Eveeifyeve)**](https://github.com/Eveeifyeve) +- John Ericson [**(@Ericson2314)**](https://github.com/Ericson2314) +- Graham Christensen [**(@grahamc)**](https://github.com/grahamc) +- Ilja [**(@iljah)**](https://github.com/iljah) +- Pol Dellaiera [**(@drupol)**](https://github.com/drupol) +- steelman [**(@steelman)**](https://github.com/steelman) +- Brian McKenna [**(@puffnfresh)**](https://github.com/puffnfresh) +- JustAGuyTryingHisBest [**(@JustAGuyTryingHisBest)**](https://github.com/JustAGuyTryingHisBest) +- zowoq [**(@zowoq)**](https://github.com/zowoq) +- Agustín Covarrubias [**(@agucova)**](https://github.com/agucova) +- Sergei Trofimovich [**(@trofi)**](https://github.com/trofi) +- Bernardo Meurer [**(@lovesegfault)**](https://github.com/lovesegfault) +- Peter Bynum [**(@pkpbynum)**](https://github.com/pkpbynum) +- Amaan Qureshi [**(@amaanq)**](https://github.com/amaanq) +- Michael Hoang [**(@Enzime)**](https://github.com/Enzime) +- Michael Daniels [**(@mdaniels5757)**](https://github.com/mdaniels5757) +- Matthew Kenigsberg [**(@mkenigs)**](https://github.com/mkenigs) +- Shea Levy [**(@shlevy)**](https://github.com/shlevy) + +# Release 2.34.1 (2026-03-08) + +## Changes + +- C API: Fix `EvalState` pointer passed to primop callbacks [#15300](https://github.com/NixOS/nix/pull/15300) [#15383](https://github.com/NixOS/nix/pull/15383) + + The `EvalState *` passed to C API primop callbacks was incorrectly pointing to + the internal `nix::EvalState` rather than the C API wrapper struct. This caused + a segfault when the callback used the pointer with C API functions such as + `nix_alloc_value()`. The same issue affected `printValueAsJSON` and + `printValueAsXML` callbacks on external values. + +- Fix daemon not applying `FileTransferSettings` from `trusted-users` [#15408](https://github.com/NixOS/nix/pull/15408) + + Previously `nix-daemon` failed to apply settings for `libcurl` configuration configured by client connections from [`trusted-users`](@docroot@/command-ref/conf-file.md#conf-trusted-users). This was a pre-existing bug, which has been exacerbated by 2.34.0 moving more settings from the global `settings` into libcurl-specific `fileTransferSettings` (e.g. `netrc-file`, `http-connections` or `ssl-cert-file`). Note that the use of `trusted-users` is heavily discouraged unless you are fine with: + + > Adding a user to `trusted-users` is essentially equivalent to giving that user root access to the system. + > For example, the user can access or replace store path contents that are critical for system security. + +- Improve formatting of error messages and warnings [#15397](https://github.com/NixOS/nix/pull/15397) + +# Release 2.34.2 (2026-03-20) + +## Changes + +- Fixed `nix upgrade-nix` without an explicitly specified `--profile` argument [#15437](https://github.com/NixOS/nix/issues/15437) [#15438](https://github.com/NixOS/nix/pull/15438) + +- Erroneous `error (ignored): write of ... bytes: Bad file descriptor` warnings on interrupted store copy operations are now fixed [#15486](https://github.com/NixOS/nix/pull/15486) + +- Reverted changes enabling keep-alive in the HTTP client [#15522](https://github.com/NixOS/nix/pull/15522) + + Connection reuse for S3 stores has caused Hydra upload errors due to stale connections being closed by the remote servers. Nix currently lacks retry mechanisms for 400 `RequestTimeout` errors used by AWS S3. + +- Fixed evaluation that accesses the logical store directory using `ssh-ng://` `eval-store`s [#15417](https://github.com/NixOS/nix/pull/15417) + + Accessing the logical store directory (typically `/nix/store`) during evaluation would fail previously: + + ``` + nix build nixpkgs#hello --store ssh-ng://host + error: path '/nix/store/' is not in the Nix store + ``` + +- S3: restore STS WebIdentity and ECS container credential providers [#15507](https://github.com/NixOS/nix/pull/15507) + + Nix 2.33 replaced the S3 backend's `aws-sdk-cpp` credential chain with a + custom chain built on `aws-c-auth`. That chain omitted two providers, + breaking S3 binary cache access in container workloads: + + - **STS WebIdentity** (`AWS_WEB_IDENTITY_TOKEN_FILE`, `AWS_ROLE_ARN`, + `AWS_ROLE_SESSION_NAME`) — used by EKS IRSA, GitHub Actions OIDC, and + any `sts:AssumeRoleWithWebIdentity` federation. + - **ECS container metadata** (`AWS_CONTAINER_CREDENTIALS_RELATIVE_URI`, + `AWS_CONTAINER_CREDENTIALS_FULL_URI`) — used by ECS tasks and EKS Pod + Identity. + + The typical symptom was a misleading IMDS error + (`Valid credentials could not be sourced by the IMDS provider`), because + IMDS is the last provider tried after the correct one was skipped. + + Both providers are now part of the chain, ordered to match the + pre-2.33 `DefaultAWSCredentialsProviderChain`: + `Environment → SSO → Profile → STS WebIdentity → (ECS | IMDS)`. + As in both the old and new AWS SDK default chains, ECS and IMDS are + mutually exclusive: when container credential environment variables are + set, IMDS is skipped. + diff --git a/doc/manual/source/store/store-path.md b/doc/manual/source/store/store-path.md index 08b024e4a846..04bdfec004c2 100644 --- a/doc/manual/source/store/store-path.md +++ b/doc/manual/source/store/store-path.md @@ -31,7 +31,7 @@ A store path is rendered to a file system path as the concatenation of - [Store directory](#store-directory) (typically `/nix/store`) - Path separator (`/`) -- Digest rendered in a custom variant of [Base32](https://en.wikipedia.org/wiki/Base32) (20 arbitrary bytes become 32 ASCII characters) +- Digest rendered in [Nix32](@docroot@/protocols/nix32.md), a variant of base-32 (20 hash bytes become 32 ASCII characters) - Hyphen (`-`) - Name diff --git a/docker.nix b/docker.nix index 72c13663488d..2bd6751762fa 100644 --- a/docker.nix +++ b/docker.nix @@ -362,7 +362,6 @@ dockerTools.buildLayeredImageWithNixDb { extraCommands = '' rm -rf nix-support - ln -s /nix/var/nix/profiles nix/var/nix/gcroots/profiles ''; fakeRootCommands = '' chmod 1777 tmp diff --git a/flake.lock b/flake.lock index f56706ec761e..d44ac1734fb9 100644 --- a/flake.lock +++ b/flake.lock @@ -58,16 +58,16 @@ }, "nixpkgs": { "locked": { - "lastModified": 1761597516, - "narHash": "sha256-wxX7u6D2rpkJLWkZ2E932SIvDJW8+ON/0Yy8+a5vsDU=", - "rev": "daf6dc47aa4b44791372d6139ab7b25269184d55", - "revCount": 811874, + "lastModified": 1773222311, + "narHash": "sha256-BHoB/XpbqoZkVYZCfXJXfkR+GXFqwb/4zbWnOr2cRcU=", + "rev": "0590cd39f728e129122770c029970378a79d076a", + "revCount": 909248, "type": "tarball", - "url": "https://api.flakehub.com/f/pinned/NixOS/nixpkgs/0.2505.811874%2Brev-daf6dc47aa4b44791372d6139ab7b25269184d55/019a3494-3498-707e-9086-1fb81badc7fe/source.tar.gz" + "url": "https://api.flakehub.com/f/pinned/NixOS/nixpkgs/0.2511.909248%2Brev-0590cd39f728e129122770c029970378a79d076a/019ce32b-8ace-7339-b129-cceaa8dd10c6/source.tar.gz" }, "original": { "type": "tarball", - "url": "https://flakehub.com/f/NixOS/nixpkgs/0.2505" + "url": "https://flakehub.com/f/NixOS/nixpkgs/0.2511" } }, "nixpkgs-23-11": { diff --git a/flake.nix b/flake.nix index b32a95b06a23..55ceca187523 100644 --- a/flake.nix +++ b/flake.nix @@ -1,7 +1,7 @@ { description = "The purely functional package manager"; - inputs.nixpkgs.url = "https://flakehub.com/f/NixOS/nixpkgs/0.2505"; + inputs.nixpkgs.url = "https://flakehub.com/f/NixOS/nixpkgs/0.2511"; inputs.nixpkgs-regression.url = "github:NixOS/nixpkgs/215d4d0fd80ca5163643b03a33fde804a29cc1e2"; inputs.nixpkgs-23-11.url = "github:NixOS/nixpkgs/a62e6edd6d5e1fa0329b8653c801147986f8d446"; @@ -107,6 +107,9 @@ } // lib.optionalAttrs (crossSystem == "x86_64-unknown-freebsd13") { useLLVM = true; + } + // lib.optionalAttrs (crossSystem == "x86_64-w64-mingw32") { + emulator = pkgs: "${pkgs.buildPackages.wineWow64Packages.stable_11}/bin/wine"; }; overlays = [ (overlayFor (pkgs: pkgs.${stdenv})) @@ -432,6 +435,10 @@ "nix-cmd" = { }; + "nix-nswrapper" = { + linuxOnly = true; + }; + "nix-cli" = { }; "nix-everything" = { }; @@ -444,10 +451,6 @@ supportsCross = false; }; - "nix-kaitai-struct-checks" = { - supportsCross = false; - }; - "nix-perl-bindings" = { supportsCross = false; }; @@ -456,36 +459,35 @@ pkgName: { supportsCross ? true, + linuxOnly ? false, }: - { - # These attributes go right into `packages.`. - "${pkgName}" = nixpkgsFor.${system}.native.nixComponents2.${pkgName}; - } - // lib.optionalAttrs supportsCross ( - flatMapAttrs (lib.genAttrs crossSystems (_: { })) ( - crossSystem: + lib.optionalAttrs (linuxOnly -> nixpkgsFor.${system}.native.stdenv.hostPlatform.isLinux) ( + { + # These attributes go right into `packages.`. + "${pkgName}" = nixpkgsFor.${system}.native.nixComponents2.${pkgName}; + "${pkgName}-static" = nixpkgsFor.${system}.native.pkgsStatic.nixComponents2.${pkgName}; + } + // flatMapAttrs (lib.genAttrs stdenvs (_: { })) ( + stdenvName: { }: { # These attributes go right into `packages.`. - "${pkgName}-${crossSystem}" = nixpkgsFor.${system}.cross.${crossSystem}.nixComponents2.${pkgName}; + "${pkgName}-${stdenvName}" = + nixpkgsFor.${system}.nativeForStdenv.${stdenvName}.nixComponents2.${pkgName}; } ) - // { - "${pkgName}-static" = nixpkgsFor.${system}.native.pkgsStatic.nixComponents2.${pkgName}; - } ) - // flatMapAttrs (lib.genAttrs stdenvs (_: { })) ( - stdenvName: - { }: - { - # These attributes go right into `packages.`. - "${pkgName}-${stdenvName}" = - nixpkgsFor.${system}.nativeForStdenv.${stdenvName}.nixComponents2.${pkgName}; - } - // lib.optionalAttrs supportsCross { - "${pkgName}-${stdenvName}-static" = - nixpkgsFor.${system}.nativeForStdenv.${stdenvName}.pkgsStatic.nixComponents2.${pkgName}; - } + // lib.optionalAttrs supportsCross ( + flatMapAttrs (lib.genAttrs crossSystems (_: { })) ( + crossSystem: + { }: + lib.optionalAttrs + (linuxOnly -> nixpkgsFor.${system}.cross.${crossSystem}.stdenv.hostPlatform.isLinux) + { + # These attributes go right into `packages.`. + "${pkgName}-${crossSystem}" = nixpkgsFor.${system}.cross.${crossSystem}.nixComponents2.${pkgName}; + } + ) ) ) // lib.optionalAttrs (builtins.elem system linux64BitSystems) { diff --git a/maintainers/flake-module.nix b/maintainers/flake-module.nix index 7f7447b19e49..03d532f5c064 100644 --- a/maintainers/flake-module.nix +++ b/maintainers/flake-module.nix @@ -88,16 +88,28 @@ ''^tests/functional/lang/eval-fail-path-slash\.nix$'' ''^tests/functional/lang/eval-fail-toJSON-non-utf-8\.nix$'' ''^tests/functional/lang/eval-fail-set\.nix$'' + + # Language tests, don't churn the formatting of strings + ''^tests/functional/lang/eval-fail-fromTOML-overflow\.nix$'' + ''^tests/functional/lang/eval-fail-fromTOML-underflow\.nix$'' + ''^tests/functional/lang/eval-fail-bad-string-interpolation-3\.nix$'' + ''^tests/functional/lang/eval-fail-bad-string-interpolation-4\.nix$'' + ''^tests/functional/lang/eval-okay-regex-match2\.nix$'' + + # URL literal tests - nixfmt converts unquoted URLs to strings + ''^tests/functional/lang/eval-fail-url-literal\.nix$'' + ''^tests/functional/lang/eval-okay-url-literal-warn\.nix$'' + ''^tests/functional/lang/eval-okay-url-literal-default\.nix$'' ]; }; clang-format = { enable = true; # https://github.com/cachix/git-hooks.nix/pull/532 - package = pkgs.llvmPackages_latest.clang-tools; + package = pkgs.llvmPackages_21.clang-tools; excludes = [ # We don't want to format test data # ''tests/(?!nixos/).*\.nix'' - ''^src/[^/]*-tests/data/.*$'' + "^src/[^/]*-tests/data/.*$" # Don't format vendored code ''^doc/manual/redirects\.js$'' diff --git a/meson.build b/meson.build index c072a4821630..3e2d29cd5495 100644 --- a/meson.build +++ b/meson.build @@ -24,6 +24,10 @@ subproject('libcmd') # Executables subproject('nix') +if host_machine.system() == 'linux' + subproject('nswrapper') +endif + # Docs if get_option('doc-gen') subproject('internal-api-docs') @@ -63,6 +67,3 @@ subproject('nix-functional-tests') if get_option('json-schema-checks') subproject('json-schema-checks') endif -if get_option('kaitai-struct-checks') - subproject('kaitai-struct-checks') -endif diff --git a/meson.options b/meson.options index 2739b0c71638..a306a84252ea 100644 --- a/meson.options +++ b/meson.options @@ -28,13 +28,6 @@ option( description : 'Build benchmarks (requires gbenchmark)', ) -option( - 'kaitai-struct-checks', - type : 'boolean', - value : true, - description : 'Check the Kaitai Struct specifications (requires Kaitai Struct)', -) - option( 'json-schema-checks', type : 'boolean', diff --git a/nix-meson-build-support/common/asan-options/meson.build b/nix-meson-build-support/common/asan-options/meson.build index 80527b5a9884..56e6a6a56a7f 100644 --- a/nix-meson-build-support/common/asan-options/meson.build +++ b/nix-meson-build-support/common/asan-options/meson.build @@ -1,7 +1,7 @@ # Clang gets grumpy about missing libasan symbols if -shared-libasan is not # passed when building shared libs, at least on Linux if cxx.get_id() == 'clang' and ('address' in get_option('b_sanitize') or 'undefined' in get_option( - 'b_sanitize', + 'b_sanitize', )) add_project_link_arguments('-shared-libasan', language : 'cpp') endif diff --git a/nix-meson-build-support/common/meson.build b/nix-meson-build-support/common/meson.build index b192c0d03521..b58e5eb1b438 100644 --- a/nix-meson-build-support/common/meson.build +++ b/nix-meson-build-support/common/meson.build @@ -22,6 +22,8 @@ add_project_arguments( '-Werror=undef', '-Werror=unused-result', '-Werror=sign-compare', + '-Werror=return-type', + '-Werror=non-virtual-dtor', '-Wignored-qualifiers', '-Wimplicit-fallthrough', '-Wno-deprecated-declarations', @@ -31,6 +33,13 @@ add_project_arguments( # GCC doesn't benefit much from precompiled headers. do_pch = cxx.get_id() == 'clang' +if cxx.get_id() == 'gcc' + add_project_arguments( + '-Wno-interference-size', # Used for C++ ABI only. We don't provide any guarantees about different march tunings. + language : 'cpp', + ) +endif + # This is a clang-only option for improving build times. # It forces the instantiation of templates in the PCH itself and # not every translation unit it's included in. @@ -40,6 +49,11 @@ do_pch = cxx.get_id() == 'clang' # instantiations in libutil and libstore. if cxx.get_id() == 'clang' add_project_arguments('-fpch-instantiate-templates', language : 'cpp') + # Catch brace elision bugs: when WorkerProto::Version changed from `unsigned int` + # to `struct { unsigned int major; uint8_t minor; }`, `.version = 16` silently + # became `.version = {16, 0}` instead of failing, breaking protocol compatibility + # in a subtle way + add_project_arguments('-Werror=c99-designator', language : 'cpp') endif # Detect if we're using libstdc++ (GCC's standard library) diff --git a/packaging/binary-tarball.nix b/packaging/binary-tarball.nix index 86aae0ac524c..a3c8c53f988c 100644 --- a/packaging/binary-tarball.nix +++ b/packaging/binary-tarball.nix @@ -1,16 +1,20 @@ { runCommand, - system, + stdenv, buildPackages, cacert, nix, + nixComponents2, }: let + inherit (stdenv.hostPlatform) system; + installerClosureInfo = buildPackages.closureInfo { rootPaths = [ nix + nixComponents2.nix-manual.man cacert ]; }; @@ -42,6 +46,7 @@ runCommand "nix-binary-tarball-${version}" env '' --subst-var-by cacert ${cacert} substitute ${../scripts/install-multi-user.sh} $TMPDIR/install-multi-user \ --subst-var-by nix ${nix} \ + --subst-var-by nix-manual ${nixComponents2.nix-manual.man} \ --subst-var-by cacert ${cacert} if type -p shellcheck; then diff --git a/packaging/components.nix b/packaging/components.nix index 6402e8b7b2f8..962f0ba05b2e 100644 --- a/packaging/components.nix +++ b/packaging/components.nix @@ -124,7 +124,7 @@ let + lib.optionalString ( - !stdenv.hostPlatform.isWindows + !(stdenv.hostPlatform.isWindows || stdenv.hostPlatform.isCygwin) # build failure && !stdenv.hostPlatform.isStatic # LTO breaks exception handling on x86-64-darwin. @@ -146,12 +146,14 @@ let ]; }; - mesonBuildLayer = finalAttrs: prevAttrs: { + mesonBuildLayer = finalAttrs: prevAttrs: rec { nativeBuildInputs = prevAttrs.nativeBuildInputs or [ ] ++ [ pkg-config ]; separateDebugInfo = !stdenv.hostPlatform.isStatic; - hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; + # needed by separateDebugInfo + # SEE: https://github.com/NixOS/nixpkgs/pull/394674/commits/a4d355342976e9e9823fb94f133bc43ebec9da5b + __structuredAttrs = separateDebugInfo; }; mesonLibraryLayer = finalAttrs: prevAttrs: { @@ -407,6 +409,8 @@ in nix-cmd = callPackage ../src/libcmd/package.nix { }; + nix-nswrapper = callPackage ../src/nswrapper/package.nix { }; + /** The Nix command line interface. Note that this does not include its tests, whereas `nix-everything` does. */ @@ -443,11 +447,6 @@ in */ nix-json-schema-checks = callPackage ../src/json-schema-checks/package.nix { }; - /** - Kaitai struct schema validation checks - */ - nix-kaitai-struct-checks = callPackage ../src/kaitai-struct-checks/package.nix { }; - nix-perl-bindings = callPackage ../src/perl/package.nix { }; /** diff --git a/packaging/dependencies.nix b/packaging/dependencies.nix index f5e39e6e005f..35f05f7db586 100644 --- a/packaging/dependencies.nix +++ b/packaging/dependencies.nix @@ -16,37 +16,6 @@ in scope: { inherit stdenv; - libblake3 = - (pkgs.libblake3.override { - inherit stdenv; - # Nixpkgs disables tbb on static - useTBB = !stdenv.hostPlatform.isStatic; - }) - # For some reason that is not clear, it is wanting to use libgcc_eh which is not available. - # Force this to be built with compiler-rt & libunwind over libgcc_eh works. - # Issue: https://github.com/NixOS/nixpkgs/issues/177129 - .overrideAttrs - ( - attrs: - lib.optionalAttrs - ( - stdenv.cc.isClang - && stdenv.hostPlatform.isStatic - && stdenv.cc.libcxx != null - && stdenv.cc.libcxx.isLLVM - ) - { - NIX_CFLAGS_COMPILE = [ - "-rtlib=compiler-rt" - "-unwindlib=libunwind" - ]; - - buildInputs = [ - pkgs.llvmPackages.libunwind - ]; - } - ); - boehmgc = (pkgs.boehmgc.override { enableLargeConfig = true; @@ -98,28 +67,60 @@ scope: { url = "https://kristaps.bsd.lv/lowdown/snapshots/lowdown-${version}.tar.gz"; hash = "sha512-cfzhuF4EnGmLJf5EGSIbWqJItY3npbRSALm+GarZ7SMU7Hr1xw0gtBFMpOdi5PBar4TgtvbnG4oRPh+COINGlA=="; }; - patches = [ ]; nativeBuildInputs = prevAttrs.nativeBuildInputs ++ [ pkgs.buildPackages.bmake ]; postInstall = lib.replaceStrings [ "lowdown.so.1" "lowdown.1.dylib" ] [ "lowdown.so.2" "lowdown.2.dylib" ] (prevAttrs.postInstall or ""); }); - # TODO: Remove this when https://github.com/NixOS/nixpkgs/pull/442682 is included in a stable release - toml11 = - if lib.versionAtLeast pkgs.toml11.version "4.4.0" then - pkgs.toml11 - else - pkgs.toml11.overrideAttrs rec { - version = "4.4.0"; - src = pkgs.fetchFromGitHub { - owner = "ToruNiina"; - repo = "toml11"; - tag = "v${version}"; - hash = "sha256-sgWKYxNT22nw376ttGsTdg0AMzOwp8QH3E8mx0BZJTQ="; - }; + curl = + (pkgs.curl.override { + http3Support = !pkgs.stdenv.hostPlatform.isWindows; + # Make sure we enable all the dependencies for Content-Encoding/Transfer-Encoding decompression. + zstdSupport = true; + brotliSupport = true; + zlibSupport = true; + # libpsl uses a data file needed at runtime, not useful for nix. + pslSupport = !stdenv.hostPlatform.isStatic; + idnSupport = !stdenv.hostPlatform.isStatic; + }).overrideAttrs + { + # TODO: Fix in nixpkgs. Static build with brotli is marked as broken, but it's not the case. + # Remove once https://github.com/NixOS/nixpkgs/pull/494111 lands in the 25.11 channel. + meta.broken = false; }; + libblake3 = + (pkgs.libblake3.override { + inherit stdenv; + # Nixpkgs disables tbb on static + useTBB = !(stdenv.hostPlatform.isWindows || stdenv.hostPlatform.isStatic); + }) + # For some reason that is not clear, it is wanting to use libgcc_eh which is not available. + # Force this to be built with compiler-rt & libunwind over libgcc_eh works. + # Issue: https://github.com/NixOS/nixpkgs/issues/177129 + .overrideAttrs + ( + attrs: + lib.optionalAttrs + ( + stdenv.cc.isClang + && stdenv.hostPlatform.isStatic + && stdenv.cc.libcxx != null + && stdenv.cc.libcxx.isLLVM + ) + { + NIX_CFLAGS_COMPILE = [ + "-rtlib=compiler-rt" + "-unwindlib=libunwind" + ]; + + buildInputs = [ + pkgs.llvmPackages.libunwind + ]; + } + ); + # TODO Hack until https://github.com/NixOS/nixpkgs/issues/45462 is fixed. boost = (pkgs.boost.override { @@ -141,10 +142,4 @@ scope: { }); wasmtime = pkgs.callPackage ./wasmtime.nix { }; - - curl = pkgs.curl.override { - # libpsl uses a data file needed at runtime, not useful for nix. - pslSupport = !stdenv.hostPlatform.isStatic; - idnSupport = !stdenv.hostPlatform.isStatic; - }; } diff --git a/packaging/dev-shell.nix b/packaging/dev-shell.nix index 8f963f961fb1..d34f2083a962 100644 --- a/packaging/dev-shell.nix +++ b/packaging/dev-shell.nix @@ -131,7 +131,7 @@ pkgs.nixComponents2.nix-util.overrideAttrs ( ignoreCrossFile = flags: builtins.filter (flag: !(lib.strings.hasInfix "cross-file" flag)) flags; availableComponents = lib.filterAttrs ( - k: v: lib.meta.availableOn pkgs.hostPlatform v + k: v: lib.meta.availableOn pkgs.stdenv.hostPlatform v ) allComponents; activeComponents = buildInputsClosureCond isInternal ( @@ -142,7 +142,9 @@ pkgs.nixComponents2.nix-util.overrideAttrs ( internalDrvs = byDrvPath ( # Drop the attr names (not present in buildInputs anyway) lib.attrValues availableComponents - ++ lib.concatMap (c: lib.attrValues c.tests or { }) (lib.attrValues availableComponents) + ++ lib.concatMap (c: lib.filter (v: !v.meta.broken) (lib.attrValues (c.tests or { }))) ( + lib.attrValues availableComponents + ) ); isInternal = @@ -280,7 +282,6 @@ pkgs.nixComponents2.nix-util.overrideAttrs ( dontUseCmakeConfigure = true; mesonFlags = [ - (lib.mesonBool "kaitai-struct-checks" (isActiveComponent "nix-kaitai-struct-checks")) (lib.mesonBool "json-schema-checks" (isActiveComponent "nix-json-schema-checks")) ] ++ map (transformFlag "libutil") (ignoreCrossFile pkgs.nixComponents2.nix-util.mesonFlags) @@ -299,7 +300,7 @@ pkgs.nixComponents2.nix-util.overrideAttrs ( lib.filter (x: !isInternal x) ( lib.lists.concatMap ( # Nix manual has a build-time dependency on nix, but we - # don't want to do a native build just to enter the ross + # don't want to do a native build just to enter the cross # dev shell. # # TODO: think of a more principled fix for this. @@ -322,7 +323,7 @@ pkgs.nixComponents2.nix-util.overrideAttrs ( pkgs.buildPackages.shellcheck pkgs.buildPackages.include-what-you-use ] - ++ lib.optional pkgs.hostPlatform.isUnix pkgs.buildPackages.gdb + ++ lib.optional stdenv.hostPlatform.isUnix pkgs.buildPackages.gdb ++ lib.optional (stdenv.cc.isClang && stdenv.hostPlatform == stdenv.buildPlatform) ( lib.hiPrio pkgs.buildPackages.clang-tools ) @@ -340,7 +341,7 @@ pkgs.nixComponents2.nix-util.overrideAttrs ( buildInputs = # TODO change Nixpkgs to mark gbenchmark as building on Windows - lib.optional pkgs.hostPlatform.isUnix pkgs.gbenchmark + lib.optional stdenv.hostPlatform.isUnix pkgs.gbenchmark ++ dedupByString (v: "${v}") ( lib.filter (x: !isInternal x) (lib.lists.concatMap (c: c.buildInputs) activeComponents) ) diff --git a/packaging/everything.nix b/packaging/everything.nix index 3206b8ba4235..de68396d6965 100644 --- a/packaging/everything.nix +++ b/packaging/everything.nix @@ -31,6 +31,8 @@ nix-cmd, + nix-nswrapper, + nix-cli, nix-functional-tests, @@ -171,6 +173,9 @@ stdenv.mkDerivation (finalAttrs: { # Forwarded outputs ln -sT ${nix-manual} $doc ln -sT ${nix-manual.man} $man + '' + + lib.optionalString stdenv.isLinux '' + lndir ${nix-nswrapper} $out ''; passthru = { diff --git a/packaging/hydra.nix b/packaging/hydra.nix index 9839dd621639..87d31a694b89 100644 --- a/packaging/hydra.nix +++ b/packaging/hydra.nix @@ -57,6 +57,7 @@ let "nix-flake" "nix-flake-c" "nix-flake-tests" + "nix-nswrapper" "nix-main" "nix-main-c" "nix-cmd" @@ -72,7 +73,6 @@ let "nix-manual-manpages-only" "nix-internal-api-docs" "nix-external-api-docs" - "nix-kaitai-struct-checks" ] ); in @@ -115,7 +115,11 @@ rec { # Binary package for various platforms. build = forAllPackages ( - pkgName: forAllSystems (system: nixpkgsFor.${system}.native.nixComponents2.${pkgName}) + pkgName: + lib.filterAttrs ( + system: _do_not_touch: + pkgName == "nix-nswrapper" -> nixpkgsFor.${system}.native.stdenv.hostPlatform.isLinux + ) (forAllSystems (system: nixpkgsFor.${system}.native.nixComponents2.${pkgName})) ); shellInputs = removeAttrs (forAllSystems ( diff --git a/packaging/wasmtime.nix b/packaging/wasmtime.nix index 3e8b71280c07..d2c2b95f6072 100644 --- a/packaging/wasmtime.nix +++ b/packaging/wasmtime.nix @@ -3,13 +3,13 @@ { lib, stdenv, - rust_1_89, + rust, fetchFromGitHub, cmake, enableShared ? !stdenv.hostPlatform.isStatic, enableStatic ? stdenv.hostPlatform.isStatic, }: -rust_1_89.packages.stable.rustPlatform.buildRustPackage (finalAttrs: { +rust.packages.stable.rustPlatform.buildRustPackage (finalAttrs: { pname = "wasmtime"; version = "40.0.2"; diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index 683beca10fde..d4ea88b5ea6c 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -52,6 +52,7 @@ readonly PROFILE_FISH_PREFIXES=( readonly PROFILE_NIX_FILE_FISH="$NIX_ROOT/var/nix/profiles/default/etc/profile.d/nix-daemon.fish" readonly NIX_INSTALLED_NIX="@nix@" +readonly NIX_INSTALLED_NIX_MAN="@nix-manual@" readonly NIX_INSTALLED_CACERT="@cacert@" #readonly NIX_INSTALLED_NIX="/nix/store/byi37zv50wnfrpp4d81z3spswd5zva37-nix-2.3.6" #readonly NIX_INSTALLED_CACERT="/nix/store/7pi45g541xa8ahwgpbpy7ggsl0xj1jj6-nss-cacert-3.49.2" @@ -969,6 +970,8 @@ setup_default_profile() { task "Setting up the default profile" _sudo "to install a bootstrapping Nix in to the default profile" \ HOME="$ROOT_HOME" "$NIX_INSTALLED_NIX/bin/nix-env" -i "$NIX_INSTALLED_NIX" + _sudo "to install Nix man pages in to the default profile" \ + HOME="$ROOT_HOME" "$NIX_INSTALLED_NIX/bin/nix-env" -i "$NIX_INSTALLED_NIX_MAN" if [ -z "${NIX_SSL_CERT_FILE:-}" ] || ! [ -f "${NIX_SSL_CERT_FILE:-}" ] || cert_in_store; then _sudo "to install a bootstrapping SSL certificate just for Nix in to the default profile" \ diff --git a/scripts/install-systemd-multi-user.sh b/scripts/install-systemd-multi-user.sh index 8abbb7af4ad2..a20a57b907c0 100755 --- a/scripts/install-systemd-multi-user.sh +++ b/scripts/install-systemd-multi-user.sh @@ -38,6 +38,7 @@ escape_systemd_env() { create_systemd_proxy_env() { vars="http_proxy https_proxy ftp_proxy all_proxy no_proxy HTTP_PROXY HTTPS_PROXY FTP_PROXY ALL_PROXY NO_PROXY" for v in $vars; do + # shellcheck disable=SC2268 if [ "x${!v:-}" != "x" ]; then echo "Environment=${v}=$(escape_systemd_env "${!v}")" fi diff --git a/src/json-schema-checks/meson.build b/src/json-schema-checks/meson.build index 20acfe411026..a1b525bc51e4 100644 --- a/src/json-schema-checks/meson.build +++ b/src/json-schema-checks/meson.build @@ -62,9 +62,11 @@ schemas = [ }, { 'stem' : 'build-trace-entry', - 'schema' : schema_dir / 'build-trace-entry-v1.yaml', + 'schema' : schema_dir / 'build-trace-entry-v2.yaml', 'files' : [ 'simple.json', + # The field is no longer supported, but we want to show that we + # ignore it during parsing. 'with-dependent-realisations.json', 'with-signature.json', ], diff --git a/src/kaitai-struct-checks/meson.build b/src/kaitai-struct-checks/meson.build deleted file mode 100644 index f705a6744c02..000000000000 --- a/src/kaitai-struct-checks/meson.build +++ /dev/null @@ -1,77 +0,0 @@ -# Run with: -# meson test --suite kaitai-struct -# Run with: (without shell / configure) -# nix build .#nix-kaitai-struct-checks - -project( - 'nix-kaitai-struct-checks', - 'cpp', - version : files('.version'), - default_options : [ - 'cpp_std=c++23', - # TODO(Qyriad): increase the warning level - 'warning_level=1', - 'errorlogs=true', # Please print logs for tests that fail - ], - meson_version : '>= 1.1', - license : 'LGPL-2.1-or-later', -) - -kaitai_runtime_dep = dependency('kaitai-struct-cpp-stl-runtime', required : true) -gtest_dep = dependency('gtest') -gtest_main_dep = dependency('gtest_main', required : true) - -# Find the Kaitai Struct compiler -ksc = find_program('ksc', required : true) - -kaitai_generated_srcs = custom_target( - 'kaitai-generated-sources', - input : [ 'nar.ksy' ], - output : [ 'nix_nar.cpp', 'nix_nar.h' ], - command : [ - ksc, - '@INPUT@', - '--target', 'cpp_stl', - '--outdir', - meson.current_build_dir(), - ], -) - -nar_kaitai_lib = library( - 'nix-nar-kaitai-lib', - kaitai_generated_srcs, - dependencies : [ kaitai_runtime_dep ], - install : true, -) - -nar_kaitai_dep = declare_dependency( - link_with : nar_kaitai_lib, - sources : kaitai_generated_srcs[1], -) - -# The nar directory is a committed symlink to the actual nars location -nars_dir = meson.current_source_dir() / 'nars' - -# Get all example files -nars = [ - 'dot.nar', -] - -test_deps = [ - nar_kaitai_dep, - kaitai_runtime_dep, - gtest_main_dep, -] - -this_exe = executable( - meson.project_name(), - 'test-parse-nar.cc', - dependencies : test_deps, -) - -test( - meson.project_name(), - this_exe, - env : [ 'NIX_NARS_DIR=' + nars_dir ], - protocol : 'gtest', -) diff --git a/src/kaitai-struct-checks/nar.ksy b/src/kaitai-struct-checks/nar.ksy deleted file mode 120000 index c3a79a3b656b..000000000000 --- a/src/kaitai-struct-checks/nar.ksy +++ /dev/null @@ -1 +0,0 @@ -../../doc/manual/source/protocols/nix-archive/nar.ksy \ No newline at end of file diff --git a/src/kaitai-struct-checks/nars b/src/kaitai-struct-checks/nars deleted file mode 120000 index ed0b4ecc75b1..000000000000 --- a/src/kaitai-struct-checks/nars +++ /dev/null @@ -1 +0,0 @@ -../libutil-tests/data/nars \ No newline at end of file diff --git a/src/kaitai-struct-checks/package.nix b/src/kaitai-struct-checks/package.nix deleted file mode 100644 index 4257ceb76815..000000000000 --- a/src/kaitai-struct-checks/package.nix +++ /dev/null @@ -1,70 +0,0 @@ -# Run with: nix build .#nix-kaitai-struct-checks -# or: `nix develop .#nix-kaitai-struct-checks` to enter a dev shell -{ - lib, - mkMesonDerivation, - gtest, - meson, - ninja, - pkg-config, - kaitai-struct-compiler, - fetchzip, - kaitai-struct-cpp-stl-runtime, - # Configuration Options - version, -}: -let - inherit (lib) fileset; -in -mkMesonDerivation (finalAttrs: { - pname = "nix-kaitai-struct-checks"; - inherit version; - - workDir = ./.; - fileset = lib.fileset.unions [ - ../../nix-meson-build-support - ./nix-meson-build-support - ./.version - ../../.version - ../../doc/manual/source/protocols/nix-archive/nar.ksy - ./nars - ../../src/libutil-tests/data - ./meson.build - ./nar.ksy - (fileset.fileFilter (file: file.hasExt "cc") ./.) - (fileset.fileFilter (file: file.hasExt "hh") ./.) - ]; - - outputs = [ "out" ]; - - buildInputs = [ - gtest - kaitai-struct-cpp-stl-runtime - ]; - - nativeBuildInputs = [ - meson - ninja - pkg-config - # This can go away when we bump up to 25.11 - (kaitai-struct-compiler.overrideAttrs (finalAttrs: { - version = "0.11"; - src = fetchzip { - url = "https://github.com/kaitai-io/kaitai_struct_compiler/releases/download/${version}/kaitai-struct-compiler-${version}.zip"; - sha256 = "sha256-j9TEilijqgIiD0GbJfGKkU1FLio9aTopIi1v8QT1b+A="; - }; - })) - ]; - - doCheck = true; - - mesonCheckFlags = [ "--print-errorlogs" ]; - - postInstall = '' - touch $out - ''; - - meta = { - platforms = lib.platforms.unix; - }; -}) diff --git a/src/kaitai-struct-checks/test-parse-nar.cc b/src/kaitai-struct-checks/test-parse-nar.cc deleted file mode 100644 index 456ffb127410..000000000000 --- a/src/kaitai-struct-checks/test-parse-nar.cc +++ /dev/null @@ -1,48 +0,0 @@ -#include -#include -#include -#include -#include - -#include - -#include -#include -#include - -#include "nix_nar.h" - -static const std::vector NarFiles = { - "empty.nar", - "dot.nar", - "dotdot.nar", - "executable-after-contents.nar", - "invalid-tag-instead-of-contents.nar", - "name-after-node.nar", - "nul-character.nar", - "slash.nar", -}; - -class NarParseTest : public ::testing::TestWithParam -{}; - -TEST_P(NarParseTest, ParseSucceeds) -{ - const auto nar_file = GetParam(); - - const char * nars_dir_env = std::getenv("NIX_NARS_DIR"); - if (nars_dir_env == nullptr) { - FAIL() << "NIX_NARS_DIR environment variable not set."; - } - - const std::filesystem::path nar_file_path = std::filesystem::path(nars_dir_env) / "dot.nar"; - ASSERT_TRUE(std::filesystem::exists(nar_file_path)) << "Missing test file: " << nar_file_path; - - std::ifstream ifs(nar_file_path, std::ifstream::binary); - ASSERT_TRUE(ifs.is_open()) << "Failed to open file: " << nar_file; - kaitai::kstream ks(&ifs); - nix_nar_t nar(&ks); - ASSERT_TRUE(nar.root_node() != nullptr) << "Failed to parse NAR file: " << nar_file; -} - -INSTANTIATE_TEST_SUITE_P(AllNarFiles, NarParseTest, ::testing::ValuesIn(NarFiles)); diff --git a/src/libcmd/command.cc b/src/libcmd/command.cc index 226b65f4a7c3..f61d38e4a3ea 100644 --- a/src/libcmd/command.cc +++ b/src/libcmd/command.cc @@ -4,6 +4,7 @@ #include "nix/cmd/command.hh" #include "nix/cmd/legacy.hh" #include "nix/cmd/markdown.hh" +#include "nix/store/globals.hh" #include "nix/store/store-open.hh" #include "nix/store/local-fs-store.hh" #include "nix/store/derivations.hh" @@ -63,6 +64,25 @@ void NixMultiCommand::run() command->second->run(); } +StoreConfigCommand::StoreConfigCommand() {} + +ref StoreConfigCommand::getStoreConfig() +{ + if (!_storeConfig) + _storeConfig = createStoreConfig(); + return ref(_storeConfig); +} + +ref StoreConfigCommand::createStoreConfig() +{ + return resolveStoreConfig(StoreReference{settings.storeUri.get()}); +} + +void StoreConfigCommand::run() +{ + run(getStoreConfig()); +} + StoreCommand::StoreCommand() {} ref StoreCommand::getStore() @@ -74,12 +94,20 @@ ref StoreCommand::getStore() ref StoreCommand::createStore() { - return openStore(); + auto store = getStoreConfig()->openStore(); + store->init(); + return store; } -void StoreCommand::run() +void StoreCommand::run(ref storeConfig) { - run(getStore()); + // We can either efficiently implement getStore/createStore with memoization, + // or use the StoreConfig passed in run. + // It's more efficient to memoize, especially since there are some direct users + // of getStore. The StoreConfig in both cases should be the same, though. + auto store = getStore(); + assert(&*storeConfig == &store->config); + run(std::move(store)); } CopyCommand::CopyCommand() @@ -88,28 +116,28 @@ CopyCommand::CopyCommand() .longName = "from", .description = "URL of the source Nix store.", .labels = {"store-uri"}, - .handler = {&srcUri}, + .handler = {[this](std::string s) { srcUri = StoreReference::parse(s); }}, }); addFlag({ .longName = "to", .description = "URL of the destination Nix store.", .labels = {"store-uri"}, - .handler = {&dstUri}, + .handler = {[this](std::string s) { dstUri = StoreReference::parse(s); }}, }); } -ref CopyCommand::createStore() +ref CopyCommand::createStoreConfig() { - return srcUri.empty() ? StoreCommand::createStore() : openStore(srcUri); + return !srcUri ? StoreCommand::createStoreConfig() : resolveStoreConfig(StoreReference{*srcUri}); } ref CopyCommand::getDstStore() { - if (srcUri.empty() && dstUri.empty()) + if (!srcUri && !dstUri) throw UsageError("you must pass '--from' and/or '--to'"); - return dstUri.empty() ? openStore() : openStore(dstUri); + return !dstUri ? openStore() : openStore(StoreReference{*dstUri}); } EvalCommand::EvalCommand() @@ -131,7 +159,7 @@ EvalCommand::~EvalCommand() ref EvalCommand::getEvalStore() { if (!evalStore) - evalStore = evalStoreUrl ? openStore(*evalStoreUrl) : getStore(); + evalStore = evalStoreUrl ? openStore(StoreReference{*evalStoreUrl}) : getStore(); return ref(evalStore); } @@ -264,18 +292,18 @@ MixProfile::MixProfile() }); } -void MixProfile::updateProfile(const StorePath & storePath) +void MixProfile::updateProfile(Store & store_, const StorePath & storePath) { if (!profile) return; - auto store = getDstStore().dynamic_pointer_cast(); + auto * store = dynamic_cast(&store_); if (!store) throw Error("'--profile' is not supported for this Nix store"); auto profile2 = absPath(*profile); switchLink(profile2, createGeneration(*store, profile2, storePath)); } -void MixProfile::updateProfile(const BuiltPaths & buildables) +void MixProfile::updateProfile(Store & store, const BuiltPaths & buildables) { if (!profile) return; @@ -299,14 +327,16 @@ void MixProfile::updateProfile(const BuiltPaths & buildables) throw UsageError( "'--profile' requires that the arguments produce a single store path, but there are %d", result.size()); - updateProfile(result[0]); + updateProfile(store, result[0]); } MixDefaultProfile::MixDefaultProfile() { - profile = getDefaultProfile().string(); + profile = getDefaultProfile(settings.getProfileDirsOptions()).string(); } +static constexpr auto environmentVariablesCategory = "Options that change environment variables"; + MixEnvironment::MixEnvironment() : ignoreEnvironment(false) { diff --git a/src/libcmd/common-eval-args.cc b/src/libcmd/common-eval-args.cc index 865901febf48..3b0e07b2b1fa 100644 --- a/src/libcmd/common-eval-args.cc +++ b/src/libcmd/common-eval-args.cc @@ -143,7 +143,7 @@ MixEvalArgs::MixEvalArgs() )", .category = category, .labels = {"store-url"}, - .handler = {&evalStoreUrl}, + .handler = {[this](std::string s) { evalStoreUrl = StoreReference::parse(s); }}, }); } diff --git a/src/libcmd/get-build-log.cc b/src/libcmd/get-build-log.cc new file mode 100644 index 000000000000..8bbbf589945c --- /dev/null +++ b/src/libcmd/get-build-log.cc @@ -0,0 +1,31 @@ +#include "nix/cmd/get-build-log.hh" +#include "nix/store/log-store.hh" +#include "nix/store/store-open.hh" + +namespace nix { + +std::string fetchBuildLog(ref store, const StorePath & path, std::string_view what) +{ + auto subs = getDefaultSubstituters(); + + subs.push_front(store); + + for (auto & sub : subs) { + auto * logSubP = dynamic_cast(&*sub); + if (!logSubP) { + printInfo("Skipped '%s' which does not support retrieving build logs", sub->config.getHumanReadableURI()); + continue; + } + auto & logSub = *logSubP; + + auto log = logSub.getBuildLog(path); + if (!log) + continue; + printInfo("got build log for '%s' from '%s'", what, logSub.config.getHumanReadableURI()); + return *log; + } + + throw Error("build log of '%s' is not available", what); +} + +} // namespace nix diff --git a/src/libcmd/include/nix/cmd/command.hh b/src/libcmd/include/nix/cmd/command.hh index ec2e0d9add4c..cfbd9eeec96e 100644 --- a/src/libcmd/include/nix/cmd/command.hh +++ b/src/libcmd/include/nix/cmd/command.hh @@ -5,6 +5,7 @@ #include "nix/util/args.hh" #include "nix/cmd/common-eval-args.hh" #include "nix/store/path.hh" +#include "nix/store/store-reference.hh" #include "nix/flake/lockfile.hh" #include @@ -41,27 +42,42 @@ struct NixMultiCommand : MultiCommand, virtual Command #pragma GCC diagnostic ignored "-Woverloaded-virtual" /** - * A command that requires a \ref Store "Nix store". + * A command that requires a \ref StoreConfig store configuration. */ -struct StoreCommand : virtual Command +struct StoreConfigCommand : virtual Command { - StoreCommand(); + StoreConfigCommand(); void run() override; /** - * Return the default Nix store. + * Return the default Nix store configuration. */ - ref getStore(); + ref getStoreConfig(); + virtual ref createStoreConfig(); /** - * Return the destination Nix store. + * Main entry point, with a `StoreConfig` provided */ - virtual ref getDstStore() - { - return getStore(); - } + virtual void run(ref) = 0; + +private: + std::shared_ptr _storeConfig; +}; + +/** + * A command that requires a \ref Store "Nix store". + */ +struct StoreCommand : virtual StoreConfigCommand +{ + StoreCommand(); + void run(ref) override; + + /** + * Return the default Nix store. + */ + ref getStore(); - virtual ref createStore(); + ref createStore(); /** * Main entry point, with a `Store` provided */ @@ -77,13 +93,13 @@ private: */ struct CopyCommand : virtual StoreCommand { - std::string srcUri, dstUri; + std::optional srcUri, dstUri; CopyCommand(); - ref createStore() override; + ref createStoreConfig() override; - ref getDstStore() override; + ref getDstStore(); }; /** @@ -299,11 +315,11 @@ struct StorePathCommand : public StorePathsCommand */ struct RegisterCommand { - typedef std::map, std::function()>> Commands; + typedef std::map, fun()>> Commands; static Commands & commands(); - RegisterCommand(std::vector && name, std::function()> command) + RegisterCommand(std::vector && name, fun()> command) { commands().emplace(name, command); } @@ -330,11 +346,11 @@ struct MixProfile : virtual StoreCommand MixProfile(); /* If 'profile' is set, make it point at 'storePath'. */ - void updateProfile(const StorePath & storePath); + void updateProfile(Store & store, const StorePath & storePath); /* If 'profile' is set, make it point at the store path produced by 'buildables'. */ - void updateProfile(const BuiltPaths & buildables); + void updateProfile(Store & store, const BuiltPaths & buildables); }; struct MixDefaultProfile : MixProfile @@ -405,7 +421,7 @@ void createOutLinks(const std::filesystem::path & outLink, const BuiltPaths & bu struct MixOutLinkBase : virtual Args { /** Prefix for any output symlinks. Empty means do not write an output symlink. */ - Path outLink; + std::filesystem::path outLink; MixOutLinkBase(const std::string & defaultOutLink) : outLink(defaultOutLink) @@ -433,7 +449,7 @@ struct MixOutLinkByDefault : MixOutLinkBase, virtual Args addFlag({ .longName = "no-link", .description = "Do not create symlinks to the build results.", - .handler = {&outLink, Path("")}, + .handler = {&outLink, std::filesystem::path{}}, }); } }; diff --git a/src/libcmd/include/nix/cmd/common-eval-args.hh b/src/libcmd/include/nix/cmd/common-eval-args.hh index 4f9ebb83df53..93ffbd435ef9 100644 --- a/src/libcmd/include/nix/cmd/common-eval-args.hh +++ b/src/libcmd/include/nix/cmd/common-eval-args.hh @@ -6,6 +6,7 @@ #include "nix/main/common-args.hh" #include "nix/expr/search-path.hh" #include "nix/expr/eval-settings.hh" +#include "nix/store/store-reference.hh" #include @@ -52,7 +53,7 @@ struct MixEvalArgs : virtual Args, virtual MixRepair LookupPath lookupPath; - std::optional evalStoreUrl; + std::optional evalStoreUrl; private: struct AutoArgExpr diff --git a/src/libcmd/include/nix/cmd/get-build-log.hh b/src/libcmd/include/nix/cmd/get-build-log.hh new file mode 100644 index 000000000000..81d3a0dd3b6a --- /dev/null +++ b/src/libcmd/include/nix/cmd/get-build-log.hh @@ -0,0 +1,23 @@ +#pragma once +///@file + +#include "nix/store/store-api.hh" + +#include +#include + +namespace nix { + +/** + * Fetch the build log for a store path, searching the store and its + * substituters. + * + * @param store The store to search (and its substituters). + * @param path The store path to get the build log for. + * @param what A description of what we're fetching the log for (used in messages). + * @return The build log content. + * @throws Error if the build log is not available. + */ +std::string fetchBuildLog(ref store, const StorePath & path, std::string_view what); + +} // namespace nix diff --git a/src/libcmd/include/nix/cmd/legacy.hh b/src/libcmd/include/nix/cmd/legacy.hh index d408cde7ac41..99cb5418876c 100644 --- a/src/libcmd/include/nix/cmd/legacy.hh +++ b/src/libcmd/include/nix/cmd/legacy.hh @@ -1,13 +1,14 @@ #pragma once ///@file -#include +#include "nix/util/fun.hh" + #include #include namespace nix { -typedef std::function MainFunction; +typedef fun MainFunction; struct RegisterLegacyCommand { @@ -15,9 +16,9 @@ struct RegisterLegacyCommand static Commands & commands(); - RegisterLegacyCommand(const std::string & name, MainFunction fun) + RegisterLegacyCommand(const std::string & name, MainFunction command) { - commands()[name] = fun; + commands().insert_or_assign(name, command); } }; diff --git a/src/libcmd/include/nix/cmd/meson.build b/src/libcmd/include/nix/cmd/meson.build index 7ab3e596ae40..6dea1ea5be8c 100644 --- a/src/libcmd/include/nix/cmd/meson.build +++ b/src/libcmd/include/nix/cmd/meson.build @@ -10,6 +10,7 @@ headers = files( 'compatibility-settings.hh', 'editor-for.hh', 'flake-schemas.hh', + 'get-build-log.hh', 'installable-attr-path.hh', 'installable-derived-path.hh', 'installable-flake.hh', @@ -21,4 +22,5 @@ headers = files( 'network-proxy.hh', 'repl-interacter.hh', 'repl.hh', + 'unix-socket-server.hh', ) diff --git a/src/libcmd/include/nix/cmd/repl-interacter.hh b/src/libcmd/include/nix/cmd/repl-interacter.hh index 89e854ad9063..7cba481059c9 100644 --- a/src/libcmd/include/nix/cmd/repl-interacter.hh +++ b/src/libcmd/include/nix/cmd/repl-interacter.hh @@ -2,7 +2,9 @@ /// @file #include "nix/util/finally.hh" +#include "nix/util/fun.hh" #include "nix/util/types.hh" +#include #include #include @@ -14,6 +16,7 @@ namespace detail { struct ReplCompleterMixin { virtual StringSet completePrefix(const std::string & prefix) = 0; + virtual ~ReplCompleterMixin() = default; }; }; // namespace detail @@ -25,7 +28,7 @@ enum class ReplPromptType { class ReplInteracter { public: - using Guard = Finally>; + using Guard = Finally>; virtual Guard init(detail::ReplCompleterMixin * repl) = 0; /** Returns a boolean of whether the interacter got EOF */ @@ -35,10 +38,10 @@ public: class ReadlineLikeInteracter : public virtual ReplInteracter { - std::string historyFile; + std::filesystem::path historyFile; public: - ReadlineLikeInteracter(std::string historyFile) - : historyFile(historyFile) + ReadlineLikeInteracter(std::filesystem::path historyFile) + : historyFile(std::move(historyFile)) { } diff --git a/src/libcmd/include/nix/cmd/repl.hh b/src/libcmd/include/nix/cmd/repl.hh index b72a9b7d1d78..d46aa94b6b50 100644 --- a/src/libcmd/include/nix/cmd/repl.hh +++ b/src/libcmd/include/nix/cmd/repl.hh @@ -2,6 +2,7 @@ ///@file #include "nix/expr/eval.hh" +#include "nix/util/os-string.hh" namespace nix { @@ -25,10 +26,9 @@ struct AbstractNixRepl * @todo this is a layer violation * * @param programName Name of the command, e.g. `nix` or `nix-env`. - * @param args aguments to the command. + * @param args arguments to the command. */ - using RunNix = - void(const std::string & programName, const Strings & args, const std::optional & input); + using RunNix = void(const std::string & programName, OsStrings args, const std::optional & input); /** * @param runNix Function to run the nix CLI to support various @@ -37,9 +37,8 @@ struct AbstractNixRepl */ static std::unique_ptr create( const LookupPath & lookupPath, - nix::ref store, ref state, - std::function getValues, + fun getValues, RunNix * runNix = nullptr); static ReplExitStatus runSimple(ref evalState, const ValMap & extraEnv); diff --git a/src/libcmd/include/nix/cmd/unix-socket-server.hh b/src/libcmd/include/nix/cmd/unix-socket-server.hh new file mode 100644 index 000000000000..3aeee5b55e1c --- /dev/null +++ b/src/libcmd/include/nix/cmd/unix-socket-server.hh @@ -0,0 +1,79 @@ +#pragma once +///@file + +#include "nix/util/file-descriptor.hh" + +#include +#include +#include +#include + +namespace nix::unix { + +/** + * Information about the identity of the peer on a Unix domain socket connection. + */ +struct PeerInfo +{ + std::optional pid; + std::optional uid; + std::optional gid; +}; + +/** + * Get the identity of the caller, if possible. + */ +PeerInfo getPeerInfo(Descriptor remote); + +/** + * Callback type for handling new connections. + * + * The callback receives ownership of the connection and is responsible + * for handling it (e.g., forking a child process, spawning a thread, etc.). + * + * @param socket The accepted connection file descriptor. + * @param closeListeners A callback to close the listening sockets. + * Useful in forked child processes to release the bound sockets. + */ +using UnixSocketHandler = fun closeListeners)>; + +/** + * Options for the serve loop. + * + * Only used if no systemd socket activation is detected. + */ +struct ServeUnixSocketOptions +{ + /** + * The Unix domain socket path to create and listen on. + */ + std::filesystem::path socketPath; + + /** + * Mode for the created socket file. + */ + mode_t socketMode = 0666; +}; + +/** + * Run a server loop that accepts connections and calls the handler for each. + * + * This function handles: + * - systemd socket activation (via LISTEN_FDS environment variable) + * - Creating and binding a Unix domain socket if no activation is detected + * - Polling for incoming connections + * - Accepting connections + * + * For each accepted connection, the handler is called with the connection + * file descriptor. The handler takes ownership of the file descriptor and + * is responsible for closing it when done. + * + * This function never returns normally. It runs until interrupted + * (e.g., via SIGINT), at which point it throws `Interrupted`. + * + * @param options Configuration for the server. + * @param handler Callback invoked for each accepted connection. + */ +[[noreturn]] void serveUnixSocket(const ServeUnixSocketOptions & options, UnixSocketHandler handler); + +} // namespace nix::unix diff --git a/src/libcmd/installable-flake.cc b/src/libcmd/installable-flake.cc index ebec82f2e30e..28ee953b2591 100644 --- a/src/libcmd/installable-flake.cc +++ b/src/libcmd/installable-flake.cc @@ -335,8 +335,10 @@ FlakeRef InstallableFlake::nixpkgsFlakeRef() const if (auto nixpkgsInput = lockedFlake->lockFile.findInput({"nixpkgs"})) { if (auto lockedNode = std::dynamic_pointer_cast(nixpkgsInput)) { - debug("using nixpkgs flake '%s'", lockedNode->lockedRef); - return std::move(lockedNode->lockedRef); + if (lockedNode->isFlake) { + debug("using nixpkgs flake '%s'", lockedNode->lockedRef); + return std::move(lockedNode->lockedRef); + } } } diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 740e53d74af4..b297685250b5 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -117,7 +117,11 @@ MixFlakeOptions::MixFlakeOptions() .labels = {"input-path"}, .handler = {[&](std::string s) { warn("'--update-input' is a deprecated alias for 'flake update' and will be removed in a future version."); - lockFlags.inputUpdates.insert(flake::parseInputAttrPath(s)); + auto path = flake::NonEmptyInputAttrPath::parse(s); + if (!path) + throw UsageError( + "--update-input was passed a zero-length input path, which would refer to the flake itself, not an input"); + lockFlags.inputUpdates.insert(*path); }}, .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) { completeFlakeInputAttrPath(completions, getEvalState(), getFlakeRefsForCompletion(), prefix); @@ -126,14 +130,18 @@ MixFlakeOptions::MixFlakeOptions() addFlag({ .longName = "override-input", - .description = "Override a specific flake input (e.g. `dwarffs/nixpkgs`). This implies `--no-write-lock-file`.", + .description = + "Override a specific flake input (e.g. `dwarffs/nixpkgs`). The input path must not be empty. This implies `--no-write-lock-file`.", .category = category, .labels = {"input-path", "flake-url"}, .handler = {[&](std::string inputAttrPath, std::string flakeRef) { lockFlags.writeLockFile = false; + auto path = flake::NonEmptyInputAttrPath::parse(inputAttrPath); + if (!path) + throw UsageError( + "--override-input was passed a zero-length input path, which would refer to the flake itself, not an input"); lockFlags.inputOverrides.insert_or_assign( - flake::parseInputAttrPath(inputAttrPath), - parseFlakeRef(fetchSettings, flakeRef, absPath(getCommandBaseDir()).string(), true)); + std::move(*path), parseFlakeRef(fetchSettings, flakeRef, absPath(getCommandBaseDir()).string(), true)); }}, .completer = {[&](AddCompletions & completions, size_t n, std::string_view prefix) { if (n == 0) { @@ -150,7 +158,7 @@ MixFlakeOptions::MixFlakeOptions() .category = category, .labels = {"flake-lock-path"}, .handler = {[&](std::string lockFilePath) { - lockFlags.referenceLockFilePath = {getFSSourceAccessor(), CanonPath(absPath(lockFilePath))}; + lockFlags.referenceLockFilePath = {getFSSourceAccessor(), CanonPath(absPath(lockFilePath).string())}; }}, .completer = completePath, }); @@ -327,7 +335,7 @@ try { {}} .getCompletions(flakeRefS, completions); } catch (Error & e) { - warn(e.msg()); + logWarning(e.info()); } void completeFlakeRef(AddCompletions & completions, ref store, std::string_view prefix) @@ -490,7 +498,7 @@ const BuiltPathWithResult & InstallableWithBuildResult::getSuccess() const if (auto * failure = std::get_if(&result)) { auto failure2 = failure->tryGetFailure(); assert(failure2); - failure2->rethrow(); + throw *failure2; } else return *std::get_if(&result); } @@ -524,7 +532,7 @@ void Installable::throwBuildErrors(std::vector & bui assert(failure2); printError("❌ " ANSI_RED "%s" ANSI_NORMAL, buildResult.installable->what()); try { - failure2->rethrow(); + throw *failure2; } catch (Error & e) { logError(e.info()); } diff --git a/src/libcmd/markdown.cc b/src/libcmd/markdown.cc index c3341da73a26..2cdc8e38c234 100644 --- a/src/libcmd/markdown.cc +++ b/src/libcmd/markdown.cc @@ -38,7 +38,9 @@ static std::string doRenderMarkdownToTerminal(std::string_view markdown) # endif .feat = LOWDOWN_COMMONMARK | LOWDOWN_FENCED | LOWDOWN_DEFLIST | LOWDOWN_TABLES, .oflags = -# if HAVE_LOWDOWN_1_4 +# if HAVE_LOWDOWN_3 + LOWDOWN_NORELLINK +# elif HAVE_LOWDOWN_1_4 LOWDOWN_TERM_NORELLINK // To render full links while skipping relative ones # else LOWDOWN_TERM_NOLINK diff --git a/src/libcmd/meson.build b/src/libcmd/meson.build index 087da84f9293..652279240657 100644 --- a/src/libcmd/meson.build +++ b/src/libcmd/meson.build @@ -44,6 +44,10 @@ configdata.set( 'HAVE_LOWDOWN_1_4', lowdown.version().version_compare('>= 1.4.0').to_int(), ) +configdata.set( + 'HAVE_LOWDOWN_3', + lowdown.version().version_compare('>= 3.0.0').to_int(), +) readline_flavor = get_option('readline-flavor') if readline_flavor == 'editline' @@ -76,6 +80,7 @@ sources = files( 'common-eval-args.cc', 'editor-for.cc', 'flake-schemas.cc', + 'get-build-log.cc', 'installable-attr-path.cc', 'installable-derived-path.cc', 'installable-flake.cc', @@ -88,6 +93,12 @@ sources = files( 'repl.cc', ) +if host_machine.system() != 'windows' + sources += files( + 'unix/unix-socket-server.cc', + ) +endif + sources += [ gen_header.process('call-flake-schemas.nix'), gen_header.process('builtin-flake-schemas.nix'), diff --git a/src/libcmd/repl-interacter.cc b/src/libcmd/repl-interacter.cc index c9b435675405..8eebeec25b5e 100644 --- a/src/libcmd/repl-interacter.cc +++ b/src/libcmd/repl-interacter.cc @@ -40,8 +40,8 @@ void sigintHandler(int signo) static detail::ReplCompleterMixin * curRepl; // ugly #if !USE_READLINE -static char * completionCallback(char * s, int * match) -{ +static char * completionCallback(char * s, int * match) noexcept +try { auto possible = curRepl->completePrefix(s); if (possible.size() == 1) { *match = 1; @@ -73,10 +73,12 @@ static char * completionCallback(char * s, int * match) *match = 0; return nullptr; +} catch (...) { + return nullptr; } -static int listPossibleCallback(char * s, char *** avp) -{ +static int listPossibleCallback(char * s, char *** avp) noexcept +try { auto possible = curRepl->completePrefix(s); if (possible.size() > (std::numeric_limits::max() / sizeof(char *))) @@ -105,6 +107,9 @@ static int listPossibleCallback(char * s, char *** avp) *avp = vp; return ac; +} catch (...) { + *avp = nullptr; + return 0; } #endif @@ -113,14 +118,14 @@ ReadlineLikeInteracter::Guard ReadlineLikeInteracter::init(detail::ReplCompleter // Allow nix-repl specific settings in .inputrc rl_readline_name = "nix-repl"; try { - createDirs(dirOf(historyFile)); + createDirs(historyFile.parent_path()); } catch (SystemError & e) { logWarning(e.info()); } #if !USE_READLINE el_hist_size = 1000; #endif - read_history(historyFile.c_str()); + read_history(historyFile.string().c_str()); auto oldRepl = curRepl; curRepl = repl; Guard restoreRepl([oldRepl] { curRepl = oldRepl; }); @@ -203,7 +208,7 @@ bool ReadlineLikeInteracter::getLine(std::string & input, ReplPromptType promptT ReadlineLikeInteracter::~ReadlineLikeInteracter() { - write_history(historyFile.c_str()); + write_history(historyFile.string().c_str()); } }; // namespace nix diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index d8e61b5b5205..19e842c2ce6e 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -1,6 +1,7 @@ #include #include #include +#include #include "nix/util/error.hh" #include "nix/cmd/repl-interacter.hh" @@ -12,9 +13,8 @@ #include "nix/expr/eval-settings.hh" #include "nix/expr/attr-path.hh" #include "nix/util/signals.hh" -#include "nix/store/store-open.hh" -#include "nix/store/log-store.hh" #include "nix/cmd/common-eval-args.hh" +#include "nix/cmd/get-build-log.hh" #include "nix/expr/get-drvs.hh" #include "nix/store/derivations.hh" #include "nix/store/globals.hh" @@ -29,6 +29,8 @@ #include "nix/util/ref.hh" #include "nix/expr/value.hh" +#include "nix/util/os-string.hh" +#include "nix/util/processes.hh" #include "nix/util/strings.hh" namespace nix { @@ -61,27 +63,22 @@ struct NixRepl : AbstractNixRepl, detail::ReplCompleterMixin, gc std::list loadedFiles; // Arguments passed to :load-flake, saved so they can be reloaded with :reload Strings loadedFlakes; - std::function getValues; + fun getValues; const static int envSize = 32768; std::shared_ptr staticEnv; - Value lastLoaded; + std::optional lastLoaded; Env * env; int displ; StringSet varNames; RunNix * runNixPtr; - void runNix(const std::string & program, const Strings & args, const std::optional & input = {}); + void runNix(const std::string & program, OsStrings args, const std::optional & input = {}); std::unique_ptr interacter; - NixRepl( - const LookupPath & lookupPath, - nix::ref store, - ref state, - std::function getValues, - RunNix * runNix); + NixRepl(const LookupPath & lookupPath, ref state, fun getValues, RunNix * runNix); virtual ~NixRepl() = default; ReplExitStatus mainLoop() override; @@ -100,6 +97,7 @@ struct NixRepl : AbstractNixRepl, detail::ReplCompleterMixin, gc void addAttrsToScope(Value & attrs); void addVarToScope(const Symbol name, Value & v); Expr * parseString(std::string s); + ExprAttrs * parseReplBindings(std::string s); void evalString(std::string s, Value & v); void loadDebugTraceEnv(DebugTrace & dt); @@ -132,17 +130,13 @@ std::string removeWhitespace(std::string s) } NixRepl::NixRepl( - const LookupPath & lookupPath, - nix::ref store, - ref state, - std::function getValues, - RunNix * runNix) + const LookupPath & lookupPath, ref state, fun getValues, RunNix * runNix) : AbstractNixRepl(state) , debugTraceIndex(0) , getValues(getValues) , staticEnv(new StaticEnv(nullptr, state->staticBaseEnv)) , runNixPtr{runNix} - , interacter(make_unique((getDataDir() / "repl-history").string())) + , interacter(std::make_unique(getDataDir() / "repl-history")) { } @@ -309,21 +303,6 @@ StringSet NixRepl::completePrefix(const std::string & prefix) return completions; } -// FIXME: DRY and match or use the parser -static bool isVarName(std::string_view s) -{ - if (s.size() == 0) - return false; - char c = s[0]; - if ((c >= '0' && c <= '9') || c == '-' || c == '\'') - return false; - for (auto & i : s) - if (!((i >= 'a' && i <= 'z') || (i >= 'A' && i <= 'Z') || (i >= '0' && i <= '9') || i == '_' || i == '-' - || i == '\'')) - return false; - return true; -} - StorePath NixRepl::getDerivationPath(Value & v) { auto packageInfo = getDerivation(*state, v, false); @@ -512,7 +491,12 @@ ProcessLineResult NixRepl::processLine(std::string line) // runProgram redirects stdout to a StringSink, // using runProgram2 to allow editors to display their UI - runProgram2(RunOptions{.program = editor, .lookupPath = true, .args = args, .isInteractive = true}); + runProgram2({ + .program = editor, + .lookupPath = true, + .args = toOsStrings(std::move(args)), + .isInteractive = true, + }); // Reload right after exiting the editor state->resetFileCache(); @@ -532,7 +516,7 @@ ProcessLineResult NixRepl::processLine(std::string line) state->callFunction(f, v, result, PosIdx()); StorePath drvPath = getDerivationPath(result); - runNix("nix-shell", {state->store->printStorePath(drvPath)}); + runNix("nix-shell", toOsStrings({state->store->printStorePath(drvPath)})); } else if (command == ":b" || command == ":bl" || command == ":i" || command == ":sh" || command == ":log") { @@ -563,37 +547,15 @@ ProcessLineResult NixRepl::processLine(std::string line) } } } else if (command == ":i") { - runNix("nix-env", {"-i", drvPathRaw}); + runNix("nix-env", toOsStrings({"-i", drvPathRaw})); } else if (command == ":log") { settings.readOnlyMode = true; Finally roModeReset([&]() { settings.readOnlyMode = false; }); - auto subs = getDefaultSubstituters(); - - subs.push_front(state->store); - - bool foundLog = false; RunPager pager; - for (auto & sub : subs) { - auto * logSubP = dynamic_cast(&*sub); - if (!logSubP) { - printInfo( - "Skipped '%s' which does not support retrieving build logs", sub->config.getHumanReadableURI()); - continue; - } - auto & logSub = *logSubP; - - auto log = logSub.getBuildLog(drvPath); - if (log) { - printInfo("got build log for '%s' from '%s'", drvPathRaw, logSub.config.getHumanReadableURI()); - logger->writeToStdout(*log); - foundLog = true; - break; - } - } - if (!foundLog) - throw Error("build log of '%s' is not available", drvPathRaw); + auto log = fetchBuildLog(state->store, drvPath, drvPathRaw); + logger->writeToStdout(log); } else { - runNix("nix-shell", {drvPathRaw}); + runNix("nix-shell", toOsStrings({drvPathRaw})); } } @@ -694,15 +656,22 @@ ProcessLineResult NixRepl::processLine(std::string line) throw Error("unknown command '%1%'", command); else { - size_t p = line.find('='); - std::string name; - if (p != std::string::npos && p < line.size() && line[p + 1] != '=' - && isVarName(name = removeWhitespace(line.substr(0, p)))) { - Expr * e = parseString(line.substr(p + 1)); - Value & v(*state->allocValue()); - v.mkThunk(env, e); - addVarToScope(state->symbols.create(name), v); + // Try parsing as bindings first (handles `x = 1`, `inherit ...`, etc.) + ExprAttrs * bindings = nullptr; + try { + bindings = parseReplBindings(line); + } catch (ParseError &) { + } + + if (bindings) { + Env * inheritEnv = bindings->inheritFromExprs ? bindings->buildInheritFromEnv(*state, *env) : nullptr; + for (auto & [symbol, def] : *bindings->attrs) { + Value & v(*state->allocValue()); + v.mkThunk(def.chooseByKind(env, env, inheritEnv), def.e); + addVarToScope(symbol, v); + } } else { + // Otherwise evaluate as expression Value v; evalString(line, v); auto suspension = logger->suspend(); @@ -736,7 +705,7 @@ void NixRepl::loadFlake(const std::string & flakeRefS) try { cwd = std::filesystem::current_path(); } catch (std::filesystem::filesystem_error & e) { - throw SysError("cannot determine current working directory"); + throw SystemError(e.code(), "cannot determine current working directory"); } auto flakeRef = parseFlakeRef(fetchSettings, flakeRefS, cwd.string(), true); @@ -774,11 +743,19 @@ void NixRepl::initEnv() void NixRepl::showLastLoaded() { - RunPager pager; + if (!lastLoaded) + throw Error("nothing has been loaded yet"); - for (auto & i : *lastLoaded.attrs()) { - std::string_view name = state->symbols[i.name]; - logger->cout(name); + RunPager pager; + try { + for (auto & i : *lastLoaded->attrs()) { + std::string_view name = state->symbols[i.name]; + logger->cout(name); + } + } catch (SystemError & e) { + /* Ignore broken pipes when the pager gets interrupted. */ + if (!e.is(std::errc::broken_pipe)) + throw; } } @@ -796,7 +773,7 @@ void NixRepl::loadFiles() loadedFiles.clear(); for (auto & i : old) { - notice("Loading '%1%'...", i); + notice("Loading %1%...", PathFmt(i)); loadFile(i); } @@ -883,6 +860,28 @@ Expr * NixRepl::parseString(std::string s) } } +ExprAttrs * NixRepl::parseReplBindings(std::string s) +{ + auto basePath = state->rootPath("."); + + // Try parsing as bindings + std::exception_ptr bindingsError; + try { + return state->parseReplBindings(s, basePath, staticEnv); + } catch (ParseError &) { + bindingsError = std::current_exception(); + } + + // Try with semicolon appended (for `inherit foo` shorthand) + // Use original source (s) for error messages, not s + ";" + try { + return state->parseReplBindings(s + ";", s, basePath, staticEnv); + } catch (ParseError &) { + // Semicolon retry failed; rethrow the original bindings error + std::rethrow_exception(bindingsError); + } +} + void NixRepl::evalString(std::string s, Value & v) { Expr * e = parseString(s); @@ -890,10 +889,10 @@ void NixRepl::evalString(std::string s, Value & v) state->forceValue(v, v.determinePos(noPos)); } -void NixRepl::runNix(const std::string & program, const Strings & args, const std::optional & input) +void NixRepl::runNix(const std::string & program, OsStrings args, const std::optional & input) { if (runNixPtr) - (*runNixPtr)(program, args, input); + (*runNixPtr)(program, std::move(args), input); else throw Error( "Cannot run '%s' because no method of calling the Nix CLI was provided. This is a configuration problem pertaining to how this program was built. See Nix 2.25 release notes", @@ -901,13 +900,9 @@ void NixRepl::runNix(const std::string & program, const Strings & args, const st } std::unique_ptr AbstractNixRepl::create( - const LookupPath & lookupPath, - nix::ref store, - ref state, - std::function getValues, - RunNix * runNix) + const LookupPath & lookupPath, ref state, fun getValues, RunNix * runNix) { - return std::make_unique(lookupPath, std::move(store), state, getValues, runNix); + return std::make_unique(lookupPath, state, getValues, runNix); } ReplExitStatus AbstractNixRepl::runSimple(ref evalState, const ValMap & extraEnv) @@ -920,7 +915,6 @@ ReplExitStatus AbstractNixRepl::runSimple(ref evalState, const ValMap // NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDelete) auto repl = std::make_unique( lookupPath, - openStore(), evalState, getValues, /*runNix=*/nullptr); diff --git a/src/libcmd/unix/unix-socket-server.cc b/src/libcmd/unix/unix-socket-server.cc new file mode 100644 index 000000000000..8cc2bd713d07 --- /dev/null +++ b/src/libcmd/unix/unix-socket-server.cc @@ -0,0 +1,126 @@ +///@file + +#include "nix/cmd/unix-socket-server.hh" +#include "nix/util/environment-variables.hh" +#include "nix/util/file-system.hh" +#include "nix/util/logging.hh" +#include "nix/util/signals.hh" +#include "nix/util/strings.hh" +#include "nix/util/unix-domain-socket.hh" +#include "nix/util/util.hh" + +#include +#include +#include + +#if defined(__APPLE__) || defined(__FreeBSD__) +# include +#endif + +namespace nix::unix { + +PeerInfo getPeerInfo(Descriptor remote) +{ + PeerInfo peer; + +#if defined(SO_PEERCRED) + +# if defined(__OpenBSD__) + struct sockpeercred cred; +# else + ucred cred; +# endif + socklen_t credLen = sizeof(cred); + if (getsockopt(remote, SOL_SOCKET, SO_PEERCRED, &cred, &credLen) == 0) { + peer.pid = cred.pid; + peer.uid = cred.uid; + peer.gid = cred.gid; + } + +#elif defined(LOCAL_PEERCRED) + +# if !defined(SOL_LOCAL) +# define SOL_LOCAL 0 +# endif + + xucred cred; + socklen_t credLen = sizeof(cred); + if (getsockopt(remote, SOL_LOCAL, LOCAL_PEERCRED, &cred, &credLen) == 0) + peer.uid = cred.cr_uid; + +#endif + + return peer; +} + +[[noreturn]] void serveUnixSocket(const ServeUnixSocketOptions & options, UnixSocketHandler handler) +{ + std::vector listeningSockets; + + static constexpr int SD_LISTEN_FDS_START = 3; + + // Handle socket-based activation by systemd. + auto listenFds = getEnv("LISTEN_FDS"); + if (listenFds) { + if (getEnv("LISTEN_PID") != std::to_string(getpid())) + throw Error("unexpected systemd environment variables"); + auto count = string2Int(*listenFds); + assert(count); + for (unsigned int i = 0; i < count; ++i) { + AutoCloseFD fdSocket(SD_LISTEN_FDS_START + i); + closeOnExec(fdSocket.get()); + listeningSockets.push_back(std::move(fdSocket)); + } + } + + // Otherwise, create and bind to a Unix domain socket. + else { + createDirs(options.socketPath.parent_path()); + listeningSockets.push_back(createUnixDomainSocket(options.socketPath.string(), options.socketMode)); + } + + std::vector fds; + for (auto & i : listeningSockets) + fds.push_back({.fd = i.get(), .events = POLLIN}); + + // Loop accepting connections. + while (1) { + try { + checkInterrupt(); + + auto count = poll(fds.data(), fds.size(), -1); + if (count == -1) { + if (errno == EINTR) + continue; + throw SysError("polling for incoming connections"); + } + + for (auto & fd : fds) { + if (!fd.revents) + continue; + + // Accept a connection. + struct sockaddr_un remoteAddr; + socklen_t remoteAddrLen = sizeof(remoteAddr); + + AutoCloseFD remote = accept(fd.fd, (struct sockaddr *) &remoteAddr, &remoteAddrLen); + checkInterrupt(); + if (!remote) { + if (errno == EINTR) + continue; + throw SysError("accepting connection"); + } + + handler(std::move(remote), [&]() { listeningSockets.clear(); }); + } + + } catch (Error & error) { + auto ei = error.info(); + // FIXME: add to trace? + ei.msg = HintFmt("while processing connection: %1%", ei.msg.str()); + logError(ei); + } + } +} + +} // namespace nix::unix diff --git a/src/libexpr-c/nix_api_expr.cc b/src/libexpr-c/nix_api_expr.cc index bfbd0a9c361f..943d90a211e6 100644 --- a/src/libexpr-c/nix_api_expr.cc +++ b/src/libexpr-c/nix_api_expr.cc @@ -186,13 +186,13 @@ EvalState * nix_eval_state_build(nix_c_context * context, nix_eval_state_builder if (context) context->last_err_code = NIX_OK; try { - return unsafe_new_with_self([&](auto * self) { - return EvalState{ - .fetchSettings = std::move(builder->fetchSettings), - .settings = std::move(builder->settings), - .state = nix::EvalState(builder->lookupPath, builder->store, self->fetchSettings, self->settings), - }; - }); + auto fetchSettings = std::make_unique(std::move(builder->fetchSettings)); + auto settings = std::make_unique(std::move(builder->settings)); + auto ownedState = + std::make_shared(builder->lookupPath, builder->store, *fetchSettings, *settings); + auto & stateRef = *ownedState; + void * p = ::operator new(sizeof(EvalState), static_cast(alignof(EvalState))); + return new (p) EvalState{stateRef, std::move(fetchSettings), std::move(settings), std::move(ownedState)}; } NIXC_CATCH_ERRS_NULL } diff --git a/src/libexpr-c/nix_api_expr_internal.h b/src/libexpr-c/nix_api_expr_internal.h index 07c7a2194df2..b38aeaf7b498 100644 --- a/src/libexpr-c/nix_api_expr_internal.h +++ b/src/libexpr-c/nix_api_expr_internal.h @@ -1,6 +1,8 @@ #ifndef NIX_API_EXPR_INTERNAL_H #define NIX_API_EXPR_INTERNAL_H +#include + #include "nix/fetchers/fetch-settings.hh" #include "nix/expr/eval.hh" #include "nix/expr/eval-settings.hh" @@ -22,9 +24,11 @@ struct nix_eval_state_builder struct EvalState { - nix::fetchers::Settings fetchSettings; - nix::EvalSettings settings; - nix::EvalState state; + nix::EvalState & state; + // Owned resources; null for temporary wrappers created in C API callbacks. + std::unique_ptr ownedFetchSettings; + std::unique_ptr ownedSettings; + std::shared_ptr ownedState; }; struct BindingsBuilder diff --git a/src/libexpr-c/nix_api_external.cc b/src/libexpr-c/nix_api_external.cc index ff2950448c69..a874d9a0861e 100644 --- a/src/libexpr-c/nix_api_external.cc +++ b/src/libexpr-c/nix_api_external.cc @@ -137,7 +137,8 @@ class NixCExternalValue : public nix::ExternalValueBase } nix_string_context ctx{context}; nix_string_return res{""}; - desc.printValueAsJSON(v, (EvalState *) &state, strict, &ctx, copyToStore, &res); + EvalState wrapper{state}; + desc.printValueAsJSON(v, &wrapper, strict, &ctx, copyToStore, &res); if (res.str.empty()) { return nix::ExternalValueBase::printValueAsJSON(state, strict, context, copyToStore); } @@ -153,22 +154,16 @@ class NixCExternalValue : public nix::ExternalValueBase bool location, nix::XMLWriter & doc, nix::NixStringContext & context, - nix::PathSet & drvsSeen, + nix::StringSet & drvsSeen, const nix::PosIdx pos) const override { if (!desc.printValueAsXML) { return nix::ExternalValueBase::printValueAsXML(state, strict, location, doc, context, drvsSeen, pos); } nix_string_context ctx{context}; + EvalState wrapper{state}; desc.printValueAsXML( - v, - (EvalState *) &state, - strict, - location, - &doc, - &ctx, - &drvsSeen, - *reinterpret_cast(&pos)); + v, &wrapper, strict, location, &doc, &ctx, &drvsSeen, *reinterpret_cast(&pos)); } virtual ~NixCExternalValue() override {}; diff --git a/src/libexpr-c/nix_api_external.h b/src/libexpr-c/nix_api_external.h index 96c479d57697..1ae4c22f89bc 100644 --- a/src/libexpr-c/nix_api_external.h +++ b/src/libexpr-c/nix_api_external.h @@ -145,6 +145,7 @@ typedef struct NixCExternalValueDesc * Optional, the default is to throw an error * @todo The mechanisms for this call are incomplete. There are no C * bindings to work with XML, pathsets and positions. + * This callback also has no test coverage. * @param[in] self the void* passed to nix_create_external_value * @param[in] state The evaluator state * @param[in] strict boolean Whether to force the value before printing diff --git a/src/libexpr-c/nix_api_value.cc b/src/libexpr-c/nix_api_value.cc index b6a838284eff..e7a439aef196 100644 --- a/src/libexpr-c/nix_api_value.cc +++ b/src/libexpr-c/nix_api_value.cc @@ -1,4 +1,5 @@ #include "nix/expr/attr-set.hh" +#include "nix/expr/eval-error.hh" #include "nix/util/configuration.hh" #include "nix/expr/eval.hh" #include "nix/store/globals.hh" @@ -104,11 +105,17 @@ static void nix_c_primop_wrapper( nix_value * external_arg = new_nix_value(args[i], state.mem); external_args.push_back(external_arg); } - f(userdata, &ctx, (EvalState *) &state, external_args.data(), vTmpPtr); + EvalState wrapper{state}; + f(userdata, &ctx, &wrapper, external_args.data(), vTmpPtr); if (ctx.last_err_code != NIX_OK) { - /* TODO: Throw different errors depending on the error code */ - state.error("Error from custom function: %s", *ctx.last_err).atPos(pos).debugThrow(); + if (ctx.last_err_code == NIX_ERR_RECOVERABLE) { + state.error("Recoverable error from custom function: %s", *ctx.last_err) + .atPos(pos) + .debugThrow(); + } else { + state.error("Error from custom function: %s", *ctx.last_err).atPos(pos).debugThrow(); + } } if (!vTmp.isValid()) { @@ -153,7 +160,7 @@ PrimOp * nix_alloc_primop( .args = {}, .arity = (size_t) arity, .doc = doc, - .fun = std::bind(nix_c_primop_wrapper, fun, user_data, arity, _1, _2, _3, _4)}; + .impl = std::bind(nix_c_primop_wrapper, fun, user_data, arity, _1, _2, _3, _4)}; if (args) for (size_t i = 0; args[i]; i++) p->args.emplace_back(*args); diff --git a/src/libexpr-tests/bench-main.cc b/src/libexpr-tests/bench-main.cc new file mode 100644 index 000000000000..13dca6b4efa3 --- /dev/null +++ b/src/libexpr-tests/bench-main.cc @@ -0,0 +1,14 @@ +#include + +#include "nix/expr/eval-gc.hh" +#include "nix/store/globals.hh" + +int main(int argc, char ** argv) +{ + nix::initLibStore(false); + nix::initGC(); + + ::benchmark::Initialize(&argc, argv); + ::benchmark::RunSpecifiedBenchmarks(); + return 0; +} diff --git a/src/libexpr-tests/dynamic-attrs-bench.cc b/src/libexpr-tests/dynamic-attrs-bench.cc new file mode 100644 index 000000000000..1b1c199bdff7 --- /dev/null +++ b/src/libexpr-tests/dynamic-attrs-bench.cc @@ -0,0 +1,56 @@ +#include + +#include "nix/expr/eval.hh" +#include "nix/expr/eval-settings.hh" +#include "nix/fetchers/fetch-settings.hh" +#include "nix/store/store-open.hh" + +using namespace nix; + +static std::string mkDynamicAttrsExpr(size_t attrCount) +{ + std::string res; + res.reserve(attrCount * 24); + res += "{ "; + for (size_t i = 0; i < attrCount; ++i) { + res += "${\"a"; + res += std::to_string(i); + res += "\"} = "; + res += std::to_string(i); + res += "; "; + } + res += "}"; + return res; +} + +static void BM_EvalDynamicAttrs(benchmark::State & state) +{ + const auto attrCount = static_cast(state.range(0)); + const auto exprStr = mkDynamicAttrsExpr(attrCount); + + for (auto _ : state) { + state.PauseTiming(); + + auto store = openStore("dummy://"); + fetchers::Settings fetchSettings{}; + bool readOnlyMode = true; + EvalSettings evalSettings{readOnlyMode}; + evalSettings.nixPath = {}; + + auto stPtr = std::make_shared(LookupPath{}, store, fetchSettings, evalSettings, nullptr); + auto & st = *stPtr; + Expr * expr = st.parseExprFromString(exprStr, st.rootPath(CanonPath::root)); + + Value v; + + state.ResumeTiming(); + + st.eval(expr, v); + st.forceValue(v, noPos); + benchmark::DoNotOptimize(v); + } + + state.SetItemsProcessed(state.iterations() * attrCount); +} + +BENCHMARK(BM_EvalDynamicAttrs)->Arg(100)->Arg(500)->Arg(2'000); diff --git a/src/libexpr-tests/get-drvs-bench.cc b/src/libexpr-tests/get-drvs-bench.cc new file mode 100644 index 000000000000..a5cd59154f22 --- /dev/null +++ b/src/libexpr-tests/get-drvs-bench.cc @@ -0,0 +1,66 @@ +#include + +#include "nix/expr/get-drvs.hh" +#include "nix/expr/eval-settings.hh" +#include "nix/fetchers/fetch-settings.hh" +#include "nix/store/store-open.hh" +#include "nix/util/fmt.hh" + +using namespace nix; + +namespace { + +struct GetDerivationsEnv +{ + ref store = openStore("dummy://"); + fetchers::Settings fetchSettings{}; + bool readOnlyMode = true; + EvalSettings evalSettings{readOnlyMode}; + std::shared_ptr statePtr; + EvalState & state; + + Bindings * autoArgs = nullptr; + Value attrsValue; + + explicit GetDerivationsEnv(size_t attrCount) + : evalSettings([&]() { + EvalSettings settings{readOnlyMode}; + settings.nixPath = {}; + return settings; + }()) + , statePtr(std::make_shared(LookupPath{}, store, fetchSettings, evalSettings, nullptr)) + , state(*statePtr) + { + autoArgs = state.buildBindings(0).finish(); + + auto attrs = state.buildBindings(attrCount); + + for (size_t i = 0; i < attrCount; ++i) { + auto name = fmt("pkg%|1$06d|", i); + auto sym = state.symbols.create(name); + auto & v = attrs.alloc(sym); + v.mkInt(i); + } + + attrsValue.mkAttrs(attrs.finish()); + } +}; + +} // namespace + +static void BM_GetDerivationsAttrScan(benchmark::State & state) +{ + const auto attrCount = static_cast(state.range(0)); + GetDerivationsEnv env(attrCount); + + for (auto _ : state) { + PackageInfos drvs; + getDerivations( + env.state, env.attrsValue, /*pathPrefix=*/"", *env.autoArgs, drvs, /*ignoreAssertionFailures=*/true); + benchmark::DoNotOptimize(drvs.size()); + } + + state.SetItemsProcessed(state.iterations() * attrCount); +} + +BENCHMARK(BM_GetDerivationsAttrScan)->Arg(1'000)->Arg(5'000)->Arg(10'000); diff --git a/src/libexpr-tests/meson.build b/src/libexpr-tests/meson.build index c5dafe0de84b..c5b72851da53 100644 --- a/src/libexpr-tests/meson.build +++ b/src/libexpr-tests/meson.build @@ -87,3 +87,33 @@ test( }, protocol : 'gtest', ) + +# Build benchmarks if enabled +if get_option('benchmarks') + gbenchmark = dependency('benchmark', required : true) + + benchmark_sources = files( + 'bench-main.cc', + 'dynamic-attrs-bench.cc', + 'get-drvs-bench.cc', + 'regex-cache-bench.cc', + ) + + benchmark_exe = executable( + 'nix-expr-benchmarks', + benchmark_sources, + config_priv_h, + dependencies : deps_private_subproject + deps_private + deps_other + [ + gbenchmark, + ], + include_directories : include_dirs, + link_args : linker_export_flags, + install : true, + cpp_pch : do_pch ? [ 'pch/precompiled-headers.hh' ] : [], + ) + + benchmark( + 'nix-expr-benchmarks', + benchmark_exe, + ) +endif diff --git a/src/libexpr-tests/meson.options b/src/libexpr-tests/meson.options new file mode 100644 index 000000000000..2b3c1af6067f --- /dev/null +++ b/src/libexpr-tests/meson.options @@ -0,0 +1,9 @@ +# vim: filetype=meson + +option( + 'benchmarks', + type : 'boolean', + value : false, + description : 'Build benchmarks (requires gbenchmark)', + yield : true, +) diff --git a/src/libexpr-tests/nix_api_expr.cc b/src/libexpr-tests/nix_api_expr.cc index c7e246c727f7..cc93a47f01ae 100644 --- a/src/libexpr-tests/nix_api_expr.cc +++ b/src/libexpr-tests/nix_api_expr.cc @@ -476,6 +476,52 @@ TEST_F(nix_api_expr_test, nix_expr_primop_nix_err_key_conversion) nix_gc_decref(ctx, result); } +static void +primop_alloc_value(void * user_data, nix_c_context * context, EvalState * state, nix_value ** args, nix_value * ret) +{ + assert(context); + assert(state); + + // Regression test: nix_c_primop_wrapper previously cast the inner + // nix::EvalState* directly to EvalState* (C wrapper). C API functions + // like nix_alloc_value() then accessed state->state at the wrong offset, + // causing a segfault. + nix_value * v = nix_alloc_value(context, state); + assert(v != nullptr); + nix_init_int(context, v, 42); + nix_copy_value(context, ret, v); + nix_gc_decref(nullptr, v); +} + +TEST_F(nix_api_expr_test, nix_primop_can_use_state_in_callback) +{ + PrimOp * primop = + nix_alloc_primop(ctx, primop_alloc_value, 1, "allocValue", nullptr, "test alloc_value in callback", nullptr); + assert_ctx_ok(); + nix_value * primopValue = nix_alloc_value(ctx, state); + assert_ctx_ok(); + nix_init_primop(ctx, primopValue, primop); + assert_ctx_ok(); + + nix_value * dummy = nix_alloc_value(ctx, state); + assert_ctx_ok(); + nix_init_int(ctx, dummy, 0); + assert_ctx_ok(); + + nix_value * result = nix_alloc_value(ctx, state); + assert_ctx_ok(); + nix_value_call(ctx, state, primopValue, dummy, result); + assert_ctx_ok(); + + auto r = nix_get_int(ctx, result); + ASSERT_EQ(42, r); + + nix_gc_decref(ctx, dummy); + nix_gc_decref(ctx, result); + nix_gc_decref(ctx, primopValue); + nix_gc_decref(ctx, primop); +} + TEST_F(nix_api_expr_test, nix_value_call_multi_no_args) { nix_value * n = nix_alloc_value(ctx, state); @@ -517,4 +563,108 @@ TEST_F(nix_api_expr_test, nix_expr_attrset_update) assert_ctx_ok(); } +// The following is a test case for retryable thunks. This is a requirement +// for the current way in which NixOps4 evaluates its deployment expressions. +// An alternative strategy could be implemented, but unwinding the stack may +// be a more efficient way to deal with many suspensions/resumptions, compared +// to e.g. using a thread or coroutine stack for each suspended dependency. +// This test models the essential bits of a deployment tool that uses such +// a strategy. + +// State for the retryable primop - simulates deployment resource availability +struct DeploymentResourceState +{ + bool vm_created = false; +}; + +static void primop_load_resource_input( + void * user_data, nix_c_context * context, EvalState * state, nix_value ** args, nix_value * ret) +{ + assert(context); + assert(state); + auto * resource_state = static_cast(user_data); + + // Get the resource input name argument + std::string input_name; + if (nix_get_string(context, args[0], OBSERVE_STRING(input_name)) != NIX_OK) + return; + + // Only handle "vm_id" input - throw for anything else + if (input_name != "vm_id") { + std::string error_msg = "unknown resource input: " + input_name; + nix_set_err_msg(context, NIX_ERR_NIX_ERROR, error_msg.c_str()); + return; + } + + if (resource_state->vm_created) { + // VM has been created, return the ID + nix_init_string(context, ret, "vm-12345"); + } else { + // VM not created yet, fail with dependency error + nix_set_err_msg(context, NIX_ERR_RECOVERABLE, "VM not yet created"); + } +} + +#if 0 +TEST_F(nix_api_expr_test, nix_expr_thunk_re_evaluation_after_deployment) +{ + // This test demonstrates NixOps4's requirement: a thunk calling a primop should be + // re-evaluable when deployment resources become available that were not available initially. + + DeploymentResourceState resource_state; + + PrimOp * primop = nix_alloc_primop( + ctx, + primop_load_resource_input, + 1, + "loadResourceInput", + nullptr, + "load a deployment resource input", + &resource_state); + assert_ctx_ok(); + + nix_value * primopValue = nix_alloc_value(ctx, state); + assert_ctx_ok(); + nix_init_primop(ctx, primopValue, primop); + assert_ctx_ok(); + + nix_value * inputName = nix_alloc_value(ctx, state); + assert_ctx_ok(); + nix_init_string(ctx, inputName, "vm_id"); + assert_ctx_ok(); + + // Create a single thunk by using nix_init_apply instead of nix_value_call + // This creates a lazy application that can be forced multiple times + nix_value * thunk = nix_alloc_value(ctx, state); + assert_ctx_ok(); + nix_init_apply(ctx, thunk, primopValue, inputName); + assert_ctx_ok(); + + // First force: VM not created yet, should fail + nix_value_force(ctx, state, thunk); + ASSERT_EQ(NIX_ERR_NIX_ERROR, nix_err_code(ctx)); + ASSERT_THAT(nix_err_msg(nullptr, ctx, nullptr), testing::HasSubstr("VM not yet created")); + + // Clear the error context for the next attempt + nix_c_context_free(ctx); + ctx = nix_c_context_create(); + + // Simulate deployment process: VM gets created + resource_state.vm_created = true; + + // Second force of the SAME thunk: this is where the "failed" value issue appears + // With failed value caching, this should fail because the thunk is marked as permanently failed + // Without failed value caching (or with retryable failures), this should succeed + nix_value_force(ctx, state, thunk); + + // If we get here without error, the thunk was successfully re-evaluated + assert_ctx_ok(); + + std::string result; + nix_get_string(ctx, thunk, OBSERVE_STRING(result)); + assert_ctx_ok(); + ASSERT_STREQ("vm-12345", result.c_str()); +} +#endif + } // namespace nixC diff --git a/src/libexpr-tests/nix_api_external.cc b/src/libexpr-tests/nix_api_external.cc index ec19f1212e9e..885b9c1d2dd3 100644 --- a/src/libexpr-tests/nix_api_external.cc +++ b/src/libexpr-tests/nix_api_external.cc @@ -66,4 +66,44 @@ TEST_F(nix_api_expr_test, nix_expr_eval_external) nix_state_free(stateFn); } +static void print_value_as_json_using_state( + void * self, EvalState * state, bool strict, nix_string_context * c, bool copyToStore, nix_string_return * res) +{ + // Regression test: same cast bug as in nix_c_primop_wrapper (see primop_alloc_value). + nix_value * v = nix_alloc_value(nullptr, state); + assert(v != nullptr); + nix_gc_decref(nullptr, v); + + nix_set_string_return(res, "42"); +} + +TEST_F(nix_api_expr_test, nix_external_printValueAsJSON_can_use_state) +{ + NixCExternalValueDesc desc{}; + desc.print = [](void *, nix_printer *) {}; + desc.showType = [](void *, nix_string_return *) {}; + desc.typeOf = [](void *, nix_string_return *) {}; + desc.printValueAsJSON = print_value_as_json_using_state; + + ExternalValue * val = nix_create_external_value(ctx, &desc, nullptr); + assert_ctx_ok(); + nix_init_external(ctx, value, val); + assert_ctx_ok(); + + nix_value * toJsonFn = nix_alloc_value(ctx, state); + nix_expr_eval_from_string(ctx, state, "builtins.toJSON", ".", toJsonFn); + assert_ctx_ok(); + + nix_value * result = nix_alloc_value(ctx, state); + nix_value_call(ctx, state, toJsonFn, value, result); + assert_ctx_ok(); + + std::string json_str; + nix_get_string(ctx, result, OBSERVE_STRING(json_str)); + ASSERT_EQ("42", json_str); + + nix_gc_decref(ctx, result); + nix_gc_decref(ctx, toJsonFn); +} + } // namespace nixC diff --git a/src/libexpr-tests/package.nix b/src/libexpr-tests/package.nix index 51d52e935bf5..3af1f52d3fbf 100644 --- a/src/libexpr-tests/package.nix +++ b/src/libexpr-tests/package.nix @@ -33,7 +33,7 @@ mkMesonExecutable (finalAttrs: { ../../.version ./.version ./meson.build - # ./meson.options + ./meson.options (fileset.fileFilter (file: file.hasExt "cc") ./.) (fileset.fileFilter (file: file.hasExt "hh") ./.) ]; diff --git a/src/libexpr-tests/primops.cc b/src/libexpr-tests/primops.cc index 36e3fa598033..ac5e893bf7d3 100644 --- a/src/libexpr-tests/primops.cc +++ b/src/libexpr-tests/primops.cc @@ -756,7 +756,7 @@ TEST_F(PrimOpTest, langVersion) TEST_F(PrimOpTest, storeDir) { auto v = eval("builtins.storeDir"); - ASSERT_THAT(v, IsStringEq(settings.nixStore)); + ASSERT_THAT(v, IsStringEq(state.store->storeDir)); } TEST_F(PrimOpTest, nixVersion) diff --git a/src/libexpr-tests/regex-cache-bench.cc b/src/libexpr-tests/regex-cache-bench.cc new file mode 100644 index 000000000000..2eb17b212ab0 --- /dev/null +++ b/src/libexpr-tests/regex-cache-bench.cc @@ -0,0 +1,46 @@ +#include + +#include "nix/expr/eval.hh" +#include "nix/expr/eval-settings.hh" +#include "nix/fetchers/fetch-settings.hh" +#include "nix/store/store-open.hh" + +using namespace nix; + +static void BM_EvalManyBuiltinsMatchSameRegex(benchmark::State & state) +{ + static constexpr int iterations = 5'000; + + static constexpr std::string_view exprStr = + "builtins.foldl' " + "(acc: _: acc + builtins.length (builtins.match \"a\" \"a\")) " + "0 " + "(builtins.genList (x: x) " + "5000)"; + + for (auto _ : state) { + state.PauseTiming(); + + auto store = openStore("dummy://"); + fetchers::Settings fetchSettings{}; + bool readOnlyMode = true; + EvalSettings evalSettings{readOnlyMode}; + evalSettings.nixPath = {}; + + auto stPtr = std::make_shared(LookupPath{}, store, fetchSettings, evalSettings, nullptr); + auto & st = *stPtr; + Expr * expr = st.parseExprFromString(std::string(exprStr), st.rootPath(CanonPath::root)); + + Value v; + + state.ResumeTiming(); + + st.eval(expr, v); + st.forceValue(v, noPos); + benchmark::DoNotOptimize(v); + } + + state.SetItemsProcessed(state.iterations() * iterations); +} + +BENCHMARK(BM_EvalManyBuiltinsMatchSameRegex); diff --git a/src/libexpr-tests/value/print.cc b/src/libexpr-tests/value/print.cc index d226062197da..c9cabf3fa421 100644 --- a/src/libexpr-tests/value/print.cc +++ b/src/libexpr-tests/value/print.cc @@ -127,7 +127,7 @@ TEST_F(ValuePrintingTests, vLambda) TEST_F(ValuePrintingTests, vPrimOp) { Value vPrimOp; - PrimOp primOp{.name = "puppy"}; + PrimOp primOp{.name = "puppy", .impl = [](EvalState &, const PosIdx, Value **, Value &) {}}; vPrimOp.mkPrimOp(&primOp); test(vPrimOp, "«primop puppy»"); @@ -135,7 +135,7 @@ TEST_F(ValuePrintingTests, vPrimOp) TEST_F(ValuePrintingTests, vPrimOpApp) { - PrimOp primOp{.name = "puppy"}; + PrimOp primOp{.name = "puppy", .impl = [](EvalState &, const PosIdx, Value **, Value &) {}}; Value vPrimOp; vPrimOp.mkPrimOp(&primOp); @@ -188,6 +188,22 @@ TEST_F(ValuePrintingTests, vBlackhole) test(vBlackhole, "«potential infinite recursion»"); } +TEST_F(ValuePrintingTests, vFailed) +{ + Value v; + try { + throw Error("nope"); + } catch (...) { + v.mkFailed(); + } + + // Historically, a tried and then ignored value (e.g. through tryEval) was + // reverted to the original thunk. + + test(v, "«failed»"); + test(v, ANSI_MAGENTA "«failed»" ANSI_NORMAL, PrintOptions{.ansiColors = true}); +} + TEST_F(ValuePrintingTests, depthAttrs) { Value vOne; @@ -515,7 +531,7 @@ TEST_F(ValuePrintingTests, ansiColorsLambda) TEST_F(ValuePrintingTests, ansiColorsPrimOp) { - PrimOp primOp{.name = "puppy"}; + PrimOp primOp{.name = "puppy", .impl = [](EvalState &, const PosIdx, Value **, Value &) {}}; Value v; v.mkPrimOp(&primOp); @@ -524,7 +540,7 @@ TEST_F(ValuePrintingTests, ansiColorsPrimOp) TEST_F(ValuePrintingTests, ansiColorsPrimOpApp) { - PrimOp primOp{.name = "puppy"}; + PrimOp primOp{.name = "puppy", .impl = [](EvalState &, const PosIdx, Value **, Value &) {}}; Value vPrimOp; vPrimOp.mkPrimOp(&primOp); diff --git a/src/libexpr-tests/value/value.cc b/src/libexpr-tests/value/value.cc index bd8f0da71213..e22e70f6e336 100644 --- a/src/libexpr-tests/value/value.cc +++ b/src/libexpr-tests/value/value.cc @@ -13,7 +13,6 @@ TEST_F(ValueTest, unsetValue) { Value unsetValue; ASSERT_EQ(false, unsetValue.isValid()); - ASSERT_DEATH(unsetValue.type(), ""); } TEST_F(ValueTest, vInt) diff --git a/src/libexpr/attr-path.cc b/src/libexpr/attr-path.cc index c8b800245881..63197e4634fd 100644 --- a/src/libexpr/attr-path.cc +++ b/src/libexpr/attr-path.cc @@ -1,5 +1,6 @@ #include "nix/expr/attr-path.hh" #include "nix/expr/eval-inline.hh" +#include "nix/util/util.hh" #include "nix/util/strings-inline.hh" namespace nix { @@ -145,16 +146,15 @@ std::pair findPackageFilename(EvalState & state, Value & v auto fail = [fn]() { throw ParseError("cannot parse 'meta.position' attribute '%s'", fn); }; - try { - auto colon = fn.rfind(':'); - if (colon == std::string::npos) - fail(); - auto lineno = std::stoi(std::string(fn, colon + 1, std::string::npos)); - return {SourcePath{path.accessor, CanonPath(fn.substr(0, colon))}, lineno}; - } catch (std::invalid_argument & e) { + auto colon = fn.rfind(':'); + if (colon == std::string::npos) fail(); - unreachable(); - } + + auto lineno = string2Int(std::string_view(fn).substr(colon + 1)); + if (!lineno) + fail(); + + return {SourcePath{path.accessor, CanonPath(fn.substr(0, colon))}, *lineno}; } } // namespace nix diff --git a/src/libexpr/diagnose.cc b/src/libexpr/diagnose.cc new file mode 100644 index 000000000000..0013281abefa --- /dev/null +++ b/src/libexpr/diagnose.cc @@ -0,0 +1,55 @@ +#include "nix/expr/diagnose.hh" +#include "nix/util/configuration.hh" +#include "nix/util/config-impl.hh" +#include "nix/util/abstract-setting-to-json.hh" + +#include + +namespace nix { + +template<> +Diagnose BaseSetting::parse(const std::string & str) const +{ + if (str == "ignore") + return Diagnose::Ignore; + else if (str == "warn") + return Diagnose::Warn; + else if (str == "fatal") + return Diagnose::Fatal; + else + throw UsageError("option '%s' has invalid value '%s' (expected 'ignore', 'warn', or 'fatal')", name, str); +} + +template<> +struct BaseSetting::trait +{ + static constexpr bool appendable = false; +}; + +template<> +std::string BaseSetting::to_string() const +{ + switch (value) { + case Diagnose::Ignore: + return "ignore"; + case Diagnose::Warn: + return "warn"; + case Diagnose::Fatal: + return "fatal"; + default: + unreachable(); + } +} + +NLOHMANN_JSON_SERIALIZE_ENUM( + Diagnose, + { + {Diagnose::Ignore, "ignore"}, + {Diagnose::Warn, "warn"}, + {Diagnose::Fatal, "fatal"}, + }); + +/* Explicit instantiation of templates */ +template class BaseSetting; + +} // namespace nix diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc index 0d6bbdaf4839..15d1e70eae70 100644 --- a/src/libexpr/eval-cache.cc +++ b/src/libexpr/eval-cache.cc @@ -11,7 +11,7 @@ namespace nix::eval_cache { CachedEvalError::CachedEvalError(ref cursor, Symbol attr) - : EvalError(cursor->root->state, "cached failure of attribute '%s'", cursor->getAttrPathStr(attr)) + : CloneableError(cursor->root->state, "cached failure of attribute '%s'", cursor->getAttrPathStr(attr)) , cursor(cursor) , attr(attr) { @@ -70,12 +70,12 @@ struct AttrDb { auto state(_state->lock()); - auto cacheDir = std::filesystem::path(getCacheDir()) / "eval-cache-v6"; + auto cacheDir = getCacheDir() / "eval-cache-v6"; createDirs(cacheDir); auto dbPath = cacheDir / (fingerprint.to_string(HashFormat::Base16, false) + ".sqlite"); - state->db = SQLite(dbPath); + state->db = SQLite(dbPath, {.useWAL = settings.useSQLiteWAL}); state->db.isCache(); state->db.exec(schema); @@ -265,7 +265,7 @@ struct AttrDb case AttrType::String: { NixStringContext context; if (!queryAttribute.isNull(3)) - for (auto & s : tokenizeString>(queryAttribute.getStr(3), ";")) + for (auto & s : tokenizeString>(queryAttribute.getStr(3), " ")) context.insert(NixStringContextElem::parse(s)); return {{rowId, string_t{queryAttribute.getStr(2), context}}}; } diff --git a/src/libexpr/eval-error.cc b/src/libexpr/eval-error.cc index 7f01747158c1..38a60883bca7 100644 --- a/src/libexpr/eval-error.cc +++ b/src/libexpr/eval-error.cc @@ -1,9 +1,16 @@ #include "nix/expr/eval-error.hh" #include "nix/expr/eval.hh" #include "nix/expr/value.hh" +#include "nix/store/store-api.hh" namespace nix { +InvalidPathError::InvalidPathError(EvalState & state, const StorePath & path) + : CloneableError(state, "path '%s' is not valid", path.to_string()) + , path{path} +{ +} + template EvalErrorBuilder & EvalErrorBuilder::withExitStatus(unsigned int exitStatus) { @@ -111,7 +118,9 @@ template class EvalErrorBuilder; template class EvalErrorBuilder; template class EvalErrorBuilder; template class EvalErrorBuilder; +template class EvalErrorBuilder; template class EvalErrorBuilder; template class EvalErrorBuilder; +template class EvalErrorBuilder; } // namespace nix diff --git a/src/libexpr/eval-profiler.cc b/src/libexpr/eval-profiler.cc index 21d2d3f8f4d9..9c7448d59076 100644 --- a/src/libexpr/eval-profiler.cc +++ b/src/libexpr/eval-profiler.cc @@ -133,13 +133,19 @@ class SampleStack : public EvalProfiler FrameInfo getPrimOpFrameInfo(const PrimOp & primOp, std::span args, PosIdx pos); public: - SampleStack(EvalState & state, std::filesystem::path profileFile, std::chrono::nanoseconds period) + SampleStack(EvalState & state, const std::filesystem::path & profileFile, std::chrono::nanoseconds period) : state(state) , sampleInterval(period) , profileFd([&]() { - AutoCloseFD fd = toDescriptor(open(profileFile.string().c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0660)); + auto fd = openNewFileForWrite( + profileFile, + 0660, + { + .truncateExisting = true, + .followSymlinksOnTruncate = true, /* FIXME: Probably shouldn't follow symlinks. */ + }); if (!fd) - throw SysError("opening file %s", profileFile); + throw SysError("opening file %s", PathFmt(profileFile)); return fd; }()) , posCache(state) diff --git a/src/libexpr/eval-settings.cc b/src/libexpr/eval-settings.cc index 27205864b8ba..daf8dbc452b3 100644 --- a/src/libexpr/eval-settings.cc +++ b/src/libexpr/eval-settings.cc @@ -1,4 +1,5 @@ #include "nix/util/users.hh" +#include "nix/util/logging.hh" #include "nix/store/globals.hh" #include "nix/store/profiles.hh" #include "nix/expr/eval.hh" @@ -6,6 +7,26 @@ namespace nix { +void DeprecatedWarnSetting::assign(const bool & v) +{ + value = v; + warn("'%s' is deprecated, use '%s = %s' instead", name, targetName, v ? "warn" : "ignore"); + if (!target.overridden) + target = v ? Diagnose::Warn : Diagnose::Ignore; +} + +void DeprecatedWarnSetting::appendOrSet(bool newValue, bool append) +{ + assert(!append); + assign(newValue); +} + +void DeprecatedWarnSetting::override(const bool & v) +{ + overridden = true; + assign(v); +} + /* Very hacky way to parse $NIX_PATH, which is colon-separated, but can contain URLs (e.g. "nixpkgs=https://bla...:foo=https://"). */ Strings EvalSettings::parseNixPath(const std::string & s) @@ -70,9 +91,10 @@ Strings EvalSettings::getDefaultNixPath() } }; - add(std::filesystem::path{getNixDefExpr()} / "channels"); - add(rootChannelsDir() / "nixpkgs", "nixpkgs"); - add(rootChannelsDir()); + add(getNixDefExpr() / "channels"); + auto profilesDirOpts = settings.getProfileDirsOptions(); + add(rootChannelsDir(profilesDirOpts) / "nixpkgs", "nixpkgs"); + add(rootChannelsDir(profilesDirOpts)); return res; } diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 1392ce38b0a6..743b18590f63 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -1,4 +1,5 @@ #include "nix/expr/eval.hh" +#include "nix/expr/eval-error.hh" #include "nix/expr/eval-settings.hh" #include "nix/expr/primops.hh" #include "nix/expr/print-options.hh" @@ -33,6 +34,7 @@ #include #include #include +#include #include #include #include @@ -163,7 +165,7 @@ std::string_view showType(ValueType type, bool withArticle) case nThunk: return WA("a", "thunk"); case nFailed: - return WA("a", "failure"); + return WA("an", "error"); } unreachable(); } @@ -249,10 +251,6 @@ static Symbol getName(const AttrName & name, EvalState & state, Env & env) static constexpr size_t BASE_ENV_SIZE = 128; EvalMemory::EvalMemory() -#if NIX_USE_BOEHMGC - : valueAllocCache(std::allocate_shared(traceable_allocator(), nullptr)) - , env1AllocCache(std::allocate_shared(traceable_allocator(), nullptr)) -#endif { assertGCInitialized(); } @@ -347,7 +345,6 @@ EvalState::EvalState( countCalls = getEnv("NIX_COUNT_CALLS").value_or("0") != "0"; static_assert(sizeof(Env) <= 16, "environment must be <= 16 bytes"); - static_assert(sizeof(Counter) == 64, "counters must be 64 bytes"); /* Construct the Nix expression search path. */ assert(lookupPath.elements.empty()); @@ -394,7 +391,7 @@ EvalState::EvalState( EvalState::~EvalState() {} -void EvalState::allowPathLegacy(const Path & path) +void EvalState::allowPathLegacy(const std::string & path) { if (auto rootFS2 = rootFS.dynamic_pointer_cast()) rootFS2->allowPrefix(CanonPath(path)); @@ -452,29 +449,36 @@ bool isAllowedURI(std::string_view uri, const Strings & allowedUris) return false; } -void EvalState::checkURI(const std::string & uri) +void EvalState::checkURI(const std::string & uri0) { if (!settings.restrictEval) return; - if (isAllowedURI(uri, settings.allowedUris.get())) + if (isAllowedURI(uri0, settings.allowedUris.get())) return; /* If the URI is a path, then check it against allowedPaths as well. */ - if (isAbsolute(uri)) { - if (auto rootFS2 = rootFS.dynamic_pointer_cast()) - rootFS2->checkAccess(CanonPath(uri)); - return; + { + std::filesystem::path path(uri0); + if (path.is_absolute()) { + if (auto rootFS2 = rootFS.dynamic_pointer_cast()) + rootFS2->checkAccess(CanonPath(path.string())); + return; + } } - if (hasPrefix(uri, "file://")) { - if (auto rootFS2 = rootFS.dynamic_pointer_cast()) - rootFS2->checkAccess(CanonPath(uri.substr(7))); - return; + try { + ParsedURL uri = parseURL(uri0); + if (uri.scheme == "file") { + if (auto rootFS2 = rootFS.dynamic_pointer_cast()) + rootFS2->checkAccess(CanonPath(urlPathToPath(uri.path).string())); + return; + } + } catch (BadURL &) { } - throw RestrictedPathError("access to URI '%s' is forbidden in restricted mode", uri); + throw RestrictedPathError("access to URI '%s' is forbidden in restricted mode", uri0); } Value * EvalState::addConstant(const std::string & name, Value & v, Constant info) @@ -773,6 +777,11 @@ class DebuggerGuard inDebugger = true; } + DebuggerGuard(DebuggerGuard &&) = delete; + DebuggerGuard(const DebuggerGuard &) = delete; + DebuggerGuard & operator=(DebuggerGuard &&) = delete; + DebuggerGuard & operator=(const DebuggerGuard &) = delete; + ~DebuggerGuard() { inDebugger = false; @@ -913,7 +922,7 @@ void Value::mkPath(const SourcePath & path, EvalMemory & mem) mkPath(&*path.accessor, StringData::make(mem, path.path.abs())); } -inline Value * EvalState::lookupVar(Env * env, const ExprVar & var, bool noEval) +[[gnu::always_inline]] inline Value * EvalState::lookupVar(Env * env, const ExprVar & var, bool noEval) { for (auto l = var.level; l; --l, env = env->up) ; @@ -931,11 +940,11 @@ inline Value * EvalState::lookupVar(Env * env, const ExprVar & var, bool noEval) while (1) { forceAttrs(*env->values[0], fromWith->pos, "while evaluating the first subexpression of a with expression"); if (auto j = env->values[0]->attrs()->get(var.name)) { - if (countCalls) + if (countCalls) [[unlikely]] attrSelects[j->pos]++; return j->value; } - if (!fromWith->parentWith) + if (!fromWith->parentWith) [[unlikely]] error("undefined variable '%1%'", symbols[var.name]) .atPos(var.pos) .withFrame(*env, var) @@ -1690,7 +1699,7 @@ void EvalState::callFunction(Value & fun, std::span args, Value & vRes, try { auto pos = vCur.determinePos(noPos); vCur.reset(); - fn->fun(*this, pos, args.data(), vCur); + fn->impl(*this, pos, args.data(), vCur); } catch (Error & e) { if (fn->addTrace) addErrorTrace(e, pos, "while calling the '%1%' builtin", fn->name); @@ -1743,7 +1752,7 @@ void EvalState::callFunction(Value & fun, std::span args, Value & vRes, // so the debugger allows to inspect the wrong parameters passed to the builtin. auto pos = vCur.determinePos(noPos); vCur.reset(); - fn->fun(*this, pos, vArgs, vCur); + fn->impl(*this, pos, vArgs, vCur); } catch (Error & e) { if (fn->addTrace) addErrorTrace(e, pos, "while calling the '%1%' builtin", fn->name); @@ -2174,9 +2183,6 @@ void ExprPos::eval(EvalState & state, Env & env, Value & v) state.mkPos(v, pos); } -// always force this to be separate, otherwise forceValue may inline it and take -// a massive perf hit -[[gnu::noinline]] void EvalState::tryFixupBlackHolePos(Value & v, PosIdx pos) { if (!v.isBlackhole()) @@ -2185,7 +2191,8 @@ void EvalState::tryFixupBlackHolePos(Value & v, PosIdx pos) try { std::rethrow_exception(e); } catch (InfiniteRecursionError & e) { - e.atPos(positions[pos]); + if (!e.hasPos()) + e.atPos(positions[pos]); } catch (...) { } } @@ -2361,13 +2368,15 @@ std::string_view EvalState::forceStringNoCtx(Value & v, const PosIdx pos, std::s if (v.context()) { NixStringContext context; copyContext(v, context); - if (hasContext(context)) + if (hasContext(context)) { + auto ctxElem = NixStringContextElem::parse((*v.context()->begin())->view()); error( "the string '%1%' is not allowed to refer to a store path (such as '%2%')", v.string_view(), - (*v.context()->begin())->view()) + ctxElem.display(*store)) .withTrace(pos, errorCtx) .debugThrow(); + } } return s; } @@ -2414,6 +2423,8 @@ BackedStringView EvalState::coerceToString( bool copyToStore, bool canonicalizePath) { + auto _level = addCallDepth(pos); + forceValue(v, pos); if (v.type() == nString) { @@ -2434,7 +2445,10 @@ BackedStringView EvalState::coerceToString( } else { auto path = v.path(); if (path.accessor == rootFS && store->isInStore(path.path.abs())) { - context.insert(NixStringContextElem::Path{.storePath = store->toStorePath(path.path.abs()).first}); + try { + context.insert(NixStringContextElem::Path{.storePath = store->toStorePath(path.path.abs()).first}); + } catch (Error &) { + } } return std::string(path.path.abs()); } @@ -2645,6 +2659,8 @@ SingleDerivedPath EvalState::coerceToSingleDerivedPath(const PosIdx pos, Value & // `assert a == b; x` are critical for our users' testing UX. void EvalState::assertEqValues(Value & v1, Value & v2, const PosIdx pos, std::string_view errorCtx) { + auto _level = addCallDepth(pos); + // This implementation must match eqValues. forceValue(v1, pos); forceValue(v2, pos); @@ -2854,6 +2870,8 @@ void EvalState::assertEqValues(Value & v1, Value & v2, const PosIdx pos, std::st // This implementation must match assertEqValues bool EvalState::eqValues(Value & v1, Value & v2, const PosIdx pos, std::string_view errorCtx) { + auto _level = addCallDepth(pos); + forceValue(v1, pos); forceValue(v2, pos); @@ -3162,6 +3180,21 @@ Expr * EvalState::parseExprFromString(std::string s, const SourcePath & basePath return parseExprFromString(std::move(s), basePath, staticBaseEnv); } +ExprAttrs * +EvalState::parseReplBindings(std::string s_, const SourcePath & basePath, const std::shared_ptr & staticEnv) +{ + return parseReplBindings(s_, s_, basePath, staticEnv); +} + +ExprAttrs * EvalState::parseReplBindings( + std::string s_, std::string errorSource, const SourcePath & basePath, const std::shared_ptr & staticEnv) +{ + auto s = make_ref(std::move(errorSource)); + // flex requires two NUL terminators for yy_scan_buffer + s_.append("\0\0", 2); + return parseReplBindings(s_.data(), s_.size(), Pos::String{.source = s}, basePath, staticEnv); +} + Expr * EvalState::parseStdin() { // NOTE this method (and parseExprFromString) must take care to *fully copy* their @@ -3303,6 +3336,30 @@ Expr * EvalState::parse( return result; } +ExprAttrs * EvalState::parseReplBindings( + char * text, + size_t length, + Pos::Origin origin, + const SourcePath & basePath, + const std::shared_ptr & staticEnv) +{ + DocCommentMap tmpDocComments; + DocCommentMap * docComments = &tmpDocComments; + + if (auto sourcePath = std::get_if(&origin)) { + auto [it, _] = positionToDocComment.lock()->try_emplace(*sourcePath, make_ref()); + docComments = &*it->second; + } + + auto bindings = parseReplBindingsFromBuf( + text, length, origin, basePath, mem.exprs, symbols, settings, positions, *docComments, rootFS); + assert(bindings); + + bindings->bindVars(*this, staticEnv); + + return bindings; +} + DocComment EvalState::getDocCommentForPos(PosIdx pos) { auto pos2 = positions[pos]; diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc index c4a2b00af3ed..03a1aa455ce0 100644 --- a/src/libexpr/get-drvs.cc +++ b/src/libexpr/get-drvs.cc @@ -101,7 +101,7 @@ StorePath PackageInfo::queryOutPath() const i->pos, *i->value, context, "while evaluating the output path of a derivation"); } if (!outPath) - throw UnimplementedError("CA derivations are not yet supported"); + throw Error("derivation does not have attribute 'outPath'"); return *outPath; } @@ -213,6 +213,8 @@ StringSet PackageInfo::queryMetaNames() bool PackageInfo::checkMeta(Value & v) { + auto _level = state->addCallDepth(v.determinePos(noPos)); + state->forceValue(v, v.determinePos(noPos)); if (v.type() == nList) { for (auto elem : v.listView()) @@ -367,7 +369,26 @@ static std::string addToPath(const std::string & s1, std::string_view s2) return s1.empty() ? std::string(s2) : s1 + "." + s2; } -static std::regex attrRegex("[A-Za-z_][A-Za-z0-9-_+]*"); +static bool isAttrPathComponent(std::string_view symbol) +{ + if (symbol.empty()) + return false; + + /* [A-Za-z_] */ + unsigned char first = symbol[0]; + if (!((first >= 'A' && first <= 'Z') || (first >= 'a' && first <= 'z') || first == '_')) + return false; + + /* [A-Za-z0-9-_+]* */ + for (unsigned char c : symbol.substr(1)) { + if ((c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') || c == '-' || c == '_' + || c == '+') + continue; + return false; + } + + return true; +} static void getDerivations( EvalState & state, @@ -378,6 +399,8 @@ static void getDerivations( Done & done, bool ignoreAssertionFailures) { + auto _level = state.addCallDepth(vIn.determinePos(noPos)); + Value v; state.autoCallFunction(autoArgs, vIn, v); @@ -400,7 +423,7 @@ static void getDerivations( std::string_view symbol{state.symbols[i->name]}; try { debug("evaluating attribute '%1%'", symbol); - if (!std::regex_match(symbol.begin(), symbol.end(), attrRegex)) + if (!isAttrPathComponent(symbol)) continue; std::string pathPrefix2 = addToPath(pathPrefix, symbol); if (combineChannels) diff --git a/src/libexpr/include/nix/expr/attr-set.hh b/src/libexpr/include/nix/expr/attr-set.hh index f57302c428dd..4d3821feda97 100644 --- a/src/libexpr/include/nix/expr/attr-set.hh +++ b/src/libexpr/include/nix/expr/attr-set.hh @@ -107,6 +107,8 @@ private: Bindings & operator=(const Bindings &) = delete; Bindings & operator=(Bindings &&) = delete; + ~Bindings() = default; + friend class BindingsBuilder; /** diff --git a/src/libexpr/include/nix/expr/counter.hh b/src/libexpr/include/nix/expr/counter.hh index efbf23de349a..f679b23caeea 100644 --- a/src/libexpr/include/nix/expr/counter.hh +++ b/src/libexpr/include/nix/expr/counter.hh @@ -11,7 +11,7 @@ namespace nix { * variable is set. This is to prevent contention on these counters * when multi-threaded evaluation is enabled. */ -struct alignas(64) Counter +struct alignas(std::hardware_destructive_interference_size) Counter { using value_type = uint64_t; diff --git a/src/libexpr/include/nix/expr/diagnose.hh b/src/libexpr/include/nix/expr/diagnose.hh new file mode 100644 index 000000000000..8dfe052134a1 --- /dev/null +++ b/src/libexpr/include/nix/expr/diagnose.hh @@ -0,0 +1,76 @@ +#pragma once +///@file + +#include + +#include "nix/util/ansicolor.hh" +#include "nix/util/configuration.hh" +#include "nix/util/error.hh" +#include "nix/util/logging.hh" + +namespace nix { + +/** + * Diagnostic level for deprecated or non-portable language features. + */ +enum struct Diagnose { + /** + * Ignore the feature without any diagnostic. + */ + Ignore, + /** + * Warn when the feature is used, but allow it. + */ + Warn, + /** + * Treat the feature as a fatal error. + */ + Fatal, +}; + +template<> +Diagnose BaseSetting::parse(const std::string & str) const; + +template<> +std::string BaseSetting::to_string() const; + +/** + * Check a diagnostic setting and either do nothing, log a warning, or throw an error. + * + * The setting name is automatically appended to the error message. + * + * @param setting The diagnostic setting to check + * @param mkError A function that takes a bool (true if fatal, false if warning) and + * returns an optional error to throw (or warn with). + * Only called if level is not `Ignore`. + * If the function returns `std::nullopt`, no diagnostic is emitted. + * + * @throws The error returned by mkError if level is `Fatal` and mkError returns a value + */ +template +void diagnose(const Setting & setting, F && mkError) +{ + auto withError = [&](bool fatal, auto && handler) { + auto maybeError = mkError(fatal); + if (!maybeError) + return; + auto & info = maybeError->unsafeInfo(); + // Append the setting name to help users find the right setting + info.msg = HintFmt("%s (" ANSI_BOLD "%s" ANSI_NORMAL ")", Uncolored(info.msg.str()), setting.name); + maybeError->recalcWhat(); + handler(std::move(*maybeError)); + }; + + switch (setting.get()) { + case Diagnose::Ignore: + return; + case Diagnose::Warn: + withError(false, [](auto && error) { logWarning(error.info()); }); + return; + case Diagnose::Fatal: + withError(true, [](auto && error) { throw std::move(error); }); + return; + } +} + +} // namespace nix diff --git a/src/libexpr/include/nix/expr/eval-cache.hh b/src/libexpr/include/nix/expr/eval-cache.hh index 208eff52b44f..4fe7278ef07d 100644 --- a/src/libexpr/include/nix/expr/eval-cache.hh +++ b/src/libexpr/include/nix/expr/eval-cache.hh @@ -14,7 +14,7 @@ namespace nix::eval_cache { struct AttrDb; class AttrCursor; -struct CachedEvalError : EvalError +struct CachedEvalError : CloneableError { const ref cursor; const Symbol attr; @@ -42,7 +42,7 @@ public: std::function cleanupAttrPath = [](AttrPath && attrPath) { return std::move(attrPath); }; private: - typedef std::function RootLoader; + typedef fun RootLoader; RootLoader rootLoader; RootValue value; diff --git a/src/libexpr/include/nix/expr/eval-error.hh b/src/libexpr/include/nix/expr/eval-error.hh index 38db9b7069e0..68aa7b0643a2 100644 --- a/src/libexpr/include/nix/expr/eval-error.hh +++ b/src/libexpr/include/nix/expr/eval-error.hh @@ -2,6 +2,7 @@ #include "nix/util/error.hh" #include "nix/util/pos-idx.hh" +#include "nix/store/path.hh" namespace nix { @@ -18,7 +19,7 @@ class EvalErrorBuilder; * * Most subclasses should inherit from `EvalError` instead of this class. */ -class EvalBaseError : public Error +class EvalBaseError : public CloneableError { template friend class EvalErrorBuilder; @@ -26,14 +27,14 @@ public: EvalState & state; EvalBaseError(EvalState & state, ErrorInfo && errorInfo) - : Error(errorInfo) + : CloneableError(errorInfo) , state(state) { } template explicit EvalBaseError(EvalState & state, const std::string & formatString, const Args &... formatArgs) - : Error(formatString, formatArgs...) + : CloneableError(formatString, formatArgs...) , state(state) { } @@ -54,17 +55,36 @@ MakeError(TypeError, EvalError); MakeError(UndefinedVarError, EvalError); MakeError(MissingArgumentError, EvalError); MakeError(InfiniteRecursionError, EvalError); + +/** + * Resource exhaustion error when evaluation exceeds max-call-depth. + * Inherits from EvalBaseError (not EvalError) because resource exhaustion + * should not be cached. + */ +struct StackOverflowError : public CloneableError +{ + StackOverflowError(EvalState & state) + : CloneableError(state, "stack overflow; max-call-depth exceeded") + { + } +}; + MakeError(IFDError, EvalBaseError); -struct InvalidPathError : public EvalError +/** + * An evaluation error which should be retried instead of rethrown. + * + * A RecoverableEvalError is not an EvalError, because we shouldn't cache it in + * the eval cache, as it should be retried anyway. + */ +MakeError(RecoverableEvalError, EvalBaseError); + +struct InvalidPathError : public CloneableError { public: - Path path; + StorePath path; - InvalidPathError(EvalState & state, const Path & path) - : EvalError(state, "path '%s' is not valid", path) - { - } + InvalidPathError(EvalState & state, const StorePath & path); }; /** diff --git a/src/libexpr/include/nix/expr/eval-gc.hh b/src/libexpr/include/nix/expr/eval-gc.hh index 813c2920d0e2..1b1eb7e627f0 100644 --- a/src/libexpr/include/nix/expr/eval-gc.hh +++ b/src/libexpr/include/nix/expr/eval-gc.hh @@ -33,6 +33,9 @@ using gc_allocator = std::allocator; struct gc {}; +struct gc_cleanup +{}; + #endif namespace nix { diff --git a/src/libexpr/include/nix/expr/eval-inline.hh b/src/libexpr/include/nix/expr/eval-inline.hh index 35b549261573..7e34029375e8 100644 --- a/src/libexpr/include/nix/expr/eval-inline.hh +++ b/src/libexpr/include/nix/expr/eval-inline.hh @@ -5,6 +5,7 @@ #include "nix/expr/eval.hh" #include "nix/expr/eval-error.hh" #include "nix/expr/eval-settings.hh" +#include namespace nix { @@ -29,13 +30,15 @@ inline void * EvalMemory::allocBytes(size_t n) Value * EvalMemory::allocValue() { #if NIX_USE_BOEHMGC + /* Allocation cache for GC'd Value objects. Boehm GC is already a global resource, so thread_local is + a natural solution. Multiple EvalState instances on the same thread will reuse the same cache. */ + static thread_local std::shared_ptr valueAllocCache{ + std::allocate_shared(traceable_allocator(), nullptr)}; + /* We use the boehm batch allocator to speed up allocations of Values (of which there are many). GC_malloc_many returns a linked list of objects of the given size, where the first word of each object is also the pointer to the next object in the list. This also means that we have to explicitly clear the first word of every object we take. */ - thread_local static std::shared_ptr valueAllocCache{ - std::allocate_shared(traceable_allocator(), nullptr)}; - if (!*valueAllocCache) { *valueAllocCache = GC_malloc_many(sizeof(Value)); if (!*valueAllocCache) @@ -65,10 +68,11 @@ Env & EvalMemory::allocEnv(size_t size) #if NIX_USE_BOEHMGC if (size == 1) { - /* see allocValue for explanations. */ - thread_local static std::shared_ptr env1AllocCache{ + /* Allocation cache for size-1 Env objects. Boehm GC is already a global resource, so thread_local is + a natural solution. Multiple EvalState instances on the same thread will reuse the same cache. */ + static thread_local std::shared_ptr env1AllocCache{ std::allocate_shared(traceable_allocator(), nullptr)}; - + /* see allocValue for explanations. */ if (!*env1AllocCache) { *env1AllocCache = GC_malloc_many(sizeof(Env) + sizeof(Value *)); if (!*env1AllocCache) @@ -186,7 +190,7 @@ inline void EvalState::forceList(Value & v, const PosIdx pos, std::string_view e inline CallDepth EvalState::addCallDepth(const PosIdx pos) { if (callDepth > settings.maxCallDepth) - error("stack overflow; max-call-depth exceeded").atPos(pos).debugThrow(); + error().atPos(pos).debugThrow(); return CallDepth(callDepth); }; diff --git a/src/libexpr/include/nix/expr/eval-settings.hh b/src/libexpr/include/nix/expr/eval-settings.hh index f367541ec2f6..b58f0cd45943 100644 --- a/src/libexpr/include/nix/expr/eval-settings.hh +++ b/src/libexpr/include/nix/expr/eval-settings.hh @@ -1,6 +1,7 @@ #pragma once ///@file +#include "nix/expr/diagnose.hh" #include "nix/expr/eval-profiler-settings.hh" #include "nix/util/configuration.hh" #include "nix/util/source-path.hh" @@ -10,6 +11,36 @@ namespace nix { class EvalState; struct PrimOp; +/** + * A deprecated bool setting that migrates to a `Setting`. + * When set to true, it emits a deprecation warning and sets the target + * `Setting` setting to `Warn`. + */ +class DeprecatedWarnSetting : public BaseSetting +{ + Setting & target; + const char * targetName; + +public: + DeprecatedWarnSetting( + Config * options, + Setting & target, + const char * targetName, + const std::string & name, + const std::string & description, + const StringSet & aliases = {}) + : BaseSetting(false, true, name, description, aliases, std::nullopt) + , target(target) + , targetName(targetName) + { + options->addSetting(this); + } + + void assign(const bool & v) override; + void appendOrSet(bool newValue, bool append) override; + void override(const bool & v) override; +}; + struct EvalSettings : Config { /** @@ -36,7 +67,7 @@ struct EvalSettings : Config * if `` is a key in this map, then `` is * passed to the hook that is the value in this map. */ - using LookupPathHooks = std::map>; + using LookupPathHooks = std::map>; EvalSettings(bool & readOnlyMode, LookupPathHooks lookupPathHooks = {}); @@ -237,7 +268,7 @@ struct EvalSettings : Config See [Using the `eval-profiler`](@docroot@/advanced-topics/eval-profiler.md). )"}; - Setting evalProfileFile{ + Setting evalProfileFile{ this, "nix.profile", "eval-profile-file", @@ -328,20 +359,94 @@ struct EvalSettings : Config This option can be enabled by setting `NIX_ABORT_ON_WARN=1` in the environment. )"}; - Setting warnShortPathLiterals{ + Setting lintShortPathLiterals{ this, - false, - "warn-short-path-literals", + Diagnose::Ignore, + "lint-short-path-literals", R"( - If set to true, the Nix evaluator will warn when encountering relative path literals - that don't start with `./` or `../`. + Controls handling of relative path literals that don't start with `./` or `../`. - For example, with this setting enabled, `foo/bar` would emit a warning - suggesting to use `./foo/bar` instead. + - `ignore`: Ignore without warning (default) + - `warn`: Emit a warning suggesting to use `./` prefix + - `fatal`: Treat as a parse error + + For example, with this setting set to `warn` or `fatal`, `foo/bar` would + suggest using `./foo/bar` instead. This is useful for improving code readability and making path literals more explicit. - )"}; + )", + }; + + DeprecatedWarnSetting warnShortPathLiterals{ + this, + lintShortPathLiterals, + "lint-short-path-literals", + "warn-short-path-literals", + R"( + Deprecated. Use [`lint-short-path-literals`](#conf-lint-short-path-literals)` = warn` instead. + )", + }; + + Setting lintAbsolutePathLiterals{ + this, + Diagnose::Ignore, + "lint-absolute-path-literals", + R"( + Controls handling of absolute path literals (paths starting with `/`) and home path literals (paths starting with `~/`). + + - `ignore`: Ignore without warning (default) + - `warn`: Emit a warning about non-portability + - `fatal`: Treat as a parse error + + It is true that some files are more difficult to reference with relative paths, + because they would require lots of `../../..` upward traversing to reach them. + But firstly, it is probably not a good idea to reference these files --- + such paths often make Nix expressions less portable and reproducible, + as they depend on the file system layout of the machine evaluating the expression. + + Secondly, with [pure evaluation mode](#conf-pure-eval), most such files are prohibited to access anyway, + whether by absolute or relative paths. + In that case, enabling this lint in fatal mode is less disruptive, + because the paths pure eval allows are usually not the ones that would be ergonomically expressed with absolute paths anyway. + )", + }; + + Setting lintUrlLiterals{ + this, + Diagnose::Ignore, + "lint-url-literals", + R"( + Controls handling of unquoted URLs as part of the Nix language syntax. + The Nix language allows for URL literals, like so: + + ``` + $ nix repl + nix-repl> http://foo + "http://foo" + ``` + + Setting this to `warn` or `fatal` will cause the Nix parser to + warn or throw an error when encountering a URL literal: + + ``` + $ nix repl --lint-url-literals fatal + nix-repl> http://foo + error: URL literal 'http://foo' is deprecated + at «string»:1:1: + + 1| http://foo + | ^ + ``` + + Unquoted URLs are being deprecated and their usage is discouraged. + + The reason is that, as opposed to path literals, URLs have no + special properties that distinguish them from regular strings, URLs + containing query parameters have to be quoted anyway, and unquoted URLs + may confuse external tooling. + )", + }; Setting bindingsUpdateLayerRhsSizeThreshold{ this, diff --git a/src/libexpr/include/nix/expr/eval.hh b/src/libexpr/include/nix/expr/eval.hh index 852ad0f7bcc9..49a4cac91239 100644 --- a/src/libexpr/include/nix/expr/eval.hh +++ b/src/libexpr/include/nix/expr/eval.hh @@ -123,7 +123,7 @@ struct PrimOp /** * Implementation of the primop. */ - std::function fun; + fun impl; /** * Optional experimental for this to be gated on. @@ -309,18 +309,6 @@ struct StaticEvalSymbols class EvalMemory { -#if NIX_USE_BOEHMGC - /** - * Allocation cache for GC'd Value objects. - */ - std::shared_ptr valueAllocCache; - - /** - * Allocation cache for size-1 Env objects. - */ - std::shared_ptr env1AllocCache; -#endif - public: struct Statistics { @@ -555,7 +543,7 @@ public: /** * Variant which accepts relative paths too. */ - SourcePath rootPath(PathView path); + SourcePath rootPath(std::string_view path); /** * Return a `SourcePath` that refers to `path` in the store. @@ -572,7 +560,7 @@ public: * Only for restrict eval: pure eval just whitelist store paths, * never arbitrary paths. */ - void allowPathLegacy(const Path & path); + void allowPathLegacy(const std::string & path); /** * Allow access to a store path. Note that this gets remapped to @@ -615,6 +603,18 @@ public: parseExprFromString(std::string s, const SourcePath & basePath, const std::shared_ptr & staticEnv); Expr * parseExprFromString(std::string s, const SourcePath & basePath); + /** + * Parse REPL bindings from the specified string. + * Returns ExprAttrs with bindings to add to scope. + */ + ExprAttrs * + parseReplBindings(std::string s, const SourcePath & basePath, const std::shared_ptr & staticEnv); + ExprAttrs * parseReplBindings( + std::string s, + std::string errorSource, + const SourcePath & basePath, + const std::shared_ptr & staticEnv); + Expr * parseStdin(); /** @@ -669,6 +669,8 @@ public: void tryFixupBlackHolePos(Value & v, PosIdx pos); +public: + /** * Force a value, then recursively force list elements and * attributes. @@ -900,6 +902,13 @@ private: const SourcePath & basePath, const std::shared_ptr & staticEnv); + ExprAttrs * parseReplBindings( + char * text, + size_t length, + Pos::Origin origin, + const SourcePath & basePath, + const std::shared_ptr & staticEnv); + /** * Current Nix call stack depth, used with `max-call-depth` * setting to throw stack overflow hopefully before we run out of diff --git a/src/libexpr/include/nix/expr/meson.build b/src/libexpr/include/nix/expr/meson.build index 5c707ed4bffe..3191fd2dc148 100644 --- a/src/libexpr/include/nix/expr/meson.build +++ b/src/libexpr/include/nix/expr/meson.build @@ -11,6 +11,7 @@ headers = [ config_pub_h ] + files( 'attr-path.hh', 'attr-set.hh', 'counter.hh', + 'diagnose.hh', 'eval-cache.hh', 'eval-error.hh', 'eval-gc.hh', diff --git a/src/libexpr/include/nix/expr/print-ambiguous.hh b/src/libexpr/include/nix/expr/print-ambiguous.hh index e64f7f9bf8d0..7e44a6b66ebc 100644 --- a/src/libexpr/include/nix/expr/print-ambiguous.hh +++ b/src/libexpr/include/nix/expr/print-ambiguous.hh @@ -5,6 +5,8 @@ namespace nix { +class EvalState; + /** * Print a value in the deprecated format used by `nix-instantiate --eval` and * `nix-env` (for manifests). @@ -15,6 +17,6 @@ namespace nix { * * See: https://github.com/NixOS/nix/issues/9730 */ -void printAmbiguous(EvalState & state, Value & v, std::ostream & str, std::set * seen, int depth); +void printAmbiguous(EvalState & state, Value & v, std::ostream & str, std::set * seen, size_t depth = 0); } // namespace nix diff --git a/src/libexpr/include/nix/expr/value.hh b/src/libexpr/include/nix/expr/value.hh index 10893347bd6e..f41f7f89af43 100644 --- a/src/libexpr/include/nix/expr/value.hh +++ b/src/libexpr/include/nix/expr/value.hh @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -171,7 +172,7 @@ public: virtual bool operator==(const ExternalValueBase & b) const noexcept; /** - * Print the value as JSON. Defaults to unconvertable, i.e. throws an error + * Print the value as JSON. Defaults to unconvertible, i.e. throws an error */ virtual nlohmann::json printValueAsJSON(EvalState & state, bool strict, NixStringContext & context, bool copyToStore = true) const; @@ -185,7 +186,7 @@ public: bool location, XMLWriter & doc, NixStringContext & context, - PathSet & drvsSeen, + StringSet & drvsSeen, const PosIdx pos) const; virtual ~ExternalValueBase() {}; @@ -201,8 +202,6 @@ public: Value ** elems; ListBuilder(EvalMemory & mem, size_t size); - // NOTE: Can be noexcept because we are just copying integral values and - // raw pointers. ListBuilder(ListBuilder && x) noexcept : size(x.size) , inlineElems{x.inlineElems[0], x.inlineElems[1]} @@ -210,6 +209,11 @@ public: { } + ListBuilder(const ListBuilder &) = delete; + ListBuilder & operator=(ListBuilder &&) = delete; + ListBuilder & operator=(const ListBuilder &) = delete; + ~ListBuilder() = default; + Value *& operator[](size_t n) { return elems[n]; @@ -573,8 +577,8 @@ inline constexpr bool useBitPackedValueStorage = (ptrSize == 8) && (__STDCPP_DEF * Packs discriminator bits into the pointer alignment niches. */ template -class alignas(16) ValueStorage>> - : public detail::ValueBase +class alignas(16) + ValueStorage>> : public detail::ValueBase { /* Needs a dependent type name in order for member functions (and * potentially ill-formed bit casts) to be SFINAE'd out. @@ -767,7 +771,7 @@ protected: case pdPath: return static_cast(tListN + (pd - pdListN)); [[unlikely]] default: - unreachable(); + nixUnreachableWhenHardened(); } } @@ -1202,7 +1206,7 @@ private: T getStorage() const noexcept { if (getInternalType() != detail::payloadTypeToInternalType) [[unlikely]] - unreachable(); + nixUnreachableWhenHardened(); T out; ValueStorage::getStorage(out); return out; @@ -1270,41 +1274,31 @@ public: */ inline ValueType type() const { - switch (getInternalType()) { - case tUninitialized: - break; - case tInt: - return nInt; - case tBool: - return nBool; - case tString: - return nString; - case tPath: - return nPath; - case tNull: - return nNull; - case tAttrs: - return nAttrs; - case tListSmall: - case tListN: - return nList; - case tLambda: - case tPrimOp: - case tPrimOpApp: - return nFunction; - case tExternal: - return nExternal; - case tFloat: - return nFloat; - case tFailed: - return nFailed; - case tThunk: - case tApp: - case tPending: - case tAwaited: - return nThunk; - } - unreachable(); + /* Explicit lookup table. switch() might compile down (and it does at least with GCC 14) + to a jump table. Let's help the compiler a bit here. */ + static constexpr auto table = [] { + std::array t{}; + t[tUninitialized] = nThunk; + t[tInt] = nInt; + t[tBool] = nBool; + t[tNull] = nNull; + t[tFloat] = nFloat; + t[tFailed] = nFailed; + t[tExternal] = nExternal; + t[tAttrs] = nAttrs; + t[tPrimOp] = nFunction; + t[tLambda] = nFunction; + t[tPrimOpApp] = nFunction; + t[tApp] = nThunk; + t[tThunk] = nThunk; + t[tListSmall] = nList; + t[tListN] = nList; + t[tString] = nString; + t[tPath] = nPath; + return t; + }(); + + return table[getInternalType()]; } /** diff --git a/src/libexpr/include/nix/expr/value/context.hh b/src/libexpr/include/nix/expr/value/context.hh index fa3d4e87c0f2..4973ce0a9d94 100644 --- a/src/libexpr/include/nix/expr/value/context.hh +++ b/src/libexpr/include/nix/expr/value/context.hh @@ -9,14 +9,14 @@ namespace nix { -class BadNixStringContextElem : public Error +class BadNixStringContextElem final : public CloneableError { public: std::string_view raw; template BadNixStringContextElem(std::string_view raw_, const Args &... args) - : Error("") + : CloneableError("") { raw = raw_; auto hf = HintFmt(args...); @@ -107,6 +107,14 @@ struct NixStringContextElem static NixStringContextElem parse(std::string_view s, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); std::string to_string() const; + + /** + * Render for use in error messages and other user-facing output. + * + * Uses store paths and `DerivedPath` syntax, unlike `to_string()` + * which uses the internal encoding. + */ + std::string display(const StoreDirConfig & store) const; }; /** diff --git a/src/libexpr/lexer.l b/src/libexpr/lexer.l index 810503bdc5bd..5bdb5335b841 100644 --- a/src/libexpr/lexer.l +++ b/src/libexpr/lexer.l @@ -14,6 +14,7 @@ %x INPATH %x INPATH_SLASH %x PATH_START +%x REPL_BINDINGS_MODE %top { #include "parser-tab.hh" // YYSTYPE @@ -113,6 +114,13 @@ URI [a-zA-Z][a-zA-Z0-9\+\-\.]*\:[a-zA-Z0-9\%\/\?\:\@\&\=\+\$\,\-\_\.\!\~ %% + /* REPL bindings mode: inject REPL_BINDINGS token at start, then switch to normal lexing */ +.|\n { + yyless(0); + yylloc->unstash(); + POP_STATE(); + return REPL_BINDINGS; +} if { return IF; } then { return THEN; } @@ -339,3 +347,17 @@ or { return OR_KW; } } %% + +#include + +// Verify that the forward declaration in parser.y matches flex's definition +static_assert(std::is_same_v); + +namespace nix { + +void setReplBindingsMode(yyscan_t scanner) +{ + yy_push_state(REPL_BINDINGS_MODE, scanner); +} + +} diff --git a/src/libexpr/meson.build b/src/libexpr/meson.build index 941cb0a8a442..b99cbb6d6cc2 100644 --- a/src/libexpr/meson.build +++ b/src/libexpr/meson.build @@ -30,7 +30,7 @@ deps_public_maybe_subproject = [ subdir('nix-meson-build-support/subprojects') subdir('nix-meson-build-support/big-objs') -# Check for each of these functions, and create a define like `#define HAVE_LCHOWN 1`. +# Check for each of these functions, and create a define like `#define HAVE_SYSCONF 1`. check_funcs = [ 'sysconf', ] @@ -166,9 +166,11 @@ endforeach sources = files( 'attr-path.cc', 'attr-set.cc', + 'diagnose.cc', 'eval-cache.cc', 'eval-error.cc', 'eval-gc.cc', + 'eval-profiler-settings.cc', 'eval-profiler.cc', 'eval-settings.cc', 'eval.cc', @@ -228,8 +230,7 @@ parser_library = static_library( 'nixexpr-parser', parser_tab, lexer_tab, - # Putting eval-profiler-settings.cc here to work around an inscrutable gcc compiler error when doing a unity build. - files('eval-profiler-settings.cc', 'lexer-helpers.cc'), + files('lexer-helpers.cc'), cpp_args : parser_library_cpp_args, dependencies : deps_public + deps_private + deps_other, include_directories : include_dirs, diff --git a/src/libexpr/package.nix b/src/libexpr/package.nix index 46c617bbcb19..6eccc0645132 100644 --- a/src/libexpr/package.nix +++ b/src/libexpr/package.nix @@ -91,7 +91,7 @@ mkMesonLibrary (finalAttrs: { # For some reason that is not clear, it is wanting to use libgcc_eh which is not available. # Force this to be built with compiler-rt over libgcc_eh works. # Issue: https://github.com/NixOS/nixpkgs/issues/177129 - NIX_CFLAGS_COMPILE = lib.optional ( + NIX_CFLAGS_COMPILE = lib.optionalString ( stdenv.cc.isClang && stdenv.hostPlatform.isStatic && stdenv.cc.libcxx != null diff --git a/src/libexpr/parallel-eval.cc b/src/libexpr/parallel-eval.cc index 0fe7820454b8..1eef13957ef5 100644 --- a/src/libexpr/parallel-eval.cc +++ b/src/libexpr/parallel-eval.cc @@ -295,7 +295,7 @@ static RegisterPrimOp r_parallel({ .doc = R"( Start evaluation of the values `xs` in the background and return `x`. )", - .fun = prim_parallel, + .impl = prim_parallel, .experimentalFeature = Xp::ParallelEval, }); diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index c9ad30407715..e5e4241ea7b0 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -3,7 +3,7 @@ %define api.namespace { ::nix::parser } %define api.parser.class { BisonParser } %locations -%define parse.error verbose +%define parse.error detailed %defines /* %no-lines */ %parse-param { void * scanner } @@ -30,6 +30,7 @@ #include "nix/expr/nixexpr.hh" #include "nix/expr/eval.hh" #include "nix/expr/eval-settings.hh" +#include "nix/expr/diagnose.hh" #include "nix/expr/parser-state.hh" #define YY_DECL \ @@ -54,6 +55,11 @@ } \ while (0) +// Forward declaration for flex scanner type. +// lexer-tab.hh is large; avoid including it just for this type. +// Asserted with static_assert in lexer.l. +typedef void * yyscan_t; + namespace nix { typedef boost::unordered_flat_map> DocCommentMap; @@ -70,6 +76,28 @@ Expr * parseExprFromBuf( DocCommentMap & docComments, const ref rootFS); +/** + * Puts the lexer in REPL bindings mode before the first token. This causes + * the parser to accept REPL bindings (attribute definitions). + */ +void setReplBindingsMode(yyscan_t scanner); + +/** + * Parse REPL bindings from a buffer. + * Returns ExprAttrs with bindings to add to scope. + */ +ExprAttrs * parseReplBindingsFromBuf( + char * text, + size_t length, + Pos::Origin origin, + const SourcePath & basePath, + Exprs & exprs, + SymbolTable & symbols, + const EvalSettings & settings, + PosTable & positions, + DocCommentMap & docComments, + const ref rootFS); + } #endif @@ -140,18 +168,42 @@ static Expr * makeCall(Exprs & exprs, PosIdx pos, Expr * fn, Expr * arg) { %type path_start %type string_parts string_attr %type attr -%token ID -%token STR IND_STR -%token INT_LIT -%token FLOAT_LIT -%token PATH HPATH SPATH PATH_END -%token URI -%token IF THEN ELSE ASSERT WITH LET IN_KW REC INHERIT EQ NEQ AND OR IMPL OR_KW -%token PIPE_FROM PIPE_INTO /* <| and |> */ -%token DOLLAR_CURLY /* == ${ */ -%token IND_STRING_OPEN IND_STRING_CLOSE -%token ELLIPSIS - +%token ID "identifier" +%token STR "string" +%token IND_STR "indented string" +%token INT_LIT "integer" +%token FLOAT_LIT "floating-point literal" +%token PATH "path" +%token HPATH "'~/…' path" +%token SPATH "'<…>' path" +%token PATH_END "end of path" +%token URI "URI" +%token IF "'if'" +%token THEN "'then'" +%token ELSE "'else'" +%token ASSERT "'assert'" +%token WITH "'with'" +%token LET "'let'" +%token IN_KW "'in'" +%token REC "'rec'" +%token INHERIT "'inherit'" +%token EQ "'=='" +%token NEQ "'!='" +%token LEQ "'<='" +%token GEQ "'>='" +%token UPDATE "'//'" +%token CONCAT "'++'" +%token AND "'&&'" +%token OR "'||'" +%token IMPL "'->'" +%token OR_KW "'or'" +%token PIPE_FROM "'<|'" +%token PIPE_INTO "'|>'" +%token DOLLAR_CURLY "'${'" +%token IND_STRING_OPEN "start of an indented string" +%token IND_STRING_CLOSE "end of an indented string" +%token ELLIPSIS "'...'" +%token REPL_BINDINGS "start of REPL bindings" %right IMPL %left OR @@ -173,6 +225,10 @@ start: expr { // This parser does not use yynerrs; suppress the warning. (void) yynerrs_; +} +| REPL_BINDINGS binds1 { + state->result = $2; + (void) yynerrs_; }; expr: expr_function; @@ -314,12 +370,14 @@ expr_simple state->exprs.add(state->exprs.alloc, path)}); } | URI { - static bool noURLLiterals = experimentalFeatureSettings.isEnabled(Xp::NoUrlLiterals); - if (noURLLiterals) - throw ParseError({ - .msg = HintFmt("URL literals are disabled"), + diagnose(state->settings.lintUrlLiterals, [&](bool fatal) -> std::optional { + return ParseError({ + .msg = HintFmt("URL literals are %s. Consider using a string literal \"%s\" instead", + fatal ? "disallowed" : "discouraged", + std::string_view($1.p, $1.l)), .pos = state->positions[CUR_POS] }); + }); $$ = state->exprs.add(state->exprs.alloc, $1); } | '(' expr ')' { $$ = $2; } @@ -357,35 +415,57 @@ path_start : PATH { std::string_view literal({$1.p, $1.l}); - /* check for short path literals */ - if (state->settings.warnShortPathLiterals && literal.front() != '/' && literal.front() != '.') { - logWarning({ - .msg = HintFmt("relative path literal '%s' should be prefixed with '.' for clarity: './%s'. (" ANSI_BOLD "warn-short-path-literals" ANSI_NORMAL " = true)", literal, literal), - .pos = state->positions[CUR_POS] + if (literal.front() == '/') { + diagnose(state->settings.lintAbsolutePathLiterals, [&](bool) -> std::optional { + return ParseError({ + .msg = HintFmt("absolute path literals are not portable. Consider replacing path literal '%s' by a string, relative path, or parameter", literal), + .pos = state->positions[CUR_POS] + }); }); - } - Path path(absPath(literal, state->basePath.path.abs())); - /* add back in the trailing '/' to the first segment */ - if (literal.size() > 1 && literal.back() == '/') - path += '/'; - $$ = /* Absolute paths are always interpreted relative to the root filesystem accessor, rather than the accessor of the current Nix expression. */ - literal.front() == '/' - ? state->exprs.add(state->exprs.alloc, state->rootFS, path) - : state->exprs.add(state->exprs.alloc, state->basePath.accessor, path); + auto path = canonPath(literal).string(); + /* add back in the trailing '/' to the first segment */ + if (literal.size() > 1 && literal.back() == '/') + path += '/'; + $$ = state->exprs.add(state->exprs.alloc, state->rootFS, path); + } else { + /* check for short path literals */ + diagnose(state->settings.lintShortPathLiterals, [&](bool) -> std::optional { + if (literal.front() != '.') + return ParseError({ + .msg = HintFmt("relative path literal '%s' should be prefixed with '.' for clarity: './%s'", literal, literal), + .pos = state->positions[CUR_POS] + }); + return std::nullopt; + }); + + auto basePath = std::filesystem::path(state->basePath.path.abs()); + auto path = absPath(literal, &basePath).string(); + /* add back in the trailing '/' to the first segment */ + if (literal.size() > 1 && literal.back() == '/') + path += '/'; + $$ = state->exprs.add(state->exprs.alloc, state->basePath.accessor, path); + } } | HPATH { + std::string_view literal($1.p, $1.l); if (state->settings.pureEval) { throw Error( "the path '%s' can not be resolved in pure mode", - std::string_view($1.p, $1.l) + literal ); } - Path path(getHome().string() + std::string($1.p + 1, $1.l - 1)); - $$ = state->exprs.add(state->exprs.alloc, ref(state->rootFS), path); + diagnose(state->settings.lintAbsolutePathLiterals, [&](bool) -> std::optional { + return ParseError({ + .msg = HintFmt("home path literals are not portable. Consider replacing path literal '%s' by a string, relative path, or parameter", literal), + .pos = state->positions[CUR_POS] + }); + }); + auto path(getHome().string() + std::string($1.p + 1, $1.l - 1)); + $$ = state->exprs.add(state->exprs.alloc, state->rootFS, path); } ; @@ -554,6 +634,51 @@ Expr * parseExprFromBuf( return state.result; } +ExprAttrs * parseReplBindingsFromBuf( + char * text, + size_t length, + Pos::Origin origin, + const SourcePath & basePath, + Exprs & exprs, + SymbolTable & symbols, + const EvalSettings & settings, + PosTable & positions, + DocCommentMap & docComments, + const ref rootFS) +{ + yyscan_t scanner; + LexerState lexerState { + .positionToDocComment = docComments, + .positions = positions, + .origin = positions.addOrigin(origin, length), + }; + ParserState state { + .lexerState = lexerState, + .exprs = exprs, + .symbols = symbols, + .positions = positions, + .basePath = basePath, + .origin = lexerState.origin, + .rootFS = rootFS, + .settings = settings, + }; + + yylex_init_extra(&lexerState, &scanner); + Finally _destroy([&] { yylex_destroy(scanner); }); + + yy_scan_buffer(text, length, scanner); + setReplBindingsMode(scanner); + Parser parser(scanner, &state); + parser.parse(); + + assert(state.result); + // state.result is Expr *, but the REPL_BINDINGS grammar rule + // always produces an ExprAttrs via the binds1 production. + auto bindings = dynamic_cast(state.result); + assert(bindings); + return bindings; +} + } #pragma GCC diagnostic pop // end ignored "-Wswitch-enum" diff --git a/src/libexpr/paths.cc b/src/libexpr/paths.cc index dd3e9bb3c9ed..00165b44cc4e 100644 --- a/src/libexpr/paths.cc +++ b/src/libexpr/paths.cc @@ -10,9 +10,9 @@ SourcePath EvalState::rootPath(CanonPath path) return {rootFS, std::move(path)}; } -SourcePath EvalState::rootPath(PathView path) +SourcePath EvalState::rootPath(std::string_view path) { - return {rootFS, CanonPath(absPath(path))}; + return {rootFS, CanonPath(absPath(path).string())}; } SourcePath EvalState::storePath(const StorePath & path) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index e404538a1a2c..a997c96c760f 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -11,6 +11,7 @@ #include "nix/store/path-references.hh" #include "nix/store/store-api.hh" #include "nix/util/util.hh" +#include "nix/util/os-string.hh" #include "nix/util/processes.hh" #include "nix/expr/value-to-json.hh" #include "nix/expr/value-to-xml.hh" @@ -78,7 +79,7 @@ StringMap EvalState::realiseContext(const NixStringContext & context, StorePathS auto ensureValid = [&](const StorePath & p) { waitForPath(p); if (!store->isValidPath(p)) - error(store->printStorePath(p)).debugThrow(); + error(p).debugThrow(); }; std::visit( overloaded{ @@ -366,7 +367,7 @@ static RegisterPrimOp primop_scopedImport( Evaluation aborts if the file doesn't exist or contains an invalid Nix expression. )", - .fun = [](EvalState & state, const PosIdx pos, Value ** args, Value & v) { + .impl = [](EvalState & state, const PosIdx pos, Value ** args, Value & v) { import(state, pos, *args[1], args[0], v); }}); @@ -441,7 +442,7 @@ static RegisterPrimOp primop_import( > > The function argument doesn’t have to be called `x` in `foo.nix`; any name would work. )", - .fun = [](EvalState & state, const PosIdx pos, Value ** args, Value & v) { + .impl = [](EvalState & state, const PosIdx pos, Value ** args, Value & v) { import(state, pos, *args[0], nullptr, v); }}); @@ -514,12 +515,12 @@ void prim_exec(EvalState & state, const PosIdx pos, Value ** args, Value & v) try { auto _ = state.realiseContext(context); // FIXME: Handle CA derivations } catch (InvalidPathError & e) { - state.error("cannot execute '%1%', since path '%2%' is not valid", program, e.path) + state.error("cannot execute '%1%', since store path '%2%' is not valid", program, e.path.to_string()) .atPos(pos) .debugThrow(); } - auto output = runProgram(program, true, commandArgs); + auto output = runProgram(program, true, toOsStrings(std::move(commandArgs))); Expr * parsed; try { parsed = state.parseExprFromString(std::move(output), state.rootPath(CanonPath::root)); @@ -586,7 +587,7 @@ static RegisterPrimOp primop_typeOf({ `"int"`, `"bool"`, `"string"`, `"path"`, `"null"`, `"set"`, `"list"`, `"lambda"` or `"float"`. )", - .fun = prim_typeOf, + .impl = prim_typeOf, }); /* Determine whether the argument is the null value. */ @@ -604,7 +605,7 @@ static RegisterPrimOp primop_isNull({ This is equivalent to `e == null`. )", - .fun = prim_isNull, + .impl = prim_isNull, }); /* Determine whether the argument is a function. */ @@ -620,7 +621,7 @@ static RegisterPrimOp primop_isFunction({ .doc = R"( Return `true` if *e* evaluates to a function, and `false` otherwise. )", - .fun = prim_isFunction, + .impl = prim_isFunction, }); /* Determine whether the argument is an integer. */ @@ -636,7 +637,7 @@ static RegisterPrimOp primop_isInt({ .doc = R"( Return `true` if *e* evaluates to an integer, and `false` otherwise. )", - .fun = prim_isInt, + .impl = prim_isInt, }); /* Determine whether the argument is a float. */ @@ -652,7 +653,7 @@ static RegisterPrimOp primop_isFloat({ .doc = R"( Return `true` if *e* evaluates to a float, and `false` otherwise. )", - .fun = prim_isFloat, + .impl = prim_isFloat, }); /* Determine whether the argument is a string. */ @@ -668,7 +669,7 @@ static RegisterPrimOp primop_isString({ .doc = R"( Return `true` if *e* evaluates to a string, and `false` otherwise. )", - .fun = prim_isString, + .impl = prim_isString, }); /* Determine whether the argument is a Boolean. */ @@ -684,7 +685,7 @@ static RegisterPrimOp primop_isBool({ .doc = R"( Return `true` if *e* evaluates to a bool, and `false` otherwise. )", - .fun = prim_isBool, + .impl = prim_isBool, }); /* Determine whether the argument is a path. */ @@ -700,7 +701,7 @@ static RegisterPrimOp primop_isPath({ .doc = R"( Return `true` if *e* evaluates to a path, and `false` otherwise. )", - .fun = prim_isPath, + .impl = prim_isPath, }); template @@ -963,7 +964,7 @@ static RegisterPrimOp primop_genericClosure( > [ { key = 5; } { key = 16; } { key = 8; } { key = 4; } { key = 2; } { key = 1; } ] > ``` )", - .fun = prim_genericClosure, + .impl = prim_genericClosure, }); static RegisterPrimOp primop_break( @@ -973,7 +974,7 @@ static RegisterPrimOp primop_break( In debug mode (enabled using `--debugger`), pause Nix expression evaluation and enter the REPL. Otherwise, return the argument `v`. )", - .fun = [](EvalState & state, const PosIdx pos, Value ** args, Value & v) { + .impl = [](EvalState & state, const PosIdx pos, Value ** args, Value & v) { if (state.canDebug()) { auto error = Error( ErrorInfo{ @@ -995,7 +996,7 @@ static RegisterPrimOp primop_abort( .doc = R"( Abort Nix expression evaluation and print the error message *s*. )", - .fun = [](EvalState & state, const PosIdx pos, Value ** args, Value & v) { + .impl = [](EvalState & state, const PosIdx pos, Value ** args, Value & v) { NixStringContext context; auto s = state.coerceToString(pos, *args[0], context, "while evaluating the error message passed to builtins.abort") @@ -1015,7 +1016,7 @@ static RegisterPrimOp primop_throw( derivations, a derivation that throws an error is silently skipped (which is not the case for `abort`). )", - .fun = [](EvalState & state, const PosIdx pos, Value ** args, Value & v) { + .impl = [](EvalState & state, const PosIdx pos, Value ** args, Value & v) { NixStringContext context; auto s = state.coerceToString(pos, *args[0], context, "while evaluating the error message passed to builtin.throw") @@ -1050,7 +1051,7 @@ static RegisterPrimOp primop_addErrorContext( .arity = 2, // The normal trace item is redundant .addTrace = false, - .fun = prim_addErrorContext, + .impl = prim_addErrorContext, }); static void prim_ceil(EvalState & state, const PosIdx pos, Value ** args, Value & v) @@ -1105,7 +1106,7 @@ static RegisterPrimOp primop_ceil({ If the datatype of *number* is neither a NixInt (signed 64-bit integer) nor a NixFloat (IEEE-754 double-precision floating-point number), an evaluation error is thrown. )", - .fun = prim_ceil, + .impl = prim_ceil, }); static void prim_floor(EvalState & state, const PosIdx pos, Value ** args, Value & v) @@ -1160,7 +1161,7 @@ static RegisterPrimOp primop_floor({ If the datatype of *number* is neither a NixInt (signed 64-bit integer) nor a NixFloat (IEEE-754 double-precision floating-point number), an evaluation error is thrown. )", - .fun = prim_floor, + .impl = prim_floor, }); /* Try evaluating the argument. Success => {success=true; value=something;}, @@ -1217,7 +1218,7 @@ static RegisterPrimOp primop_tryEval({ `tryEval` intentionally does not return the error message, because that risks bringing non-determinism into the evaluation result, and it would become very difficult to improve error reporting without breaking existing expressions. Instead, use [`builtins.addErrorContext`](@docroot@/language/builtins.md#builtins-addErrorContext) to add context to the error message, and use a Nix unit testing tool for testing. )", - .fun = prim_tryEval, + .impl = prim_tryEval, }); /* Return an environment variable. Use with care. */ @@ -1242,7 +1243,7 @@ static RegisterPrimOp primop_getEnv({ Packages. (That is, it does a `getEnv "HOME"` to locate the user’s home directory.) )", - .fun = prim_getEnv, + .impl = prim_getEnv, }); /* Evaluate the first argument, then return the second argument. */ @@ -1260,7 +1261,7 @@ static RegisterPrimOp primop_seq({ Evaluate *e1*, then evaluate and return *e2*. This ensures that a computation is strict in the value of *e1*. )", - .fun = prim_seq, + .impl = prim_seq, }); /* Evaluate the first argument deeply (i.e. recursing into lists and @@ -1280,7 +1281,7 @@ static RegisterPrimOp primop_deepSeq({ if it’s a list or set, its elements or attributes are also evaluated recursively. )", - .fun = prim_deepSeq, + .impl = prim_deepSeq, }); /* Evaluate the first expression and print it on standard error. Then @@ -1313,7 +1314,7 @@ static RegisterPrimOp primop_trace({ interactive debugger is started when `trace` is called (like [`break`](@docroot@/language/builtins.md#builtins-break)). )", - .fun = prim_trace, + .impl = prim_trace, }); static void prim_warn(EvalState & state, const PosIdx pos, Value ** args, Value & v) @@ -1324,11 +1325,12 @@ static void prim_warn(EvalState & state, const PosIdx pos, Value ** args, Value state.forceString(*args[0], pos, "while evaluating the first argument; the message passed to builtins.warn"); { - BaseError msg(std::string{msgStr}); - msg.atPos(state.positions[pos]); - auto info = msg.info(); - info.level = lvlWarn; - info.isFromExpr = true; + ErrorInfo info{ + .level = lvlWarn, + .msg = HintFmt(std::string(msgStr)), + .pos = state.positions[pos], + .isFromExpr = true, + }; logWarning(info); } @@ -1365,7 +1367,7 @@ static RegisterPrimOp primop_warn({ option is set, the evaluation is aborted after the warning is printed. This is useful to reveal the stack trace of the warning, when the context is non-interactive and a debugger can not be launched. )", - .fun = prim_warn, + .impl = prim_warn, }); /* Takes two arguments and evaluates to the second one. Used as the @@ -1445,7 +1447,7 @@ static RegisterPrimOp primop_derivationStrictWithMeta( PrimOp{ .name = "derivationStrictWithMeta", .arity = 1, - .fun = + .impl = [](EvalState & state, const PosIdx pos, Value ** args, Value & v) { prim_derivationStrictGeneric(state, pos, args, v, /*acceptMeta=*/true); }, @@ -1877,8 +1879,14 @@ static void derivationStrictInternal( drv.fillInOutputPaths(*state.store); } - /* Write the resulting term into the Nix store directory. */ - auto drvPath = writeDerivation(*state.store, *state.asyncPathWriter, drv, state.repair, false, provenance); + /* Write the resulting term into the Nix store directory. + + Unless we are in read-only mode, that is, in which case we do not + write anything. Users commonly do this to speed up evaluation in + contexts where they don't actually want to build anything. */ + auto drvPath = settings.readOnlyMode + ? computeStorePath(*state.store, drv) + : state.store->writeDerivation(*state.asyncPathWriter, drv, state.repair, provenance); auto drvPathS = state.store->printStorePath(drvPath); printMsg(lvlChatty, "instantiated '%1%' -> '%2%'", drvName, drvPathS); @@ -1916,7 +1924,7 @@ static RegisterPrimOp primop_derivationStrict( PrimOp{ .name = "derivationStrict", .arity = 1, - .fun = + .impl = [](EvalState & state, const PosIdx pos, Value ** args, Value & v) { prim_derivationStrictGeneric(state, pos, args, v, /*acceptMeta=*/false); }, @@ -1949,7 +1957,7 @@ static RegisterPrimOp primop_placeholder({ Typical outputs would be `"out"`, `"bin"` or `"dev"`. )", - .fun = prim_placeholder, + .impl = prim_placeholder, }); /************************************************************* @@ -1973,7 +1981,7 @@ static RegisterPrimOp primop_toPath({ **DEPRECATED.** Use `/. + "/path"` to convert a string into an absolute path. For relative paths, use `./. + "/path"`. )", - .fun = prim_toPath, + .impl = prim_toPath, }); /* Allow a valid store path to be used in an expression. This is @@ -1999,7 +2007,7 @@ static void prim_storePath(EvalState & state, const PosIdx pos, Value ** args, V directly in the store. The latter condition is necessary so e.g. nix-push does the right thing. */ if (!state.store->isStorePath(path.abs())) - path = CanonPath(canonPath(path.abs(), true)); + path = CanonPath(canonPath(path.abs(), true).string()); if (!state.store->isInStore(path.abs())) state.error("path '%1%' is not in the Nix store", path).atPos(pos).debugThrow(); auto path2 = state.store->toStorePath(path.abs()).first; @@ -2026,7 +2034,7 @@ static RegisterPrimOp primop_storePath({ See also [`builtins.fetchClosure`](#builtins-fetchClosure). )", - .fun = prim_storePath, + .impl = prim_storePath, }); static void prim_pathExists(EvalState & state, const PosIdx pos, Value ** args, Value & v) @@ -2057,7 +2065,7 @@ static RegisterPrimOp primop_pathExists({ Return `true` if the path *path* exists at evaluation time, and `false` otherwise. )", - .fun = prim_pathExists, + .impl = prim_pathExists, }); // Ideally, all trailing slashes should have been removed, but it's been like this for @@ -2107,7 +2115,7 @@ static RegisterPrimOp primop_baseNameOf({ This is somewhat similar to the [GNU `basename`](https://www.gnu.org/software/coreutils/manual/html_node/basename-invocation.html) command, but GNU `basename` strips any number of trailing slashes. )", - .fun = prim_baseNameOf, + .impl = prim_baseNameOf, }); /* Return the directory of the given path, i.e., everything before the @@ -2141,7 +2149,7 @@ static RegisterPrimOp primop_dirOf({ before the final slash in the string. This is similar to the GNU `dirname` command. )", - .fun = prim_dirOf, + .impl = prim_dirOf, }); /* Return the contents of a file as a string. */ @@ -2183,7 +2191,7 @@ static RegisterPrimOp primop_readFile({ .doc = R"( Return the contents of the file *path* as a string. )", - .fun = prim_readFile, + .impl = prim_readFile, }); /* Find a file in the Nix search path. Used to implement paths, @@ -2223,7 +2231,7 @@ static void prim_findFile(EvalState & state, const PosIdx pos, Value ** args, Va auto rewrites = state.realiseContext(context); path = rewriteStrings(std::move(path), rewrites); } catch (InvalidPathError & e) { - state.error("cannot find '%1%', since path '%2%' is not valid", path, e.path) + state.error("cannot find '%1%', since path '%2%' is not valid", path, e.path.to_string()) .atPos(pos) .debugThrow(); } @@ -2373,7 +2381,7 @@ static RegisterPrimOp primop_findFile( > > makes `` refer to a particular branch of the `NixOS/nixpkgs` repository on GitHub. )", - .fun = prim_findFile, + .impl = prim_findFile, }); /* Return the cryptographic hash of a file in base-16. */ @@ -2398,7 +2406,7 @@ static RegisterPrimOp primop_hashFile({ file at path *p*. The hash algorithm specified by *type* must be one of `"md5"`, `"sha1"`, `"sha256"` or `"sha512"`. )", - .fun = prim_hashFile, + .impl = prim_hashFile, }); static RegisterPrimOp primop_narHash({ @@ -2407,7 +2415,7 @@ static RegisterPrimOp primop_narHash({ .doc = R"( Return an SRI representation of the SHA-256 hash of the NAR serialisation of the path *p*. )", - .fun = + .impl = [](EvalState & state, const PosIdx pos, Value ** args, Value & v) { auto path = state.realisePath(pos, *args[0]); auto hash = @@ -2466,7 +2474,7 @@ static RegisterPrimOp primop_readFileType({ Determine the directory entry type of a filesystem node, being one of `"directory"`, `"regular"`, `"symlink"`, or `"unknown"`. )", - .fun = prim_readFileType, + .impl = prim_readFileType, }); /* Read a directory (without . or ..) */ @@ -2525,7 +2533,7 @@ static RegisterPrimOp primop_readDir({ The possible values for the file type are `"regular"`, `"directory"`, `"symlink"` and `"unknown"`. )", - .fun = prim_readDir, + .impl = prim_readDir, }); /* Extend single element string context with another output. */ @@ -2571,7 +2579,7 @@ static RegisterPrimOp primop_outputOf({ This primop corresponds to the `^` sigil for [deriving paths](@docroot@/glossary.md#gloss-deriving-path), e.g. as part of installable syntax on the command line. )", - .fun = prim_outputOf, + .impl = prim_outputOf, .experimentalFeature = Xp::DynamicDerivations, }); @@ -2684,7 +2692,7 @@ static RegisterPrimOp primop_toXML({ stylesheet is spliced into the builder using the syntax `xsltproc ${stylesheet}`. )", - .fun = prim_toXML, + .impl = prim_toXML, }); /* Convert the argument (which can be any Nix expression) to a JSON @@ -2709,7 +2717,7 @@ static RegisterPrimOp primop_toJSON({ derivation’s output path. Paths are copied to the store and represented as a JSON string of the resulting store path. )", - .fun = prim_toJSON, + .impl = prim_toJSON, }); /* Parse a JSON string to a value. */ @@ -2736,7 +2744,7 @@ static RegisterPrimOp primop_fromJSON({ returns the value `{ x = [ 1 2 3 ]; y = null; }`. )", - .fun = prim_fromJSON, + .impl = prim_fromJSON, }); /* Store a string in the Nix store as a source file that can be used @@ -2877,7 +2885,7 @@ static RegisterPrimOp primop_toFile({ you are using Nixpkgs, the `writeTextFile` function is able to do that. )", - .fun = prim_toFile, + .impl = prim_toFile, }); bool EvalState::callPathFilter(Value * filterFun, const SourcePath & path, PosIdx pos) @@ -2925,7 +2933,7 @@ static void addPath( std::unique_ptr filter; if (filterFun) - filter = std::make_unique([&](const Path & p) { + filter = std::make_unique([&](const std::string & p) { auto p2 = CanonPath(p); return state.callPathFilter(filterFun, {path.accessor, p2}, pos); }); @@ -3053,7 +3061,7 @@ static RegisterPrimOp primop_filterSource({ `true` for them, the copy fails). If you exclude a directory, the entire corresponding subtree of *e2* is excluded. )", - .fun = prim_filterSource, + .impl = prim_filterSource, }); static void prim_path(EvalState & state, const PosIdx pos, Value ** args, Value & v) @@ -3124,18 +3132,21 @@ static RegisterPrimOp primop_path({ - recursive\ When `false`, when `path` is added to the store it is with a - flat hash, rather than a hash of the NAR serialization of the - file. Thus, `path` must refer to a regular file, not a + [flat hash](@docroot@/store/file-system-object/content-address.md#serial-flat), + rather than a hash of the + [NAR serialization](@docroot@/store/file-system-object/content-address.md#serial-nix-archive) + of the file. Thus, `path` must refer to a regular file, not a directory. This allows similar behavior to `fetchurl`. Defaults to `true`. - sha256\ - When provided, this is the expected hash of the file at the - path. Evaluation fails if the hash is incorrect, and - providing a hash allows `builtins.path` to be used even when the - `pure-eval` nix config option is on. + When provided, this is the expected + [content hash](@docroot@/store/file-system-object/content-address.md) + of the path. Evaluation fails if the hash is incorrect, + and providing a hash allows `builtins.path` to be used even + when the `pure-eval` nix config option is on. )", - .fun = prim_path, + .impl = prim_path, }); /************************************************************* @@ -3166,7 +3177,7 @@ static RegisterPrimOp primop_attrNames({ alphabetically sorted list. For instance, `builtins.attrNames { y = 1; x = "foo"; }` evaluates to `[ "x" "y" ]`. )", - .fun = prim_attrNames, + .impl = prim_attrNames, }); /* Return the values of the attributes in a set as a list, in the same @@ -3198,7 +3209,7 @@ static RegisterPrimOp primop_attrValues({ Return the values of the attributes in the set *set* in the order corresponding to the sorted attribute names. )", - .fun = prim_attrValues, + .impl = prim_attrValues, }); /* Dynamic version of the `.' operator. */ @@ -3223,7 +3234,7 @@ static RegisterPrimOp primop_getAttr({ the `.` operator, since *s* is an expression rather than an identifier. )", - .fun = prim_getAttr, + .impl = prim_getAttr, }); /* Return position information of the specified attribute. */ @@ -3249,7 +3260,7 @@ static RegisterPrimOp primop_unsafeGetAttrPos( from *set*. This is used by Nixpkgs to provide location information in error messages. )", - .fun = prim_unsafeGetAttrPos, + .impl = prim_unsafeGetAttrPos, }); // access to exact position information (ie, line and column numbers) is deferred @@ -3266,10 +3277,10 @@ static RegisterPrimOp primop_unsafeGetAttrPos( // for in the very hot path that is forceValue. static struct LazyPosAccessors { - PrimOp primop_lineOfPos{.arity = 1, .fun = [](EvalState & state, PosIdx pos, Value ** args, Value & v) { + PrimOp primop_lineOfPos{.arity = 1, .impl = [](EvalState & state, PosIdx pos, Value ** args, Value & v) { v.mkInt(state.positions[PosIdx(args[0]->integer().value)].line); }}; - PrimOp primop_columnOfPos{.arity = 1, .fun = [](EvalState & state, PosIdx pos, Value ** args, Value & v) { + PrimOp primop_columnOfPos{.arity = 1, .impl = [](EvalState & state, PosIdx pos, Value ** args, Value & v) { v.mkInt(state.positions[PosIdx(args[0]->integer().value)].column); }}; @@ -3311,7 +3322,7 @@ static RegisterPrimOp primop_hasAttr({ `false` otherwise. This is a dynamic version of the `?` operator, since *s* is an expression rather than an identifier. )", - .fun = prim_hasAttr, + .impl = prim_hasAttr, }); /* Determine whether the argument is a set. */ @@ -3327,7 +3338,7 @@ static RegisterPrimOp primop_isAttrs({ .doc = R"( Return `true` if *e* evaluates to a set, and `false` otherwise. )", - .fun = prim_isAttrs, + .impl = prim_isAttrs, }); static void prim_removeAttrs(EvalState & state, const PosIdx pos, Value ** args, Value & v) @@ -3370,7 +3381,7 @@ static RegisterPrimOp primop_removeAttrs({ evaluates to `{ y = 2; }`. )", - .fun = prim_removeAttrs, + .impl = prim_removeAttrs, }); /* Builds a set from a list specifying (name, value) pairs. To be @@ -3457,7 +3468,7 @@ static RegisterPrimOp primop_listToAttrs({ { foo = 123; bar = 456; } ``` )", - .fun = prim_listToAttrs, + .impl = prim_listToAttrs, }); static void prim_intersectAttrs(EvalState & state, const PosIdx pos, Value ** args, Value & v) @@ -3534,7 +3545,7 @@ static RegisterPrimOp primop_intersectAttrs({ Performs in O(*n* log *m*) where *n* is the size of the smaller set and *m* the larger set's size. )", - .fun = prim_intersectAttrs, + .impl = prim_intersectAttrs, }); static void prim_catAttrs(EvalState & state, const PosIdx pos, Value ** args, Value & v) @@ -3573,7 +3584,7 @@ static RegisterPrimOp primop_catAttrs({ evaluates to `[1 2]`. )", - .fun = prim_catAttrs, + .impl = prim_catAttrs, }); static void prim_functionArgs(EvalState & state, const PosIdx pos, Value ** args, Value & v) @@ -3616,7 +3627,7 @@ static RegisterPrimOp primop_functionArgs({ the function. Plain lambdas are not included, e.g. `functionArgs (x: ...) = { }`. )", - .fun = prim_functionArgs, + .impl = prim_functionArgs, }); /* */ @@ -3648,7 +3659,7 @@ static RegisterPrimOp primop_mapAttrs({ evaluates to `{ a = 10; b = 20; }`. )", - .fun = prim_mapAttrs, + .impl = prim_mapAttrs, }); static void prim_filterAttrs(EvalState & state, const PosIdx pos, Value ** args, Value & v) @@ -3691,7 +3702,7 @@ static RegisterPrimOp primop_filterAttrs({ evaluates to `{ foo = 1; }`. )", - .fun = prim_filterAttrs, + .impl = prim_filterAttrs, }); static void prim_zipAttrsWith(EvalState & state, const PosIdx pos, Value ** args, Value & v) @@ -3778,7 +3789,7 @@ static RegisterPrimOp primop_zipAttrsWith({ } ``` )", - .fun = prim_zipAttrsWith, + .impl = prim_zipAttrsWith, }); /************************************************************* @@ -3798,7 +3809,7 @@ static RegisterPrimOp primop_isList({ .doc = R"( Return `true` if *e* evaluates to a list, and `false` otherwise. )", - .fun = prim_isList, + .impl = prim_isList, }); /* Return the n-1'th element of a list. */ @@ -3822,7 +3833,7 @@ static RegisterPrimOp primop_elemAt({ Return element *n* from the list *xs*. Elements are counted starting from 0. A fatal error occurs if the index is out of bounds. )", - .fun = prim_elemAt, + .impl = prim_elemAt, }); /* Return the first element of a list. */ @@ -3843,7 +3854,7 @@ static RegisterPrimOp primop_head({ isn’t a list or is an empty list. You can test whether a list is empty by comparing it with `[]`. )", - .fun = prim_head, + .impl = prim_head, }); /* Return a list consisting of everything but the first element of @@ -3874,7 +3885,7 @@ static RegisterPrimOp primop_tail({ > unlike Haskell's `tail`, it takes O(n) time, so recursing over a > list by repeatedly calling `tail` takes O(n^2) time. )", - .fun = prim_tail, + .impl = prim_tail, }); /* Apply a function to every element of a list. */ @@ -3908,7 +3919,7 @@ static RegisterPrimOp primop_map({ evaluates to `[ "foobar" "foobla" "fooabc" ]`. )", - .fun = prim_map, + .impl = prim_map, }); /* Filter a list using a predicate; that is, return a list containing @@ -3957,7 +3968,7 @@ static RegisterPrimOp primop_filter({ Return a list consisting of the elements of *list* for which the function *f* returns `true`. )", - .fun = prim_filter, + .impl = prim_filter, }); /* Return true if a list contains a given element. */ @@ -3980,7 +3991,7 @@ static RegisterPrimOp primop_elem({ Return `true` if a value equal to *x* occurs in the list *xs*, and `false` otherwise. )", - .fun = prim_elem, + .impl = prim_elem, }); /* Concatenate a list of lists. */ @@ -4002,7 +4013,7 @@ static RegisterPrimOp primop_concatLists({ .doc = R"( Concatenate a list of lists into a single list. )", - .fun = prim_concatLists, + .impl = prim_concatLists, }); /* Return the length of a list. This is an O(1) time operation. */ @@ -4018,7 +4029,7 @@ static RegisterPrimOp primop_length({ .doc = R"( Return the length of the list *e*. )", - .fun = prim_length, + .impl = prim_length, }); /* Reduce a list by applying a binary operator, from left to @@ -4061,7 +4072,7 @@ static RegisterPrimOp primop_foldlStrict({ of each application of `op` is evaluated immediately, even for intermediate values. )", - .fun = prim_foldlStrict, + .impl = prim_foldlStrict, }); static void anyOrAll(bool any, EvalState & state, const PosIdx pos, Value ** args, Value & v) @@ -4099,7 +4110,7 @@ static RegisterPrimOp primop_any({ Return `true` if the function *pred* returns `true` for at least one element of *list*, and `false` otherwise. )", - .fun = prim_any, + .impl = prim_any, }); static void prim_all(EvalState & state, const PosIdx pos, Value ** args, Value & v) @@ -4114,7 +4125,7 @@ static RegisterPrimOp primop_all({ Return `true` if the function *pred* returns `true` for all elements of *list*, and `false` otherwise. )", - .fun = prim_all, + .impl = prim_all, }); static void prim_genList(EvalState & state, const PosIdx pos, Value ** args, Value & v) @@ -4152,7 +4163,7 @@ static RegisterPrimOp primop_genList({ returns the list `[ 0 1 4 9 16 ]`. )", - .fun = prim_genList, + .impl = prim_genList, }); static void prim_lessThan(EvalState & state, const PosIdx pos, Value ** args, Value & v); @@ -4177,7 +4188,7 @@ static void prim_sort(EvalState & state, const PosIdx pos, Value ** args, Value /* Optimization: if the comparator is lessThan, bypass callFunction. */ if (args[0]->isPrimOp()) { - auto ptr = args[0]->primOp()->fun.target(); + auto ptr = args[0]->primOp()->impl.get_fn().target(); if (ptr && *ptr == prim_lessThan) return CompareValues(state, noPos, "while evaluating the ordering function passed to builtins.sort")( a, b); @@ -4226,6 +4237,8 @@ static RegisterPrimOp primop_sort({ 1. Transitivity + If a is less than b and b is less than c, then it follows that a is less than c. + ```nix comparator a b && comparator b c -> comparator a c ``` @@ -4238,15 +4251,29 @@ static RegisterPrimOp primop_sort({ 1. Transitivity of equivalence + First, two values a and b are considered equivalent with respect to the comparator if: + + ``` + !comparator a b && !comparator b a + ``` + + In other words, neither is considered "less than" the other. + + Transitivity of equivalence means: + + If a is equivalent to b, and b is equivalent to c, then a must also be equivalent to c. + ```nix - let equiv = a: b: (!comparator a b && !comparator b a); in - equiv a b && equiv b c -> equiv a c + let + equiv = x: y: (!comparator x y && !comparator y x); + in + equiv a b && equiv b c -> equiv a c ``` If the *comparator* violates any of these properties, then `builtins.sort` reorders elements in an unspecified manner. )", - .fun = prim_sort, + .impl = prim_sort, }); static void prim_partition(EvalState & state, const PosIdx pos, Value ** args, Value & v) @@ -4307,7 +4334,7 @@ static RegisterPrimOp primop_partition({ { right = [ 23 42 ]; wrong = [ 1 9 3 ]; } ``` )", - .fun = prim_partition, + .impl = prim_partition, }); static void prim_groupBy(EvalState & state, const PosIdx pos, Value ** args, Value & v) @@ -4360,7 +4387,7 @@ static RegisterPrimOp primop_groupBy({ { b = [ "bar" "baz" ]; f = [ "foo" ]; } ``` )", - .fun = prim_groupBy, + .impl = prim_groupBy, }); static void prim_concatMap(EvalState & state, const PosIdx pos, Value ** args, Value & v) @@ -4402,7 +4429,7 @@ static RegisterPrimOp primop_concatMap({ This function is equivalent to `builtins.concatLists (map f list)` but is more efficient. )", - .fun = prim_concatMap, + .impl = prim_concatMap, }); /************************************************************* @@ -4436,7 +4463,7 @@ static RegisterPrimOp primop_add({ .doc = R"( Return the sum of the numbers *e1* and *e2*. )", - .fun = prim_add, + .impl = prim_add, }); static void prim_sub(EvalState & state, const PosIdx pos, Value ** args, Value & v) @@ -4467,7 +4494,7 @@ static RegisterPrimOp primop_sub({ .doc = R"( Return the difference between the numbers *e1* and *e2*. )", - .fun = prim_sub, + .impl = prim_sub, }); static void prim_mul(EvalState & state, const PosIdx pos, Value ** args, Value & v) @@ -4498,7 +4525,7 @@ static RegisterPrimOp primop_mul({ .doc = R"( Return the product of the numbers *e1* and *e2*. )", - .fun = prim_mul, + .impl = prim_mul, }); static void prim_div(EvalState & state, const PosIdx pos, Value ** args, Value & v) @@ -4531,7 +4558,7 @@ static RegisterPrimOp primop_div({ .doc = R"( Return the quotient of the numbers *e1* and *e2*. )", - .fun = prim_div, + .impl = prim_div, }); static void prim_bitAnd(EvalState & state, const PosIdx pos, Value ** args, Value & v) @@ -4547,7 +4574,7 @@ static RegisterPrimOp primop_bitAnd({ .doc = R"( Return the bitwise AND of the integers *e1* and *e2*. )", - .fun = prim_bitAnd, + .impl = prim_bitAnd, }); static void prim_bitOr(EvalState & state, const PosIdx pos, Value ** args, Value & v) @@ -4564,7 +4591,7 @@ static RegisterPrimOp primop_bitOr({ .doc = R"( Return the bitwise OR of the integers *e1* and *e2*. )", - .fun = prim_bitOr, + .impl = prim_bitOr, }); static void prim_bitXor(EvalState & state, const PosIdx pos, Value ** args, Value & v) @@ -4581,7 +4608,7 @@ static RegisterPrimOp primop_bitXor({ .doc = R"( Return the bitwise XOR of the integers *e1* and *e2*. )", - .fun = prim_bitXor, + .impl = prim_bitXor, }); static void prim_lessThan(EvalState & state, const PosIdx pos, Value ** args, Value & v) @@ -4601,7 +4628,7 @@ static RegisterPrimOp primop_lessThan({ Evaluation aborts if either *e1* or *e2* does not evaluate to a number, string or path. Furthermore, it aborts if *e2* does not match *e1*'s type according to the aforementioned classification of number, string or path. )", - .fun = prim_lessThan, + .impl = prim_lessThan, }); /************************************************************* @@ -4640,7 +4667,7 @@ static RegisterPrimOp primop_toString({ - `null`, which yields the empty string. )", - .fun = prim_toString, + .impl = prim_toString, }); /* `substring start len str' returns the substring of `str' starting @@ -4710,7 +4737,7 @@ static RegisterPrimOp primop_substring({ evaluates to `"nix"`. )", - .fun = prim_substring, + .impl = prim_substring, }); static void prim_stringLength(EvalState & state, const PosIdx pos, Value ** args, Value & v) @@ -4728,7 +4755,7 @@ static RegisterPrimOp primop_stringLength({ Return the number of bytes of the string *e*. If *e* is not a string, evaluation is aborted. )", - .fun = prim_stringLength, + .impl = prim_stringLength, }); /* Return the cryptographic hash of a string in base-16. */ @@ -4756,7 +4783,7 @@ static RegisterPrimOp primop_hashString({ *s*. The hash algorithm specified by *type* must be one of `"md5"`, `"sha1"`, `"sha256"` or `"sha512"`. )", - .fun = prim_hashString, + .impl = prim_hashString, }); static void prim_convertHash(EvalState & state, const PosIdx pos, Value ** args, Value & v) @@ -4854,26 +4881,33 @@ static RegisterPrimOp primop_convertHash({ > > "sha256-47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=" )", - .fun = prim_convertHash, + .impl = prim_convertHash, }); struct RegexCache { - boost::concurrent_flat_map> cache; + struct Entry + { + ref regex; + + Entry(const char * s, size_t count) + : regex(make_ref(s, count, std::regex::extended)) + { + } + }; + + boost::concurrent_flat_map> cache; - std::regex get(std::string_view re) + ref get(std::string_view re) { - std::regex regex; - /* No std::regex constructor overload from std::string_view, but can be constructed - from a pointer + size or an iterator range. */ + std::optional> regex; cache.try_emplace_and_cvisit( re, /*s=*/re.data(), /*count=*/re.size(), - std::regex::extended, - [®ex](const auto & kv) { regex = kv.second; }, - [®ex](const auto & kv) { regex = kv.second; }); - return regex; + [®ex](const auto & kv) { regex = kv.second.regex; }, + [®ex](const auto & kv) { regex = kv.second.regex; }); + return *regex; } }; @@ -4895,7 +4929,7 @@ void prim_match(EvalState & state, const PosIdx pos, Value ** args, Value & v) state.forceString(*args[1], context, pos, "while evaluating the second argument passed to builtins.match"); std::cmatch match; - if (!std::regex_match(str.begin(), str.end(), match, regex)) { + if (!std::regex_match(str.begin(), str.end(), match, *regex)) { v.mkNull(); return; } @@ -4951,7 +4985,7 @@ static RegisterPrimOp primop_match({ Evaluates to `[ "FOO" ]`. )s", - .fun = prim_match, + .impl = prim_match, }); /* Split a string with a regular expression, and return a list of the @@ -4968,7 +5002,7 @@ void prim_split(EvalState & state, const PosIdx pos, Value ** args, Value & v) const auto str = state.forceString(*args[1], context, pos, "while evaluating the second argument passed to builtins.split"); - auto begin = std::cregex_iterator(str.begin(), str.end(), regex); + auto begin = std::cregex_iterator(str.begin(), str.end(), *regex); auto end = std::cregex_iterator(); // Any matches results are surrounded by non-matching results. @@ -5055,7 +5089,7 @@ static RegisterPrimOp primop_split({ Evaluates to `[ " " [ "FOO" ] " " ]`. )s", - .fun = prim_split, + .impl = prim_split, }); static void prim_concatStringsSep(EvalState & state, const PosIdx pos, Value ** args, Value & v) @@ -5099,7 +5133,7 @@ static RegisterPrimOp primop_concatStringsSep({ element, e.g. `concatStringsSep "/" ["usr" "local" "bin"] == "usr/local/bin"`. )", - .fun = prim_concatStringsSep, + .impl = prim_concatStringsSep, }); static void prim_replaceStrings(EvalState & state, const PosIdx pos, Value ** args, Value & v) @@ -5183,7 +5217,7 @@ static RegisterPrimOp primop_replaceStrings({ evaluates to `"fabir"`. )", - .fun = prim_replaceStrings, + .impl = prim_replaceStrings, }); /************************************************************* @@ -5212,7 +5246,7 @@ static RegisterPrimOp primop_parseDrvName({ `builtins.parseDrvName "nix-0.12pre12876"` returns `{ name = "nix"; version = "0.12pre12876"; }`. )", - .fun = prim_parseDrvName, + .impl = prim_parseDrvName, }); static void prim_compareVersions(EvalState & state, const PosIdx pos, Value ** args, Value & v) @@ -5235,7 +5269,7 @@ static RegisterPrimOp primop_compareVersions({ algorithm is the same as the one used by [`nix-env -u`](../command-ref/nix-env/upgrade.md). )", - .fun = prim_compareVersions, + .impl = prim_compareVersions, }); static void prim_splitVersion(EvalState & state, const PosIdx pos, Value ** args, Value & v) @@ -5264,7 +5298,7 @@ static RegisterPrimOp primop_splitVersion({ same version splitting logic underlying the version comparison in [`nix-env -u`](../command-ref/nix-env/upgrade.md). )", - .fun = prim_splitVersion, + .impl = prim_splitVersion, }); /************************************************************* @@ -5365,7 +5399,7 @@ void EvalState::createBaseEnv(const EvalSettings & evalSettings) )", }); - v.mkInt(time(0)); + v.mkInt(time(nullptr)); addConstant( "__currentTime", v, @@ -5489,12 +5523,12 @@ void EvalState::createBaseEnv(const EvalSettings & evalSettings) addPrimOp({ .name = "__importNative", .arity = 2, - .fun = prim_importNative, + .impl = prim_importNative, }); addPrimOp({ .name = "__exec", .arity = 1, - .fun = prim_exec, + .impl = prim_exec, }); } #endif @@ -5508,7 +5542,7 @@ void EvalState::createBaseEnv(const EvalSettings & evalSettings) error if `--trace-verbose` is enabled. Then return *e2*. This function is useful for debugging. )", - .fun = settings.traceVerbose ? prim_trace : prim_second, + .impl = settings.traceVerbose ? prim_trace : prim_second, }); /* Add a value containing the current Nix expression search path. */ diff --git a/src/libexpr/primops/context.cc b/src/libexpr/primops/context.cc index d4824d9b9e50..569e8a924d23 100644 --- a/src/libexpr/primops/context.cc +++ b/src/libexpr/primops/context.cc @@ -26,7 +26,7 @@ static RegisterPrimOp primop_unsafeDiscardStringContext({ .doc = R"( Discard the [string context](@docroot@/language/string-context.md) from a value that can be coerced to a string. )", - .fun = prim_unsafeDiscardStringContext, + .impl = prim_unsafeDiscardStringContext, }); bool hasContext(const NixStringContext & context) @@ -65,7 +65,7 @@ static RegisterPrimOp primop_hasContext( > else { ${name} = meta; } > ``` )", - .fun = prim_hasContext}); + .impl = prim_hasContext}); static void prim_unsafeDiscardOutputDependency(EvalState & state, const PosIdx pos, Value ** args, Value & v) { @@ -107,7 +107,7 @@ static RegisterPrimOp primop_unsafeDiscardOutputDependency( [`builtins.addDrvOutputDependencies`]: #builtins-addDrvOutputDependencies )", - .fun = prim_unsafeDiscardOutputDependency}); + .impl = prim_unsafeDiscardOutputDependency}); static void prim_addDrvOutputDependencies(EvalState & state, const PosIdx pos, Value ** args, Value & v) { @@ -177,7 +177,7 @@ static RegisterPrimOp primop_addDrvOutputDependencies( This is the opposite of [`builtins.unsafeDiscardOutputDependency`](#builtins-unsafeDiscardOutputDependency). )", - .fun = prim_addDrvOutputDependencies}); + .impl = prim_addDrvOutputDependencies}); /* Extract the context of a string as a structured Nix value. @@ -270,7 +270,7 @@ static RegisterPrimOp primop_getContext( { "/nix/store/arhvjaf6zmlyn8vh8fgn55rpwnxq0n7l-a.drv" = { outputs = [ "out" ]; }; } ``` )", - .fun = prim_getContext}); + .impl = prim_getContext}); /* Append the given context to a given string. @@ -345,6 +345,6 @@ static void prim_appendContext(EvalState & state, const PosIdx pos, Value ** arg v.mkString(orig, context, state.mem); } -static RegisterPrimOp primop_appendContext({.name = "__appendContext", .arity = 2, .fun = prim_appendContext}); +static RegisterPrimOp primop_appendContext({.name = "__appendContext", .arity = 2, .impl = prim_appendContext}); } // namespace nix diff --git a/src/libexpr/primops/fetchClosure.cc b/src/libexpr/primops/fetchClosure.cc index f849d0debb87..0f7eafefe9e6 100644 --- a/src/libexpr/primops/fetchClosure.cc +++ b/src/libexpr/primops/fetchClosure.cc @@ -191,19 +191,25 @@ static void prim_fetchClosure(EvalState & state, const PosIdx pos, Value ** args {.msg = HintFmt("attribute '%s' is missing in call to 'fetchClosure'", "fromStore"), .pos = state.positions[pos]}); - auto parsedURL = parseURL(*fromStoreUrl, /*lenient=*/true); - - if (parsedURL.scheme != "http" && parsedURL.scheme != "https" - && !(getEnv("_NIX_IN_TEST").has_value() && parsedURL.scheme == "file")) - throw Error( - {.msg = HintFmt("'fetchClosure' only supports http:// and https:// stores"), .pos = state.positions[pos]}); - - if (!parsedURL.query.empty()) + auto storeRef = StoreReference::parse(*fromStoreUrl); + + if ([&] { + auto * specified = std::get_if(&storeRef.variant); + return !specified + || (specified->scheme != "http" && specified->scheme != "https" + && !(getEnv("_NIX_IN_TEST").has_value() && specified->scheme == "file")); + }()) + throw Error({ + .msg = HintFmt("'fetchClosure' only supports http:// and https:// stores"), + .pos = state.positions[pos], + }); + + if (!storeRef.params.empty()) throw Error( {.msg = HintFmt("'fetchClosure' does not support URL query parameters (in '%s')", *fromStoreUrl), .pos = state.positions[pos]}); - auto fromStore = openStore(parsedURL.to_string()); + auto fromStore = openStore(std::move(storeRef)); if (toPath) runFetchClosureWithRewrite(state, pos, *fromStore, *fromPath, *toPath, v); @@ -285,7 +291,7 @@ static RegisterPrimOp primop_fetchClosure({ However, `fetchClosure` is more reproducible because it specifies a binary cache from which the path can be fetched. Also, using content-addressed store paths does not require users to configure [`trusted-public-keys`](@docroot@/command-ref/conf-file.md#conf-trusted-public-keys) to ensure their authenticity. )", - .fun = prim_fetchClosure, + .impl = prim_fetchClosure, .experimentalFeature = Xp::FetchClosure, }); diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc index 4ab060f7807f..59ffeabf1ef8 100644 --- a/src/libexpr/primops/fetchMercurial.cc +++ b/src/libexpr/primops/fetchMercurial.cc @@ -99,6 +99,6 @@ static void prim_fetchMercurial(EvalState & state, const PosIdx pos, Value ** ar state.allowPath(storePath); } -static RegisterPrimOp r_fetchMercurial({.name = "fetchMercurial", .arity = 1, .fun = prim_fetchMercurial}); +static RegisterPrimOp r_fetchMercurial({.name = "fetchMercurial", .arity = 1, .impl = prim_fetchMercurial}); } // namespace nix diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 691d13404f66..7b36809903d0 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -344,7 +344,7 @@ static RegisterPrimOp primop_fetchTree({ return doc; }(), - .fun = prim_fetchTree, + .impl = prim_fetchTree, }); static void fetch( @@ -505,7 +505,7 @@ static RegisterPrimOp primop_fetchurl({ Not available in [restricted evaluation mode](@docroot@/command-ref/conf-file.md#conf-restrict-eval). )", - .fun = prim_fetchurl, + .impl = prim_fetchurl, }); static void prim_fetchTarball(EvalState & state, const PosIdx pos, Value ** args, Value & v) @@ -555,7 +555,7 @@ static RegisterPrimOp primop_fetchTarball({ Not available in [restricted evaluation mode](@docroot@/command-ref/conf-file.md#conf-restrict-eval). )", - .fun = prim_fetchTarball, + .impl = prim_fetchTarball, }); static void prim_fetchGit(EvalState & state, const PosIdx pos, Value ** args, Value & v) @@ -771,7 +771,7 @@ static RegisterPrimOp primop_fetchGit({ files, even if they are not committed or added to Git's index. It only considers files added to the Git repository, as listed by `git ls-files`. )", - .fun = prim_fetchGit, + .impl = prim_fetchGit, }); } // namespace nix diff --git a/src/libexpr/primops/fromTOML.cc b/src/libexpr/primops/fromTOML.cc index 562ff3d1497f..37685007bd98 100644 --- a/src/libexpr/primops/fromTOML.cc +++ b/src/libexpr/primops/fromTOML.cc @@ -187,6 +187,6 @@ static RegisterPrimOp primop_fromTOML( returns the value `{ s = "a"; table = { y = 2; }; x = 1; }`. )", - .fun = prim_fromTOML}); + .impl = prim_fromTOML}); } // namespace nix diff --git a/src/libexpr/primops/wasm.cc b/src/libexpr/primops/wasm.cc index c0dd9f40e08d..3782303198e4 100644 --- a/src/libexpr/primops/wasm.cc +++ b/src/libexpr/primops/wasm.cc @@ -703,7 +703,7 @@ static RegisterPrimOp primop_wasm( } { x = 42; } ``` )", - .fun = prim_wasm, + .impl = prim_wasm, .experimentalFeature = Xp::WasmBuiltin}); } // namespace nix diff --git a/src/libexpr/print-ambiguous.cc b/src/libexpr/print-ambiguous.cc index f80ef2b044bf..1912262ee576 100644 --- a/src/libexpr/print-ambiguous.cc +++ b/src/libexpr/print-ambiguous.cc @@ -2,18 +2,17 @@ #include "nix/expr/print.hh" #include "nix/util/signals.hh" #include "nix/expr/eval.hh" +#include "nix/expr/eval-error.hh" namespace nix { // See: https://github.com/NixOS/nix/issues/9730 -void printAmbiguous(EvalState & state, Value & v, std::ostream & str, std::set * seen, int depth) +void printAmbiguous(EvalState & state, Value & v, std::ostream & str, std::set * seen, size_t depth) { checkInterrupt(); - if (depth <= 0) { - str << "«too deep»"; - return; - } + if (depth > state.settings.maxCallDepth) + state.error().atPos(v.determinePos(noPos)).debugThrow(); switch (v.type()) { case nInt: str << v.integer(); @@ -41,7 +40,7 @@ void printAmbiguous(EvalState & state, Value & v, std::ostream & str, std::setlexicographicOrder(state.symbols)) { str << state.symbols[i->name] << " = "; - printAmbiguous(state, *i->value, str, seen, depth - 1); + printAmbiguous(state, *i->value, str, seen, depth + 1); str << "; "; } str << "}"; @@ -57,7 +56,7 @@ void printAmbiguous(EvalState & state, Value & v, std::ostream & str, std::set state.settings.maxCallDepth) + state.error().atPos(v.determinePos(noPos)).debugThrow(); + try { if (options.force) { state.forceValue(v, v.determinePos(noPos)); @@ -589,12 +605,12 @@ class Printer printFunction(v); break; - case nThunk: - printThunk(v); + case nFailed: + printFailed(); break; - case nFailed: - printFailed(v); + case nThunk: + printThunk(v); break; case nExternal: @@ -605,6 +621,11 @@ class Printer printUnknown(); break; } + } catch (StackOverflowError &) { + // Always re-throw because stack overflow is a serious condition + // that expressions should avoid, unlike say `throw`, which can + // be part of legitimate expression patterns. + throw; } catch (Error & e) { if (options.errors == ErrorPrintBehavior::Throw || (options.errors == ErrorPrintBehavior::ThrowTopLevel && depth == 0)) { diff --git a/src/libexpr/value-to-xml.cc b/src/libexpr/value-to-xml.cc index 21de85a17173..f2b004753a6f 100644 --- a/src/libexpr/value-to-xml.cc +++ b/src/libexpr/value-to-xml.cc @@ -21,7 +21,7 @@ static void printValueAsXML( Value & v, XMLWriter & doc, NixStringContext & context, - PathSet & drvsSeen, + StringSet & drvsSeen, const PosIdx pos); static void posToXML(EvalState & state, XMLAttrs & xmlAttrs, const Pos & pos) @@ -39,7 +39,7 @@ static void showAttrs( const Bindings & attrs, XMLWriter & doc, NixStringContext & context, - PathSet & drvsSeen) + StringSet & drvsSeen) { StringSet names; @@ -61,11 +61,13 @@ static void printValueAsXML( Value & v, XMLWriter & doc, NixStringContext & context, - PathSet & drvsSeen, + StringSet & drvsSeen, const PosIdx pos) { checkInterrupt(); + auto _level = state.addCallDepth(pos); + if (strict) state.forceValue(v, pos); @@ -97,7 +99,7 @@ static void printValueAsXML( if (state.isDerivation(v)) { XMLAttrs xmlAttrs; - Path drvPath; + std::string drvPath; if (auto a = v.attrs()->get(state.s.drvPath)) { if (strict) state.forceValue(*a->value, a->pos); @@ -169,11 +171,10 @@ static void printValueAsXML( break; case nThunk: - doc.writeEmptyElement("unevaluated"); - break; - + // Historically, a tried and then ignored value (e.g. through tryEval) was + // reverted to the original thunk. case nFailed: - doc.writeEmptyElement("failed"); + doc.writeEmptyElement("unevaluated"); break; } } @@ -184,7 +185,7 @@ void ExternalValueBase::printValueAsXML( bool location, XMLWriter & doc, NixStringContext & context, - PathSet & drvsSeen, + StringSet & drvsSeen, const PosIdx pos) const { doc.writeEmptyElement("unevaluated"); @@ -201,7 +202,7 @@ void printValueAsXML( { XMLWriter doc(true, out); XMLOpenElement root(doc, "expr"); - PathSet drvsSeen; + StringSet drvsSeen; printValueAsXML(state, strict, location, v, doc, context, drvsSeen, pos); } diff --git a/src/libexpr/value/context.cc b/src/libexpr/value/context.cc index a06d79ddebf6..3bdc73f94aee 100644 --- a/src/libexpr/value/context.cc +++ b/src/libexpr/value/context.cc @@ -1,5 +1,6 @@ #include "nix/util/util.hh" #include "nix/expr/value/context.hh" +#include "nix/store/store-dir-config.hh" #include @@ -105,4 +106,22 @@ std::string NixStringContextElem::to_string() const return res; } +std::string NixStringContextElem::display(const StoreDirConfig & store) const +{ + return std::visit( + overloaded{ + [&](const NixStringContextElem::Opaque & o) -> std::string { + return SingleDerivedPath{o}.to_string(store); + }, + [&](const NixStringContextElem::DrvDeep & d) -> std::string { + return store.printStorePath(d.drvPath) + " (deep)"; + }, + [&](const NixStringContextElem::Built & b) -> std::string { return SingleDerivedPath{b}.to_string(store); }, + [&](const NixStringContextElem::Path & p) -> std::string { + return store.printStorePath(p.storePath) + " (untracked)"; + }, + }, + raw); +} + } // namespace nix diff --git a/src/libfetchers/cache.cc b/src/libfetchers/cache.cc index 1db3ed8dc896..09947e7f8f0f 100644 --- a/src/libfetchers/cache.cc +++ b/src/libfetchers/cache.cc @@ -34,14 +34,20 @@ struct CacheImpl : Cache Sync _state; - CacheImpl() + /** + * This is a back-reference to the `Settings` that owns us. + */ + const Settings & settings; + + CacheImpl(const Settings & _settings) + : settings(_settings) { auto state(_state.lock()); - auto dbPath = (getCacheDir() / "fetcher-cache-v4.sqlite").string(); - createDirs(dirOf(dbPath)); + auto dbPath = getCacheDir() / "fetcher-cache-v4.sqlite"; + createDirs(dbPath.parent_path()); - state->db = SQLite(dbPath); + state->db = SQLite(dbPath, {.useWAL = nix::settings.useSQLiteWAL}); state->db.isCache(); state->db.exec(schema); @@ -54,7 +60,7 @@ struct CacheImpl : Cache void upsert(const Key & key, const Attrs & value) override { _state.lock() - ->upsert.use()(key.first)(attrsToJSON(key.second).dump())(attrsToJSON(value).dump())(time(0)) + ->upsert.use()(key.first)(attrsToJSON(key.second).dump())(attrsToJSON(value).dump())(time(nullptr)) .exec(); } @@ -93,7 +99,7 @@ struct CacheImpl : Cache debug("using cache entry '%s:%s' -> '%s'", key.first, keyJSON, valueJSON); return Result{ - .expired = settings.tarballTtl.get() == 0 || timestamp + settings.tarballTtl < time(0), + .expired = settings.tarballTtl.get() == 0 || timestamp + settings.tarballTtl < time(nullptr), .value = jsonToAttrs(nlohmann::json::parse(valueJSON)), }; } @@ -154,7 +160,7 @@ ref Settings::getCache() const { auto cache(_cache.lock()); if (!*cache) - *cache = std::make_shared(); + *cache = std::make_shared(*this); return ref(*cache); } diff --git a/src/libfetchers/fetch-to-store.cc b/src/libfetchers/fetch-to-store.cc index 8dfb74a9c5af..3e932454dcd2 100644 --- a/src/libfetchers/fetch-to-store.cc +++ b/src/libfetchers/fetch-to-store.cc @@ -6,10 +6,11 @@ namespace nix { fetchers::Cache::Key -makeSourcePathToHashCacheKey(const std::string & fingerprint, ContentAddressMethod method, const std::string & path) +makeSourcePathToHashCacheKey(std::string_view fingerprint, ContentAddressMethod method, const CanonPath & path) { return fetchers::Cache::Key{ - "sourcePathToHash", {{"fingerprint", fingerprint}, {"method", std::string{method.render()}}, {"path", path}}}; + "sourcePathToHash", + {{"fingerprint", std::string(fingerprint)}, {"method", std::string{method.render()}}, {"path", path.abs()}}}; } StorePath fetchToStore( @@ -41,7 +42,7 @@ std::pair fetchToStore2( : path.accessor->getFingerprint(path.path); if (fingerprint) { - cacheKey = makeSourcePathToHashCacheKey(*fingerprint, method, subpath.abs()); + cacheKey = makeSourcePathToHashCacheKey(*fingerprint, method, subpath); if (auto res = settings.getCache()->lookup(*cacheKey)) { auto hash = Hash::parseSRI(fetchers::getStrAttr(*res, "hash")); auto storePath = @@ -80,7 +81,7 @@ std::pair fetchToStore2( auto [storePath, hash] = mode == FetchMode::DryRun - ? ({ + ? [&]() { auto [storePath, hash] = store.computeStorePath(name, path, method, HashAlgorithm::SHA256, {}, filter2); debug( @@ -88,9 +89,9 @@ std::pair fetchToStore2( path, store.printStorePath(storePath), hash.to_string(HashFormat::SRI, true)); - std::make_pair(storePath, hash); - }) - : ({ + return std::make_pair(storePath, hash); + }() + : [&]() { // FIXME: ideally addToStore() would return the hash // right away (like computeStorePath()). auto storePath = store.addToStore(name, path, method, HashAlgorithm::SHA256, {}, filter2, repair); @@ -106,8 +107,8 @@ std::pair fetchToStore2( path, store.printStorePath(storePath), hash.to_string(HashFormat::SRI, true)); - std::make_pair(storePath, hash); - }); + return std::make_pair(storePath, hash); + }(); if (cacheKey) settings.getCache()->upsert(*cacheKey, {{"hash", hash.to_string(HashFormat::SRI, true)}}); diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 4534cf54c3ee..ecbab27741ec 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -168,7 +168,7 @@ bool Input::isFinal() const return maybeGetBoolAttr(attrs, "__final").value_or(false); } -std::optional Input::isRelative() const +std::optional Input::isRelative() const { assert(scheme); return scheme->isRelative(*this); @@ -334,7 +334,8 @@ std::pair, Input> Input::getAccessorUnchecked(const Settings // input back into the store on every evaluation. if (accessor->fingerprint) { settings.getCache()->upsert( - makeSourcePathToHashCacheKey(*accessor->fingerprint, ContentAddressMethod::Raw::NixArchive, "/"), + makeSourcePathToHashCacheKey( + *accessor->fingerprint, ContentAddressMethod::Raw::NixArchive, CanonPath::root), {{"hash", store.queryPathInfo(*storePath)->narHash.to_string(HashFormat::SRI, true)}}); } @@ -519,11 +520,11 @@ void InputScheme::clone( const Settings & settings, Store & store, const Input & input, const std::filesystem::path & destDir) const { if (std::filesystem::exists(destDir)) - throw Error("cannot clone into existing path %s", destDir); + throw Error("cannot clone into existing path %s", PathFmt(destDir)); auto [accessor, input2] = getAccessor(settings, store, input); - Activity act(*logger, lvlTalkative, actUnknown, fmt("copying '%s' to %s...", input2.to_string(), destDir)); + Activity act(*logger, lvlTalkative, actUnknown, fmt("copying '%s' to %s...", input2.to_string(), PathFmt(destDir))); RestoreSink sink(/*startFsync=*/false); sink.dstPath = destDir; diff --git a/src/libfetchers/filtering-source-accessor.cc b/src/libfetchers/filtering-source-accessor.cc index 68eb21566939..a42b5822245e 100644 --- a/src/libfetchers/filtering-source-accessor.cc +++ b/src/libfetchers/filtering-source-accessor.cc @@ -11,13 +11,7 @@ std::optional FilteringSourceAccessor::getPhysicalPath(co return next->getPhysicalPath(prefix / path); } -std::string FilteringSourceAccessor::readFile(const CanonPath & path) -{ - checkAccess(path); - return next->readFile(prefix / path); -} - -void FilteringSourceAccessor::readFile(const CanonPath & path, Sink & sink, std::function sizeCallback) +void FilteringSourceAccessor::readFile(const CanonPath & path, Sink & sink, fun sizeCallback) { checkAccess(path); return next->readFile(prefix / path, sink, sizeCallback); @@ -83,8 +77,7 @@ void FilteringSourceAccessor::invalidateCache(const CanonPath & path) void FilteringSourceAccessor::checkAccess(const CanonPath & path) { if (!isAllowed(path)) - throw makeNotAllowedError ? makeNotAllowedError(path) - : RestrictedPathError("access to path '%s' is forbidden", showPath(path)); + throw makeNotAllowedError(path); } struct AllowListSourceAccessorImpl : AllowListSourceAccessor diff --git a/src/libfetchers/git-lfs-fetch.cc b/src/libfetchers/git-lfs-fetch.cc index e2b2c2e7dda7..4585e68e58ee 100644 --- a/src/libfetchers/git-lfs-fetch.cc +++ b/src/libfetchers/git-lfs-fetch.cc @@ -1,9 +1,11 @@ #include "nix/fetchers/git-lfs-fetch.hh" #include "nix/fetchers/git-utils.hh" #include "nix/store/filetransfer.hh" +#include "nix/util/os-string.hh" #include "nix/util/processes.hh" #include "nix/util/url.hh" #include "nix/util/users.hh" +#include "nix/util/util.hh" #include "nix/util/hash.hh" #include "nix/store/ssh.hh" @@ -59,24 +61,27 @@ static LfsApiInfo getLfsApi(const ParsedURL & url) auto args = getNixSshOpts(); if (url.authority->port) - args.push_back(fmt("-p%d", *url.authority->port)); + args.push_back(string_to_os_string(fmt("-p%d", *url.authority->port))); std::ostringstream hostnameAndUser; if (url.authority->user) hostnameAndUser << *url.authority->user << "@"; hostnameAndUser << url.authority->host; - args.push_back(std::move(hostnameAndUser).str()); + args.push_back(string_to_os_string(std::move(hostnameAndUser).str())); - args.push_back("--"); - args.push_back("git-lfs-authenticate"); + args.push_back(OS_STR("--")); + args.push_back(OS_STR("git-lfs-authenticate")); // FIXME %2F encode slashes? Does this command take/accept percent encoding? - args.push_back(url.renderPath(/*encode=*/false)); - args.push_back("download"); + args.push_back(string_to_os_string(url.renderPath(/*encode=*/false))); + args.push_back(OS_STR("download")); auto [status, output] = runProgram({.program = "ssh", .args = args}); if (output.empty()) - throw Error("git-lfs-authenticate: no output (cmd: 'ssh %s')", concatStringsSep(" ", args)); + throw Error( + "git-lfs-authenticate: no output (cmd: 'ssh %s')", + concatMapStringsSep( + " ", args, [](const OsString & s) { return escapeShellArgAlways(os_string_to_string(s)); })); auto queryResp = nlohmann::json::parse(output); auto headerIt = queryResp.find("header"); @@ -268,12 +273,12 @@ void Fetch::fetch( return; } - std::filesystem::path cacheDir = getCacheDir() / "git-lfs"; + auto cacheDir = getCacheDir() / "git-lfs"; std::string key = hashString(HashAlgorithm::SHA256, pointerFilePath.rel()).to_string(HashFormat::Base16, false) + "/" + pointer->oid; - std::filesystem::path cachePath = cacheDir / key; + auto cachePath = cacheDir / key; if (pathExists(cachePath)) { - debug("using cache entry %s -> %s", key, cachePath); + debug("using cache entry %s -> %s", key, PathFmt(cachePath)); sink(readFile(cachePath)); return; } @@ -301,7 +306,7 @@ void Fetch::fetch( sizeCallback(size); downloadToSink(ourl, authHeader, sink, sha256, size); - debug("creating cache entry %s -> %s", key, cachePath); + debug("creating cache entry %s -> %s", key, PathFmt(cachePath)); if (!pathExists(cachePath.parent_path())) createDirs(cachePath.parent_path()); writeFile(cachePath, sink.s); diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc index f21313a10404..0b3e213b3ab9 100644 --- a/src/libfetchers/git-utils.cc +++ b/src/libfetchers/git-utils.cc @@ -4,6 +4,7 @@ #include "nix/fetchers/fetch-settings.hh" #include "nix/util/base-n.hh" #include "nix/util/finally.hh" +#include "nix/util/os-string.hh" #include "nix/util/processes.hh" #include "nix/util/signals.hh" #include "nix/util/users.hh" @@ -75,6 +76,29 @@ namespace nix { struct GitSourceAccessor; +struct GitError final : public CloneableError +{ + template + GitError(const git_error & error, Ts &&... args) + : CloneableError("") + { + auto hf = HintFmt(std::forward(args)...); + err.msg = HintFmt("%1%: %2% (libgit2 error code = %3%)", Uncolored(hf.str()), error.message, error.klass); + } + + template + GitError(Ts &&... args) + : GitError( + []() -> const git_error & { + const git_error * p = git_error_last(); + assert(p && "git_error_last() is unexpectedly null"); + return *p; + }(), + std::forward(args)...) + { + } +}; + typedef std::unique_ptr> Repository; typedef std::unique_ptr> TreeEntry; typedef std::unique_ptr> Tree; @@ -107,7 +131,7 @@ static void initLibGit2() static std::once_flag initialized; std::call_once(initialized, []() { if (git_libgit2_init() < 0) - throw Error("initialising libgit2: %s", git_error_last()->message); + throw GitError("initialising libgit2"); }); } @@ -115,7 +139,7 @@ static git_oid hashToOID(const Hash & hash) { git_oid oid; if (git_oid_fromstr(&oid, hash.gitRev().c_str())) - throw Error("cannot convert '%s' to a Git OID", hash.gitRev()); + throw GitError("cannot convert '%s' to a Git OID", hash.gitRev()); return oid; } @@ -123,8 +147,7 @@ static Object lookupObject(git_repository * repo, const git_oid & oid, git_objec { Object obj; if (git_object_lookup(Setter(obj), repo, &oid, type)) { - auto err = git_error_last(); - throw Error("getting Git object '%s': %s", oid, err->message); + throw GitError("getting Git object '%s'", oid); } return obj; } @@ -134,8 +157,7 @@ static T peelObject(git_object * obj, git_object_t type) { T obj2; if (git_object_peel((git_object **) (typename T::pointer *) Setter(obj2), obj, type)) { - auto err = git_error_last(); - throw Error("peeling Git object '%s': %s", *git_object_id(obj), err->message); + throw Error("peeling Git object '%s'", *git_object_id(obj)); } return obj2; } @@ -145,7 +167,7 @@ static T dupObject(typename T::pointer obj) { T obj2; if (git_object_dup((git_object **) (typename T::pointer *) Setter(obj2), (git_object *) obj)) - throw Error("duplicating object '%s': %s", *git_object_id((git_object *) obj), git_error_last()->message); + throw GitError("duplicating object '%s'", *git_object_id((git_object *) obj)); return obj2; } @@ -210,14 +232,14 @@ static void initRepoAtomically(std::filesystem::path & path, GitRepo::Options op return; if (!options.create) - throw Error("Git repository %s does not exist.", path); + throw Error("Git repository %s does not exist.", PathFmt(path)); std::filesystem::path tmpDir = createTempDir(path.parent_path()); AutoDelete delTmpDir(tmpDir, true); Repository tmpRepo; if (git_repository_init(Setter(tmpRepo), tmpDir.string().c_str(), options.bare)) - throw Error("creating Git repository %s: %s", path, git_error_last()->message); + throw GitError("creating Git repository %s", PathFmt(path)); try { std::filesystem::rename(tmpDir, path); } catch (std::filesystem::filesystem_error & e) { @@ -227,7 +249,8 @@ static void initRepoAtomically(std::filesystem::path & path, GitRepo::Options op || e.code() == std::errc::directory_not_empty) { return; } else - throw SysError("moving temporary git repository from %s to %s", tmpDir, path); + throw SystemError( + e.code(), "moving temporary git repository from %s to %s", PathFmt(tmpDir), PathFmt(path)); } // we successfully moved the repository, so the temporary directory no longer exists. delTmpDir.cancel(); @@ -267,7 +290,7 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this initRepoAtomically(path, options); if (git_repository_open(Setter(repo), path.string().c_str())) - throw Error("opening Git repository %s: %s", path, git_error_last()->message); + throw GitError("opening Git repository %s", PathFmt(path)); ObjectDb odb; if (options.packfilesOnly) { @@ -280,28 +303,28 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this */ if (git_odb_new(Setter(odb))) - throw Error("creating Git object database: %s", git_error_last()->message); + throw GitError("creating Git object database"); if (git_odb_backend_pack(&packBackend, (path / "objects").string().c_str())) - throw Error("creating pack backend: %s", git_error_last()->message); + throw GitError("creating pack backend"); if (git_odb_add_backend(odb.get(), packBackend, 1)) - throw Error("adding pack backend to Git object database: %s", git_error_last()->message); + throw GitError("adding pack backend to Git object database"); } else { if (git_repository_odb(Setter(odb), repo.get())) - throw Error("getting Git object database: %s", git_error_last()->message); + throw GitError("getting Git object database"); } // mempack_backend will be owned by the repository, so we are not expected to free it ourselves. if (git_mempack_new(&mempackBackend)) - throw Error("creating mempack backend: %s", git_error_last()->message); + throw GitError("creating mempack backend"); if (git_odb_add_backend(odb.get(), mempackBackend, 999)) - throw Error("adding mempack backend to Git object database: %s", git_error_last()->message); + throw GitError("adding mempack backend to Git object database"); if (options.packfilesOnly) { if (git_repository_set_odb(repo.get(), odb.get())) - throw Error("setting Git object database: %s", git_error_last()->message); + throw GitError("setting Git object database"); } } @@ -341,7 +364,7 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this Indexer indexer; git_indexer_progress stats; if (git_indexer_new(Setter(indexer), pack_dir_path.c_str(), 0, nullptr, nullptr)) - throw Error("creating git packfile indexer: %s", git_error_last()->message); + throw GitError("creating git packfile indexer"); // TODO: provide index callback for checkInterrupt() termination // though this is about an order of magnitude faster than the packbuilder @@ -349,15 +372,15 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this constexpr size_t chunkSize = 128 * 1024; for (size_t offset = 0; offset < buf.size; offset += chunkSize) { if (git_indexer_append(indexer.get(), buf.ptr + offset, std::min(chunkSize, buf.size - offset), &stats)) - throw Error("appending to git packfile index: %s", git_error_last()->message); + throw GitError("appending to git packfile index"); checkInterrupt(); } if (git_indexer_commit(indexer.get(), &stats)) - throw Error("committing git packfile index: %s", git_error_last()->message); + throw GitError("committing git packfile index"); if (git_mempack_reset(mempackBackend)) - throw Error("resetting git mempack backend: %s", git_error_last()->message); + throw GitError("resetting git mempack backend"); checkInterrupt(); } @@ -450,7 +473,7 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this void setRemote(const std::string & name, const std::string & url) override { if (git_remote_set_url(*this, name.c_str(), url.c_str())) - throw Error("setting remote '%s' URL to '%s': %s", name, url, git_error_last()->message); + throw GitError("setting remote '%s' URL to '%s'", name, url); } Hash resolveRef(std::string ref) override @@ -463,7 +486,7 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this // an object_id. std::string peeledRef = ref + "^{commit}"; if (git_revparse_single(Setter(object), *this, peeledRef.c_str())) - throw Error("resolving Git reference '%s': %s", ref, git_error_last()->message); + throw GitError("resolving Git reference '%s'", ref); auto oid = git_object_id(object.get()); return toHash(*oid); } @@ -472,11 +495,11 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this { GitConfig config; if (git_config_open_ondisk(Setter(config), configFile.string().c_str())) - throw Error("parsing .gitmodules file: %s", git_error_last()->message); + throw GitError("parsing .gitmodules file"); ConfigIterator it; if (git_config_iterator_glob_new(Setter(it), config.get(), "^submodule\\..*\\.(path|url|branch)$")) - throw Error("iterating over .gitmodules: %s", git_error_last()->message); + throw GitError("iterating over .gitmodules"); StringMap entries; @@ -485,7 +508,7 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this if (auto err = git_config_next(&entry, it.get())) { if (err == GIT_ITEROVER) break; - throw Error("iterating over .gitmodules: %s", git_error_last()->message); + throw GitError("iterating over .gitmodules"); } entries.emplace(entry->name + 10, entry->value); } @@ -522,7 +545,7 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this git_oid headRev; if (auto err = git_reference_name_to_id(&headRev, *this, "HEAD")) { if (err != GIT_ENOTFOUND) - throw Error("resolving HEAD: %s", git_error_last()->message); + throw GitError("resolving HEAD"); } else info.headRev = toHash(headRev); @@ -545,7 +568,7 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this options.flags |= GIT_STATUS_OPT_INCLUDE_UNMODIFIED; options.flags |= GIT_STATUS_OPT_EXCLUDE_SUBMODULES; if (git_status_foreach_ext(*this, &options, &statusCallbackTrampoline, &statusCallback)) - throw Error("getting working directory status: %s", git_error_last()->message); + throw GitError("getting working directory status"); /* Get submodule info. */ auto modulesFile = path / ".gitmodules"; @@ -588,8 +611,7 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this if (auto errCode = git_object_lookup(Setter(obj), *this, &oid, GIT_OBJECT_ANY)) { if (errCode == GIT_ENOTFOUND) return false; - auto err = git_error_last(); - throw Error("getting Git object '%s': %s", oid, err->message); + throw GitError("getting Git object '%s'", oid); } return true; @@ -618,10 +640,14 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this if (ExecutablePath::load().findName("git")) { auto dir = this->path; - Strings gitArgs{"-C", dir.string(), "--git-dir", ".", "fetch", "--progress", "--force"}; - if (shallow) - append(gitArgs, {"--depth", "1"}); - append(gitArgs, {std::string("--"), url, refspec}); + OsStrings gitArgs{"-C", dir.native(), "--git-dir", ".", "fetch", "--progress", "--force"}; + if (shallow) { + gitArgs.push_back(OS_STR("--depth")); + gitArgs.push_back(OS_STR("1")); + } + gitArgs.push_back(OS_STR("--")); + gitArgs.push_back(string_to_os_string(url)); + gitArgs.push_back(string_to_os_string(refspec)); auto status = runProgram(RunOptions{.program = "git", .args = gitArgs, .isInteractive = true}).first; @@ -683,18 +709,18 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this writeFile(allowedSignersFile, allowedSigners); // Run verification command - auto [status, output] = runProgram( - RunOptions{ - .program = "git", - .args = - {"-c", - "gpg.ssh.allowedSignersFile=" + allowedSignersFile, - "-C", - path.string(), - "verify-commit", - rev.gitRev()}, - .mergeStderrToStdout = true, - }); + auto [status, output] = runProgram({ + .program = "git", + .args{ + OS_STR("-c"), + OS_STR("gpg.ssh.allowedSignersFile=") + allowedSignersFile.native(), + OS_STR("-C"), + path.native(), + OS_STR("verify-commit"), + string_to_os_string(rev.gitRev()), + }, + .mergeStderrToStdout = true, + }); /* Evaluate result through status code and checking if public key fingerprints appear on stderr. This is necessary @@ -793,7 +819,7 @@ struct GitSourceAccessor : SourceAccessor fingerprint = options.makeFingerprint(rev); } - std::string readBlob(const CanonPath & path, bool symlink) + void readBlob(const CanonPath & path, bool symlink, Sink & sink, std::function sizeCallback) { auto state(state_.lock()); @@ -812,16 +838,22 @@ struct GitSourceAccessor : SourceAccessor e.addTrace({}, "while smudging git-lfs file '%s'", path); throw; } - return s.s; + sizeCallback(s.s.size()); + StringSource source{s.s}; + source.drainInto(sink); + return; } } - return std::string((const char *) git_blob_rawcontent(blob.get()), git_blob_rawsize(blob.get())); + auto view = std::string_view((const char *) git_blob_rawcontent(blob.get()), git_blob_rawsize(blob.get())); + sizeCallback(view.size()); + StringSource source{view}; + source.drainInto(sink); } - std::string readFile(const CanonPath & path) override + void readFile(const CanonPath & path, Sink & sink, fun sizeCallback) override { - return readBlob(path, false); + return readBlob(path, false, sink, sizeCallback); } bool pathExists(const CanonPath & path) override @@ -888,7 +920,9 @@ struct GitSourceAccessor : SourceAccessor std::string readLink(const CanonPath & path) override { - return readBlob(path, true); + StringSink s; + readBlob(path, true, s, [&](uint64_t size) { s.s.reserve(size); }); + return std::move(s.s); } /** @@ -937,7 +971,7 @@ struct GitSourceAccessor : SourceAccessor TreeEntry copy; if (git_tree_entry_dup(Setter(copy), entry)) - throw Error("dupping tree entry: %s", git_error_last()->message); + throw GitError("dupping tree entry"); auto entryName = std::string_view(git_tree_entry_name(entry)); @@ -967,7 +1001,7 @@ struct GitSourceAccessor : SourceAccessor Tree tree; if (git_tree_entry_to_object((git_object **) (git_tree **) Setter(tree), *state.repo, entry)) - throw Error("looking up directory '%s': %s", showPath(path), git_error_last()->message); + throw GitError("looking up directory '%s'", showPath(path)); return tree; } @@ -1002,7 +1036,7 @@ struct GitSourceAccessor : SourceAccessor Tree tree; if (git_tree_entry_to_object((git_object **) (git_tree **) Setter(tree), *state.repo, entry)) - throw Error("looking up directory '%s': %s", showPath(path), git_error_last()->message); + throw GitError("looking up directory '%s'", showPath(path)); return tree; } @@ -1035,7 +1069,7 @@ struct GitSourceAccessor : SourceAccessor Blob blob; if (git_tree_entry_to_object((git_object **) (git_blob **) Setter(blob), *state.repo, entry)) - throw Error("looking up file '%s': %s", showPath(path), git_error_last()->message); + throw GitError("looking up file '%s'", showPath(path)); return blob; } @@ -1087,7 +1121,7 @@ struct GitExportIgnoreSourceAccessor : CachingFilteringSourceAccessor if (git_error_last()->klass == GIT_ENOTFOUND) return false; else - throw Error("looking up '%s': %s", showPath(path), git_error_last()->message); + throw GitError("looking up '%s'", showPath(path)); } else { // Official git will silently reject export-ignore lines that have // values. We do the same. @@ -1199,7 +1233,7 @@ struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink cur->children.insert_or_assign(name, std::move(child)); } - void createRegularFile(const CanonPath & path, std::function func) override + void createRegularFile(const CanonPath & path, fun func) override { checkInterrupt(); @@ -1243,17 +1277,17 @@ struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink repo.emplace(parent.repoPool.get()); if (git_blob_create_from_stream(Setter(stream), **repo, nullptr)) - throw Error("creating a blob stream object: %s", git_error_last()->message); + throw GitError("creating a blob stream object"); if (stream->write(stream.get(), contents.data(), contents.size())) - throw Error("writing a blob for tarball member '%s': %s", path, git_error_last()->message); + throw GitError("writing a blob for tarball member '%s'", path); parent.totalBufSize -= contents.size(); contents.clear(); } } else { if (stream->write(stream.get(), data.data(), data.size())) - throw Error("writing a blob for tarball member '%s': %s", path, git_error_last()->message); + throw GitError("writing a blob for tarball member '%s'", path); } } @@ -1275,7 +1309,7 @@ struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink acquires ownership and frees the stream. */ git_oid oid; if (git_blob_create_from_stream_commit(&oid, crf->stream.release())) - throw Error("creating a blob object for '%s': %s", path, git_error_last()->message); + throw GitError("creating a blob object for '%s'", path); addNode( *_state.lock(), crf->path, @@ -1289,8 +1323,7 @@ struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink git_oid oid; if (git_blob_create_from_buffer(&oid, *repo, crf->contents.data(), crf->contents.size())) - throw Error( - "creating a blob object for '%s' from in-memory buffer: %s", crf->path, git_error_last()->message); + throw GitError("creating a blob object for '%s' from in-memory buffer", crf->path); addNode( *_state.lock(), @@ -1314,8 +1347,7 @@ struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink git_oid oid; if (git_blob_create_from_buffer(&oid, *repo, target.c_str(), target.size())) - throw Error( - "creating a blob object for tarball symlink member '%s': %s", path, git_error_last()->message); + throw GitError("creating a blob object for tarball symlink member '%s'", path); auto state(_state.lock()); addNode(*state, path, Child{GIT_FILEMODE_LINK, oid}); @@ -1376,19 +1408,19 @@ struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink // Write this directory. git_treebuilder * b; if (git_treebuilder_new(&b, *repo, nullptr)) - throw Error("creating a tree builder: %s", git_error_last()->message); + throw GitError("creating a tree builder"); TreeBuilder builder(b); for (auto & [name, child] : node.children) { auto oid_p = std::get_if(&child.file); auto oid = oid_p ? *oid_p : std::get(child.file).oid.value(); if (git_treebuilder_insert(nullptr, builder.get(), name.c_str(), &oid, child.mode)) - throw Error("adding a file to a tree builder: %s", git_error_last()->message); + throw GitError("adding a file to a tree builder"); } git_oid oid; if (git_treebuilder_write(&oid, builder.get())) - throw Error("creating a tree object: %s", git_error_last()->message); + throw GitError("creating a tree object"); node.oid = oid; }(_state.lock()->root); @@ -1452,7 +1484,7 @@ std::vector> GitRepoImpl::getSubmodules auto [fdTemp, pathTemp] = createTempFile("nix-git-submodules"); try { writeFull(fdTemp.get(), configS); - } catch (SysError & e) { + } catch (SystemError & e) { e.addTrace({}, "while writing .gitmodules file to temporary file"); throw; } diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 2cdcb6ef7c24..03c4b55a1022 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -7,6 +7,7 @@ #include "nix/store/store-api.hh" #include "nix/util/url-parts.hh" #include "nix/store/pathlocks.hh" +#include "nix/util/os-string.hh" #include "nix/util/processes.hh" #include "nix/util/git.hh" #include "nix/fetchers/git-utils.hh" @@ -32,15 +33,16 @@ namespace nix::fetchers { namespace { -bool isCacheFileWithinTtl(time_t now, const struct stat & st) +static bool isCacheFileWithinTtl(const Settings & settings, time_t now, const PosixStat & st) { return st.st_mtime + static_cast(settings.tarballTtl) > now; } std::filesystem::path getCachePath(std::string_view key, bool shallow) { - return getCacheDir() / "gitv3" - / (hashString(HashAlgorithm::SHA256, key).to_string(HashFormat::Nix32, false) + (shallow ? "-shallow" : "")); + auto name = + hashString(HashAlgorithm::SHA256, key).to_string(HashFormat::Nix32, false) + (shallow ? "-shallow" : ""); + return getCacheDir() / "gitv3" / std::move(name); } // Returns the name of the HEAD branch. @@ -56,7 +58,7 @@ std::optional readHead(const std::filesystem::path & path) RunOptions{ .program = "git", // FIXME: use 'HEAD' to avoid returning all refs - .args = {"ls-remote", "--symref", path.string()}, + .args = {OS_STR("ls-remote"), OS_STR("--symref"), path.native()}, .isInteractive = true, }); if (status != 0) @@ -67,10 +69,10 @@ std::optional readHead(const std::filesystem::path & path) if (const auto parseResult = git::parseLsRemoteLine(line); parseResult && parseResult->reference == "HEAD") { switch (parseResult->kind) { case git::LsRemoteRefLine::Kind::Symbolic: - debug("resolved HEAD ref '%s' for repo '%s'", parseResult->target, path); + debug("resolved HEAD ref '%s' for repo %s", parseResult->target, PathFmt(path)); break; case git::LsRemoteRefLine::Kind::Object: - debug("resolved HEAD rev '%s' for repo '%s'", parseResult->target, path); + debug("resolved HEAD rev '%s' for repo %s", parseResult->target, PathFmt(path)); break; } return parseResult->target; @@ -83,7 +85,19 @@ bool storeCachedHead(const std::string & actualUrl, bool shallow, const std::str { std::filesystem::path cacheDir = getCachePath(actualUrl, shallow); try { - runProgram("git", true, {"-C", cacheDir.string(), "--git-dir", ".", "symbolic-ref", "--", "HEAD", headRef}); + runProgram( + "git", + true, + { + OS_STR("-C"), + cacheDir.native(), + OS_STR("--git-dir"), + OS_STR("."), + OS_STR("symbolic-ref"), + OS_STR("--"), + OS_STR("HEAD"), + string_to_os_string(headRef), + }); } catch (ExecError & e) { if ( #ifndef WIN32 // TODO abstract over exit status handling on Windows @@ -100,19 +114,19 @@ bool storeCachedHead(const std::string & actualUrl, bool shallow, const std::str return true; } -std::optional readHeadCached(const std::string & actualUrl, bool shallow) +static std::optional readHeadCached(const Settings & settings, const std::string & actualUrl, bool shallow) { // Create a cache path to store the branch of the HEAD ref. Append something // in front of the URL to prevent collision with the repository itself. std::filesystem::path cacheDir = getCachePath(actualUrl, shallow); std::filesystem::path headRefFile = cacheDir / "HEAD"; - time_t now = time(0); - struct stat st; + time_t now = time(nullptr); + auto st = maybeStat(headRefFile); std::optional cachedRef; - if (stat(headRefFile.string().c_str(), &st) == 0) { + if (st) { cachedRef = readHead(cacheDir); - if (cachedRef != std::nullopt && isCacheFileWithinTtl(now, st)) { + if (cachedRef != std::nullopt && isCacheFileWithinTtl(settings, now, *st)) { debug("using cached HEAD ref '%s' for repo '%s'", *cachedRef, actualUrl); return cachedRef; } @@ -435,19 +449,19 @@ struct GitInputScheme : InputScheme { auto repoInfo = getRepoInfo(input); - Strings args = {"clone"}; + OsStrings args = {OS_STR("clone")}; - args.push_back(repoInfo.locationToArg()); + args.push_back(string_to_os_string(repoInfo.locationToArg())); if (auto ref = input.getRef()) { - args.push_back("--branch"); - args.push_back(*ref); + args.push_back(OS_STR("--branch")); + args.push_back(string_to_os_string(*ref)); } if (input.getRev()) throw UnimplementedError("cloning a specific revision is not implemented"); - args.push_back(destDir.string()); + args.push_back(destDir.native()); runProgram("git", true, args, {}, true); } @@ -474,14 +488,15 @@ struct GitInputScheme : InputScheme auto result = runProgram( RunOptions{ .program = "git", - .args = - {"-C", - repoPath->string(), - "--git-dir", - repoInfo.gitDir, - "check-ignore", - "--quiet", - std::string(path.rel())}, + .args{ + OS_STR("-C"), + repoPath->native(), + OS_STR("--git-dir"), + string_to_os_string(repoInfo.gitDir), + OS_STR("check-ignore"), + OS_STR("--quiet"), + string_to_os_string(std::string(path.rel())), + }, }); auto exitCode = #ifndef WIN32 // TODO abstract over exit status handling on Windows @@ -496,14 +511,16 @@ struct GitInputScheme : InputScheme runProgram( "git", true, - {"-C", - repoPath->string(), - "--git-dir", - repoInfo.gitDir, - "add", - "--intent-to-add", - "--", - std::string(path.rel())}); + { + OS_STR("-C"), + repoPath->native(), + OS_STR("--git-dir"), + string_to_os_string(repoInfo.gitDir), + OS_STR("add"), + OS_STR("--intent-to-add"), + OS_STR("--"), + string_to_os_string(std::string(path.rel())), + }); if (commitMsg) { // Pause the logger to allow for user input (such as a gpg passphrase) in `git commit` @@ -511,14 +528,16 @@ struct GitInputScheme : InputScheme runProgram( "git", true, - {"-C", - repoPath->string(), - "--git-dir", - repoInfo.gitDir, - "commit", - std::string(path.rel()), - "-F", - "-"}, + { + OS_STR("-C"), + repoPath->native(), + OS_STR("--git-dir"), + string_to_os_string(repoInfo.gitDir), + OS_STR("commit"), + string_to_os_string(std::string(path.rel())), + OS_STR("-F"), + OS_STR("-"), + }, *commitMsg); } } @@ -613,7 +632,9 @@ struct GitInputScheme : InputScheme // Why are we checking for bare repository? // well if it's a bare repository we want to force a git fetch rather than copying the folder - auto isBareRepository = [](PathView path) { return pathExists(path) && !pathExists(path + "/.git"); }; + auto isBareRepository = [](const std::filesystem::path & path) { + return pathExists(path) && !pathExists(path / ".git"); + }; // FIXME: here we turn a possibly relative path into an absolute path. // This allows relative git flake inputs to be resolved against the @@ -623,10 +644,12 @@ struct GitInputScheme : InputScheme // // See: https://discourse.nixos.org/t/57783 and #9708 // - if (url.scheme == "file" && !forceHttp && !isBareRepository(renderUrlPathEnsureLegal(url.path))) { - auto path = renderUrlPathEnsureLegal(url.path); + auto maybeUrlFsPathForFileUrl = + url.scheme == "file" ? std::make_optional(urlPathToPath(url.path)) : std::nullopt; + if (maybeUrlFsPathForFileUrl && !forceHttp && !isBareRepository(*maybeUrlFsPathForFileUrl)) { + auto & path = *maybeUrlFsPathForFileUrl; - if (!isAbsolute(path)) { + if (!path.is_absolute()) { warn( "Fetching Git repository '%s', which uses a path relative to the current directory. " "This is not supported and will stop working in a future release. " @@ -636,7 +659,7 @@ struct GitInputScheme : InputScheme repoInfo.location = std::filesystem::absolute(path); } else { - if (url.scheme == "file") + if (maybeUrlFsPathForFileUrl) /* Query parameters are meaningless for file://, but Git interprets them as part of the file name. So get rid of them. */ @@ -725,12 +748,12 @@ struct GitInputScheme : InputScheme return revCount; } - std::string getDefaultRef(const RepoInfo & repoInfo, bool shallow) const + std::string getDefaultRef(const Settings & settings, const RepoInfo & repoInfo, bool shallow) const { auto head = std::visit( overloaded{ [&](const std::filesystem::path & path) { return GitRepo::openRepo(path, {})->getWorkdirRef(); }, - [&](const ParsedURL & url) { return readHeadCached(url.to_string(), shallow); }}, + [&](const ParsedURL & url) { return readHeadCached(settings, url.to_string(), shallow); }}, repoInfo.location); if (!head) { warn("could not read HEAD ref from repo at '%s', using 'master'", repoInfo.locationToArg()); @@ -750,9 +773,10 @@ struct GitInputScheme : InputScheme "\n" "git -C %2% add \"%1%\"", path.rel(), - repoPath); + PathFmt(repoPath)); else - return RestrictedPathError("Path '%s' does not exist in Git repository %s.", path.rel(), repoPath); + return RestrictedPathError( + "Path '%s' does not exist in Git repository %s.", path.rel(), PathFmt(repoPath)); }; } @@ -806,7 +830,8 @@ struct GitInputScheme : InputScheme auto fingerprint = options.makeFingerprint(rev) + ";legacy"; - auto cacheKey = makeSourcePathToHashCacheKey(fingerprint, ContentAddressMethod::Raw::NixArchive, "/"); + auto cacheKey = + makeSourcePathToHashCacheKey(fingerprint, ContentAddressMethod::Raw::NixArchive, CanonPath::root); auto makeAccessor = [&](const auto & storePath) -> ref { auto accessor = store.getFSAccessor(storePath); @@ -839,7 +864,7 @@ struct GitInputScheme : InputScheme runProgram( {.program = "git", .args = {"-C", tmpDir, "fetch", "--quiet", "origin", rev.gitRev()}}); runProgram({.program = "git", .args = {"-C", tmpDir, "checkout", "--quiet", rev.gitRev()}}); - PathFilter filter = [&](const Path & path) { return baseNameOf(path) != ".git"; }; + PathFilter filter = [&](const std::string & path) { return baseNameOf(path) != ".git"; }; return store.addToStore( "source", {getFSSourceAccessor(), CanonPath(tmpDir.string())}, @@ -877,7 +902,7 @@ struct GitInputScheme : InputScheme auto originalRef = input.getRef(); bool shallow = canDoShallow(input); - auto ref = originalRef ? *originalRef : getDefaultRef(repoInfo, shallow); + auto ref = originalRef ? *originalRef : getDefaultRef(settings, repoInfo, shallow); input.attrs.insert_or_assign("ref", ref); std::filesystem::path repoDir; @@ -919,7 +944,7 @@ struct GitInputScheme : InputScheme auto localRefFile = ref.compare(0, 5, "refs/") == 0 ? cacheDir / ref : cacheDir / "refs/heads" / ref; bool doFetch = false; - time_t now = time(0); + time_t now = time(nullptr); /* If a rev was specified, we need to fetch if it's not in the repo. */ @@ -929,10 +954,10 @@ struct GitInputScheme : InputScheme if (getAllRefsAttr(input)) { doFetch = true; } else { - /* If the local ref is older than ‘tarball-ttl’ seconds, do a + /* If the local ref is older than 'tarball-ttl' seconds, do a git fetch to update the local ref to the remote ref. */ - struct stat st; - doFetch = stat(localRefFile.string().c_str(), &st) != 0 || !isCacheFileWithinTtl(now, st); + auto st = maybeStat(localRefFile); + doFetch = !st || !isCacheFileWithinTtl(settings, now, *st); } } @@ -961,7 +986,7 @@ struct GitInputScheme : InputScheme if (!input.getRev()) setWriteTime(localRefFile, now, now); } catch (Error & e) { - warn("could not update mtime for file %s: %s", localRefFile, e.info().msg); + warn("could not update mtime for file %s: %s", PathFmt(localRefFile), e.info().msg); } if (!originalRef && !storeCachedHead(repoUrl.to_string(), shallow, ref)) warn("could not update cached head '%s' for '%s'", ref, repoInfo.locationToArg()); diff --git a/src/libfetchers/include/nix/fetchers/fetch-settings.hh b/src/libfetchers/include/nix/fetchers/fetch-settings.hh index e2268203b56b..0e7edaa00e68 100644 --- a/src/libfetchers/include/nix/fetchers/fetch-settings.hh +++ b/src/libfetchers/include/nix/fetchers/fetch-settings.hh @@ -133,6 +133,25 @@ struct Settings : public Config The resulting locks may not be compatible with Nix >= 2.20. )"}; + Setting tarballTtl{ + this, + 60 * 60, + "tarball-ttl", + R"( + The number of seconds a downloaded tarball is considered fresh. If + the cached tarball is stale, Nix checks whether it is still up + to date using the ETag header. Nix downloads a new version if + the ETag header is unsupported, or the cached ETag doesn't match. + + Setting the TTL to `0` forces Nix to always check if the tarball is + up to date. + + Nix caches tarballs in `$XDG_CACHE_HOME/nix/tarballs`. + + Files fetched via `NIX_PATH`, `fetchGit`, `fetchMercurial`, + `fetchTarball`, and `fetchurl` respect this TTL. + )"}; + ref getCache() const; ref getTarballCache() const; diff --git a/src/libfetchers/include/nix/fetchers/fetch-to-store.hh b/src/libfetchers/include/nix/fetchers/fetch-to-store.hh index e7f880724911..0fc20a98e125 100644 --- a/src/libfetchers/include/nix/fetchers/fetch-to-store.hh +++ b/src/libfetchers/include/nix/fetchers/fetch-to-store.hh @@ -35,6 +35,6 @@ std::pair fetchToStore2( RepairFlag repair = NoRepair); fetchers::Cache::Key -makeSourcePathToHashCacheKey(const std::string & fingerprint, ContentAddressMethod method, const std::string & path); +makeSourcePathToHashCacheKey(std::string_view fingerprint, ContentAddressMethod method, const CanonPath & path); } // namespace nix diff --git a/src/libfetchers/include/nix/fetchers/fetchers.hh b/src/libfetchers/include/nix/fetchers/fetchers.hh index 0f7d933131e9..d830d83c840a 100644 --- a/src/libfetchers/include/nix/fetchers/fetchers.hh +++ b/src/libfetchers/include/nix/fetchers/fetchers.hh @@ -86,7 +86,7 @@ public: * Only for relative path flakes, i.e. 'path:./foo', returns the * relative path, i.e. './foo'. */ - std::optional isRelative() const; + std::optional isRelative() const; /** * Return whether this is a "final" input, meaning that fetching @@ -274,7 +274,7 @@ struct InputScheme return false; } - virtual std::optional isRelative(const Input & input) const + virtual std::optional isRelative(const Input & input) const { return std::nullopt; } diff --git a/src/libfetchers/include/nix/fetchers/filtering-source-accessor.hh b/src/libfetchers/include/nix/fetchers/filtering-source-accessor.hh index b53c8db5bd74..67fb29228ea8 100644 --- a/src/libfetchers/include/nix/fetchers/filtering-source-accessor.hh +++ b/src/libfetchers/include/nix/fetchers/filtering-source-accessor.hh @@ -11,7 +11,7 @@ namespace nix { * `RestrictedPathError` explaining that access to `path` is * forbidden. */ -typedef std::function MakeNotAllowedError; +typedef fun MakeNotAllowedError; /** * An abstract wrapping `SourceAccessor` that performs access @@ -34,9 +34,9 @@ struct FilteringSourceAccessor : SourceAccessor std::optional getPhysicalPath(const CanonPath & path) override; - std::string readFile(const CanonPath & path) override; + using SourceAccessor::readFile; - void readFile(const CanonPath & path, Sink & sink, std::function sizeCallback) override; + void readFile(const CanonPath & path, Sink & sink, fun sizeCallback) override; bool pathExists(const CanonPath & path) override; diff --git a/src/libfetchers/include/nix/fetchers/tarball.hh b/src/libfetchers/include/nix/fetchers/tarball.hh index e9e569d3c7fb..84aa8e5adc6e 100644 --- a/src/libfetchers/include/nix/fetchers/tarball.hh +++ b/src/libfetchers/include/nix/fetchers/tarball.hh @@ -6,6 +6,7 @@ #include "nix/store/path.hh" #include "nix/util/ref.hh" #include "nix/util/types.hh" +#include "nix/util/url.hh" namespace nix { class Store; @@ -27,7 +28,7 @@ struct DownloadFileResult DownloadFileResult downloadFile( Store & store, const Settings & settings, - const std::string & url, + const VerbatimURL & url, const std::string & name, const Headers & headers = {}); diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc index f9297ce8c2f1..a53025dd7ff9 100644 --- a/src/libfetchers/mercurial.cc +++ b/src/libfetchers/mercurial.cc @@ -1,5 +1,10 @@ #include "nix/fetchers/fetchers.hh" +#include "nix/util/file-system.hh" +#include "nix/util/fmt.hh" +#include "nix/util/os-string.hh" #include "nix/util/processes.hh" +#include "nix/util/util.hh" +#include "nix/util/environment-variables.hh" #include "nix/util/users.hh" #include "nix/fetchers/cache.hh" #include "nix/store/globals.hh" @@ -9,24 +14,25 @@ #include "nix/fetchers/fetch-settings.hh" #include +#include using namespace std::string_literals; namespace nix::fetchers { -static RunOptions hgOptions(const Strings & args) +static RunOptions hgOptions(OsStrings args) { - auto env = getEnv(); + auto env = getEnvOs(); // Set HGPLAIN: this means we get consistent output from hg and avoids leakage from a user or system .hgrc. - env["HGPLAIN"] = ""; + env[OS_STR("HGPLAIN")] = OS_STR(""); - return {.program = "hg", .lookupPath = true, .args = args, .environment = env}; + return {.program = "hg", .lookupPath = true, .args = std::move(args), .environment = env}; } // runProgram wrapper that uses hgOptions instead of stock RunOptions. -static std::string runHg(const Strings & args, const std::optional & input = {}) +static std::string runHg(OsStrings args, const std::optional & input = {}) { - RunOptions opts = hgOptions(args); + RunOptions opts = hgOptions(std::move(args)); opts.input = input; auto res = runProgram(std::move(opts)); @@ -144,7 +150,7 @@ struct MercurialInputScheme : InputScheme { auto url = parseURL(getStrAttr(input.attrs, "url")); if (url.scheme == "file" && !input.getRef() && !input.getRev()) - return renderUrlPathEnsureLegal(url.path); + return urlPathToPath(url.path); return {}; } @@ -154,29 +160,36 @@ struct MercurialInputScheme : InputScheme std::string_view contents, std::optional commitMsg) const override { - auto [isLocal, repoPath] = getActualUrl(input); - if (!isLocal) - throw Error( - "cannot commit '%s' to Mercurial repository '%s' because it's not a working tree", - path, - input.to_string()); - - auto absPath = CanonPath(repoPath) / path; - - writeFile(absPath.abs(), contents); - - // FIXME: shut up if file is already tracked. - runHg({"add", absPath.abs()}); - - if (commitMsg) - runHg({"commit", absPath.abs(), "-m", *commitMsg}); + std::visit( + overloaded{ + [&](const std::filesystem::path & repoPath) { + auto absPath = repoPath / path.rel(); + + writeFile(absPath, contents); + + // FIXME: shut up if file is already tracked. + runHg({OS_STR("add"), absPath.native()}); + + if (commitMsg) + runHg({OS_STR("commit"), absPath.native(), OS_STR("-m"), string_to_os_string(*commitMsg)}); + }, + [&](const std::string &) { + throw Error( + "cannot commit '%s' to Mercurial repository '%s' because it's not a working tree", + path, + input.to_string()); + }, + }, + getActualUrl(input)); } - std::pair getActualUrl(const Input & input) const + std::variant getActualUrl(const Input & input) const { auto url = parseURL(getStrAttr(input.attrs, "url")); - bool isLocal = url.scheme == "file"; - return {isLocal, isLocal ? renderUrlPathEnsureLegal(url.path) : url.to_string()}; + if (url.scheme == "file") + return urlPathToPath(url.path); + else + return url.to_string(); } StorePath fetchToStore(const Settings & settings, Store & store, Input & input) const @@ -185,37 +198,54 @@ struct MercurialInputScheme : InputScheme auto name = input.getName(); - auto [isLocal, actualUrl_] = getActualUrl(input); - auto actualUrl = actualUrl_; // work around clang bug + auto actualUrl_ = getActualUrl(input); // FIXME: return lastModified. // FIXME: don't clone local repositories. - if (!input.getRef() && !input.getRev() && isLocal && pathExists(actualUrl + "/.hg")) { - - bool clean = runHg({"status", "-R", actualUrl, "--modified", "--added", "--removed"}) == ""; - - if (!clean) { - + if (auto * localPathP = std::get_if(&actualUrl_)) { + auto & localPath = *localPathP; + auto unlocked = !input.getRef() && !input.getRev(); + auto isValidLocalRepo = pathExists(localPath / ".hg"); + // short circuiting to not bother checking if locked / no repo is important. + bool dirty = unlocked && isValidLocalRepo + && runHg({ + OS_STR("status"), + OS_STR("-R"), + localPath.native(), + OS_STR("--modified"), + OS_STR("--added"), + OS_STR("--removed"), + }) != ""; + if (dirty) { /* This is an unclean working tree. So copy all tracked files. */ if (!settings.allowDirty) - throw Error("Mercurial tree '%s' is unclean", actualUrl); + throw Error("Mercurial tree '%s' is unclean", PathFmt{localPath}); if (settings.warnDirty) - warn("Mercurial tree '%s' is unclean", actualUrl); + warn("Mercurial tree '%s' is unclean", PathFmt{localPath}); - input.attrs.insert_or_assign("ref", chomp(runHg({"branch", "-R", actualUrl}))); + input.attrs.insert_or_assign("ref", chomp(runHg({OS_STR("branch"), OS_STR("-R"), localPath.native()}))); auto files = tokenizeString( - runHg({"status", "-R", actualUrl, "--clean", "--modified", "--added", "--no-status", "--print0"}), + runHg({ + OS_STR("status"), + OS_STR("-R"), + localPath.native(), + OS_STR("--clean"), + OS_STR("--modified"), + OS_STR("--added"), + OS_STR("--no-status"), + OS_STR("--print0"), + }), "\0"s); - std::filesystem::path actualPath(absPath(actualUrl)); + auto actualPath = absPath(localPath); - PathFilter filter = [&](const Path & p) -> bool { + PathFilter filter = [&](const std::string & p) -> bool { assert(hasPrefix(p, actualPath.string())); std::string file(p, actualPath.string().size() + 1); @@ -230,18 +260,23 @@ struct MercurialInputScheme : InputScheme return files.count(file); }; - auto storePath = store.addToStore( + return store.addToStore( input.getName(), {getFSSourceAccessor(), CanonPath(actualPath.string())}, ContentAddressMethod::Raw::NixArchive, HashAlgorithm::SHA256, {}, filter); - - return storePath; } } + auto [actualUrl, actualUrlOs] = std::visit( + overloaded{ + [&](const std::filesystem::path & p) { return std::make_pair(p.string(), p.native()); }, + [&](const std::string & s) { return std::make_pair(s, string_to_os_string(s)); }, + }, + actualUrl_); + if (!input.getRef()) input.attrs.insert_or_assign("ref", "default"); @@ -279,40 +314,48 @@ struct MercurialInputScheme : InputScheme /* If this is a commit hash that we already have, we don't have to pull again. */ if (!(input.getRev() && pathExists(cacheDir) - && runProgram( - hgOptions({"log", "-R", cacheDir.string(), "-r", input.getRev()->gitRev(), "--template", "1"})) + && runProgram(hgOptions({ + OS_STR("log"), + OS_STR("-R"), + cacheDir.native(), + OS_STR("-r"), + string_to_os_string(input.getRev()->gitRev()), + OS_STR("--template"), + OS_STR("1"), + })) .second == "1")) { Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Mercurial repository '%s'", actualUrl)); if (pathExists(cacheDir)) { try { - runHg({"pull", "-R", cacheDir.string(), "--", actualUrl}); + runHg({OS_STR("pull"), OS_STR("-R"), cacheDir.native(), OS_STR("--"), actualUrlOs}); } catch (ExecError & e) { auto transJournal = cacheDir / ".hg" / "store" / "journal"; /* hg throws "abandoned transaction" error only if this file exists */ if (pathExists(transJournal)) { - runHg({"recover", "-R", cacheDir.string()}); - runHg({"pull", "-R", cacheDir.string(), "--", actualUrl}); + runHg({OS_STR("recover"), OS_STR("-R"), cacheDir.native()}); + runHg({OS_STR("pull"), OS_STR("-R"), cacheDir.native(), OS_STR("--"), actualUrlOs}); } else { throw ExecError(e.status, "'hg pull' %s", statusToString(e.status)); } } } else { - createDirs(dirOf(cacheDir.string())); - runHg({"clone", "--noupdate", "--", actualUrl, cacheDir.string()}); + createDirs(cacheDir.parent_path()); + runHg({OS_STR("clone"), OS_STR("--noupdate"), OS_STR("--"), actualUrlOs, cacheDir.native()}); } } /* Fetch the remote rev or ref. */ - auto tokens = tokenizeString>(runHg( - {"log", - "-R", - cacheDir.string(), - "-r", - input.getRev() ? input.getRev()->gitRev() : *input.getRef(), - "--template", - "{node} {rev} {branch}"})); + auto tokens = tokenizeString>(runHg({ + OS_STR("log"), + OS_STR("-R"), + cacheDir.native(), + OS_STR("-r"), + string_to_os_string(input.getRev() ? input.getRev()->gitRev() : *input.getRef()), + OS_STR("--template"), + OS_STR("{node} {rev} {branch}"), + })); assert(tokens.size() == 3); auto rev = Hash::parseAny(tokens[0], HashAlgorithm::SHA1); @@ -328,7 +371,14 @@ struct MercurialInputScheme : InputScheme std::filesystem::path tmpDir = createTempDir(); AutoDelete delTmpDir(tmpDir, true); - runHg({"archive", "-R", cacheDir.string(), "-r", rev.gitRev(), tmpDir.string()}); + runHg({ + OS_STR("archive"), + OS_STR("-R"), + cacheDir.native(), + OS_STR("-r"), + string_to_os_string(rev.gitRev()), + tmpDir.native(), + }); deletePath(tmpDir / ".hg_archival.txt"); diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc index dccc75a89875..30f02c731f7f 100644 --- a/src/libfetchers/path.cc +++ b/src/libfetchers/path.cc @@ -19,7 +19,7 @@ struct PathInputScheme : InputScheme Input input{}; input.attrs.insert_or_assign("type", "path"); - input.attrs.insert_or_assign("path", renderUrlPathEnsureLegal(url.path)); + input.attrs.insert_or_assign("path", urlPathToPath(url.path).string()); for (auto & [name, value] : url.query) if (name == "rev" || name == "narHash") @@ -114,10 +114,10 @@ struct PathInputScheme : InputScheme writeFile(getAbsPath(input) / path.rel(), contents); } - std::optional isRelative(const Input & input) const override + std::optional isRelative(const Input & input) const override { - auto path = getStrAttr(input.attrs, "path"); - if (isAbsolute(path)) + std::filesystem::path path = getStrAttr(input.attrs, "path"); + if (path.is_absolute()) return std::nullopt; else return path; @@ -130,9 +130,9 @@ struct PathInputScheme : InputScheme std::filesystem::path getAbsPath(const Input & input) const { - auto path = getStrAttr(input.attrs, "path"); + std::filesystem::path path = getStrAttr(input.attrs, "path"); - if (isAbsolute(path)) + if (path.is_absolute()) return canonPath(path); throw Error("cannot fetch input '%s' because it uses a relative path", input.to_string()); @@ -162,7 +162,8 @@ struct PathInputScheme : InputScheme if (info) { accessor->fingerprint = fmt("path:%s", info->narHash.to_string(HashFormat::SRI, true)); settings.getCache()->upsert( - makeSourcePathToHashCacheKey(*accessor->fingerprint, ContentAddressMethod::Raw::NixArchive, "/"), + makeSourcePathToHashCacheKey( + *accessor->fingerprint, ContentAddressMethod::Raw::NixArchive, CanonPath::root), {{"hash", info->narHash.to_string(HashFormat::SRI, true)}}); } } diff --git a/src/libfetchers/registry.cc b/src/libfetchers/registry.cc index 83de80bbccfd..89990fe5f15c 100644 --- a/src/libfetchers/registry.cc +++ b/src/libfetchers/registry.cc @@ -100,7 +100,7 @@ void Registry::remove(const Input & input) static std::filesystem::path getSystemRegistryPath() { - return settings.nixConfDir / "registry.json"; + return nixConfDir() / "registry.json"; } static std::shared_ptr getSystemRegistry(const Settings & settings) @@ -156,13 +156,14 @@ static std::shared_ptr getGlobalRegistry(const Settings & settings, St return Registry::read( settings, [&] -> SourcePath { - if (!isAbsolute(path)) { + std::filesystem::path fsPath{path}; + if (!fsPath.is_absolute()) { auto storePath = downloadFile(store, settings, path, "flake-registry.json").storePath; if (auto store2 = dynamic_cast(&store)) store2->addPermRoot(storePath, (getCacheDir() / "flake-registry.json").string()); return {store.requireStoreObjectAccessor(storePath)}; } else { - return SourcePath{getFSSourceAccessor(), CanonPath{path}}.resolveSymlinks(); + return SourcePath{getFSSourceAccessor(), CanonPath{fsPath.string()}}.resolveSymlinks(); } }(), Registry::Global); diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index 4ecf7ba9e194..5586229e56cf 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -18,7 +18,7 @@ namespace nix::fetchers { DownloadFileResult downloadFile( Store & store, const Settings & settings, - const std::string & url, + const VerbatimURL & url, const std::string & name, const Headers & headers) { @@ -27,7 +27,7 @@ DownloadFileResult downloadFile( Cache::Key key{ "file", {{ - {"url", url}, + {"url", url.to_string()}, {"name", name}, }}}; @@ -45,7 +45,7 @@ DownloadFileResult downloadFile( if (cached && !cached->expired) return useCached(); - FileTransferRequest request(VerbatimURL{url}); + FileTransferRequest request(url); request.headers = headers; if (cached) request.expectedETag = getStrAttr(cached->value, "etag"); @@ -54,7 +54,7 @@ DownloadFileResult downloadFile( res = getFileTransfer()->download(request); } catch (FileTransferError & e) { if (cached) { - warn("%s; using cached version", e.msg()); + warn("%s; using cached version", e.message()); return useCached(); } else throw; @@ -122,16 +122,23 @@ static std::optional downloadTarball_( // Namely lets catch when the url is a local file path, but // it is not in fact a tarball. if (url.scheme == "file") { - std::filesystem::path localPath = renderUrlPathEnsureLegal(url.path); + std::filesystem::path localPath = urlPathToPath(url.path); + if (!localPath.is_absolute()) { + throw Error( + "tarball '%s' must use an absolute path. " + "The 'file' scheme does not support relative paths.", + url); + } if (!exists(localPath)) { - throw Error("tarball '%s' does not exist.", localPath); + throw Error("tarball %s does not exist.", PathFmt(localPath)); } if (is_directory(localPath)) { if (exists(localPath / ".git")) { throw Error( - "tarball '%s' is a git repository, not a tarball. Please use `git+file` as the scheme.", localPath); + "tarball %s is a git repository, not a tarball. Please use `git+file` as the scheme.", + PathFmt(localPath)); } - throw Error("tarball '%s' is a directory, not a file.", localPath); + throw Error("tarball %s is a directory, not a file.", PathFmt(localPath)); } } @@ -182,8 +189,9 @@ static std::optional downloadTarball_( the entire file to disk so libarchive can access it in random-access mode. */ auto [fdTemp, path] = createTempFile("nix-zipfile"); - cleanupTemp.reset(path); - debug("downloading '%s' into '%s'...", url, path); + cleanupTemp.cancel(); + cleanupTemp = {path}; + debug("downloading '%s' into %s...", url, PathFmt(path)); { FdSink sink(fdTemp.get()); source->drainInto(sink); diff --git a/src/libflake-c/nix_api_flake.cc b/src/libflake-c/nix_api_flake.cc index 793db44b438c..2558236a7e5a 100644 --- a/src/libflake-c/nix_api_flake.cc +++ b/src/libflake-c/nix_api_flake.cc @@ -62,7 +62,7 @@ nix_err nix_flake_reference_parse_flags_set_base_directory( { nix_clear_err(context); try { - flags->baseDirectory.emplace(nix::Path{std::string(baseDirectory, baseDirectoryLen)}); + flags->baseDirectory.emplace(std::string(baseDirectory, baseDirectoryLen)); return NIX_OK; } NIXC_CATCH_ERRS @@ -162,8 +162,11 @@ nix_err nix_flake_lock_flags_add_input_override( { nix_clear_err(context); try { - auto path = nix::flake::parseInputAttrPath(inputPath); - flags->lockFlags->inputOverrides.emplace(path, *flakeRef->flakeRef); + auto path = nix::flake::NonEmptyInputAttrPath::parse(inputPath); + if (!path) + throw nix::UsageError( + "input override path cannot be zero-length; it would refer to the flake itself, not an input"); + flags->lockFlags->inputOverrides.emplace(std::move(*path), *flakeRef->flakeRef); if (flags->lockFlags->writeLockFile) { return nix_flake_lock_flags_set_mode_virtual(context, flags); } diff --git a/src/libflake-c/nix_api_flake.h b/src/libflake-c/nix_api_flake.h index 925463483119..9884a3d39a49 100644 --- a/src/libflake-c/nix_api_flake.h +++ b/src/libflake-c/nix_api_flake.h @@ -160,8 +160,9 @@ nix_err nix_flake_lock_flags_set_mode_write_as_needed(nix_c_context * context, n * @brief Add input overrides to the lock flags * @param[out] context Optional, stores error information * @param[in] flags The flags to modify - * @param[in] inputPath The input path to override + * @param[in] inputPath The input path to override (must not be empty) * @param[in] flakeRef The flake reference to use as the override + * @return NIX_ERR_NIX_ERROR if inputPath is empty * * This switches the `flags` to `nix_flake_lock_flags_set_mode_virtual` if not in mode * `nix_flake_lock_flags_set_mode_check`. diff --git a/src/libflake-c/nix_api_flake_internal.hh b/src/libflake-c/nix_api_flake_internal.hh index fbc6574d68ab..e048a47d3332 100644 --- a/src/libflake-c/nix_api_flake_internal.hh +++ b/src/libflake-c/nix_api_flake_internal.hh @@ -13,7 +13,7 @@ struct nix_flake_settings struct nix_flake_reference_parse_flags { - std::optional baseDirectory; + std::optional baseDirectory; }; struct nix_flake_reference diff --git a/src/libflake-tests/flakeref.cc b/src/libflake-tests/flakeref.cc index 3cc655907b3f..332ffe18b7dc 100644 --- a/src/libflake-tests/flakeref.cc +++ b/src/libflake-tests/flakeref.cc @@ -278,4 +278,16 @@ TEST(to_string, doesntReencodeUrl) ASSERT_EQ(unparsed, expected); } +TEST(parseFlakeRef, malformedGithubUrlDoesNotCrash) +{ + fetchers::Settings fetchSettings; + + // Using ref= instead of rev= with a github: URL should produce an + // error, not an assertion failure in renderAuthorityAndPath + // (https://github.com/NixOS/nix/issues/15196). + EXPECT_THROW( + parseFlakeRef(fetchSettings, "github:nixos/nixpkgs/nixpkgs.git?ref=aead170c1a49253ebfa5027010dfd89a77b73ca4"), + Error); +} + } // namespace nix diff --git a/src/libflake-tests/nix_api_flake.cc b/src/libflake-tests/nix_api_flake.cc index ec690b8124d4..063dfc4d7d40 100644 --- a/src/libflake-tests/nix_api_flake.cc +++ b/src/libflake-tests/nix_api_flake.cc @@ -377,6 +377,7 @@ TEST_F(nix_api_store_test, nix_api_load_flake_with_flags) assert_ctx_ok(); ASSERT_EQ("Claire", helloStr); + nix_locked_flake_free(lockedFlake); nix_flake_reference_parse_flags_free(parseFlags); nix_flake_lock_flags_free(lockFlags); nix_flake_reference_free(flakeReference); @@ -384,4 +385,62 @@ TEST_F(nix_api_store_test, nix_api_load_flake_with_flags) nix_flake_settings_free(settings); } +TEST_F(nix_api_store_test, nix_api_flake_lock_flags_add_input_override_empty_path) +{ + auto tmpDir = nix::createTempDir(); + nix::AutoDelete delTmpDir(tmpDir, true); + + nix::writeFile(tmpDir / "flake.nix", R"( + { + outputs = { ... }: { }; + } + )"); + + nix_libstore_init(ctx); + assert_ctx_ok(); + + auto fetchSettings = nix_fetchers_settings_new(ctx); + assert_ctx_ok(); + ASSERT_NE(nullptr, fetchSettings); + + auto settings = nix_flake_settings_new(ctx); + assert_ctx_ok(); + ASSERT_NE(nullptr, settings); + + auto lockFlags = nix_flake_lock_flags_new(ctx, settings); + assert_ctx_ok(); + ASSERT_NE(nullptr, lockFlags); + + auto parseFlags = nix_flake_reference_parse_flags_new(ctx, settings); + assert_ctx_ok(); + ASSERT_NE(nullptr, parseFlags); + + auto r0 = nix_flake_reference_parse_flags_set_base_directory( + ctx, parseFlags, tmpDir.string().c_str(), tmpDir.string().size()); + assert_ctx_ok(); + ASSERT_EQ(NIX_OK, r0); + + nix_flake_reference * flakeReference = nullptr; + std::string fragment; + nix_flake_reference_and_fragment_from_string( + ctx, fetchSettings, settings, parseFlags, ".", 1, &flakeReference, OBSERVE_STRING(fragment)); + assert_ctx_ok(); + ASSERT_NE(nullptr, flakeReference); + + // Test that empty input path is rejected (issue #14816) + auto r = nix_flake_lock_flags_add_input_override(ctx, lockFlags, "", flakeReference); + ASSERT_EQ(NIX_ERR_NIX_ERROR, r); + assert_ctx_err(); + + // Verify error message contains expected text + const char * errMsg = nix_err_msg(nullptr, ctx, nullptr); + ASSERT_NE(nullptr, errMsg); + ASSERT_NE(std::string(errMsg).find("input override path cannot be zero-length"), std::string::npos); + + nix_flake_reference_free(flakeReference); + nix_flake_reference_parse_flags_free(parseFlags); + nix_flake_lock_flags_free(lockFlags); + nix_flake_settings_free(settings); +} + } // namespace nixC diff --git a/src/libflake/config.cc b/src/libflake/config.cc index fd0e9c75fdcb..b7d5cd8999e5 100644 --- a/src/libflake/config.cc +++ b/src/libflake/config.cc @@ -31,7 +31,7 @@ namespace nix::flake { // setting name -> setting value -> allow or ignore. typedef std::map> TrustedList; -std::filesystem::path trustedListPath() +static std::filesystem::path trustedListPath() { return getDataDir() / "trusted-settings.json"; } diff --git a/src/libflake/flake-primops.cc b/src/libflake/flake-primops.cc index 4d27c6848272..ad5059335139 100644 --- a/src/libflake/flake-primops.cc +++ b/src/libflake/flake-primops.cc @@ -88,7 +88,7 @@ PrimOp getFlake(const Settings & settings) (builtins.getFlake "github:edolstra/dwarffs").rev ``` )", - .fun = prim_getFlake, + .impl = prim_getFlake, }; } @@ -129,7 +129,7 @@ nix::PrimOp parseFlakeRef({ { dir = "lib"; owner = "NixOS"; ref = "23.05"; repo = "nixpkgs"; type = "github"; } ``` )", - .fun = prim_parseFlakeRef, + .impl = prim_parseFlakeRef, }); static void prim_flakeRefToString(EvalState & state, const PosIdx pos, Value ** args, Value & v) @@ -192,7 +192,7 @@ nix::PrimOp flakeRefToString({ "github:NixOS/nixpkgs/23.05?dir=lib" ``` )", - .fun = prim_flakeRefToString, + .impl = prim_flakeRefToString, }); } // namespace nix::flake::primops diff --git a/src/libflake/flake.cc b/src/libflake/flake.cc index 5cea4e567f62..1dc030314e56 100644 --- a/src/libflake/flake.cc +++ b/src/libflake/flake.cc @@ -457,9 +457,10 @@ LockedFlake lockFlake( std::optional parentInputAttrPath; // FIXME: rename to inputAttrPathPrefix? }; - std::map overrides; - std::set explicitCliOverrides; - std::set overridesUsed, updatesUsed; + std::map overrides; + std::set explicitCliOverrides; + std::set overridesUsed; + std::set updatesUsed; std::map, SourcePath> nodePaths; for (auto & i : lockFlags.inputOverrides) { @@ -516,8 +517,7 @@ LockedFlake lockFlake( auto addOverrides = [&](this const auto & addOverrides, const FlakeInput & input, const InputAttrPath & prefix) -> void { for (auto & [idOverride, inputOverride] : input.overrides) { - auto inputAttrPath(prefix); - inputAttrPath.push_back(idOverride); + auto inputAttrPath = NonEmptyInputAttrPath::append(prefix, idOverride); if (inputOverride.ref || inputOverride.follows) overrides.emplace( inputAttrPath, @@ -538,9 +538,8 @@ LockedFlake lockFlake( /* Check whether this input has overrides for a non-existent input. */ for (auto [inputAttrPath, inputOverride] : overrides) { - auto inputAttrPath2(inputAttrPath); - auto follow = inputAttrPath2.back(); - inputAttrPath2.pop_back(); + auto follow = inputAttrPath.inputName(); + auto inputAttrPath2 = inputAttrPath.parent(); if (inputAttrPath2 == inputAttrPathPrefix && !flakeInputs.count(follow)) warn( "input '%s' has an override for a non-existent input '%s'", @@ -552,8 +551,8 @@ LockedFlake lockFlake( necessary (i.e. if they're new or the flakeref changed from what's in the lock file). */ for (auto & [id, input2] : flakeInputs) { - auto inputAttrPath(inputAttrPathPrefix); - inputAttrPath.push_back(id); + auto nonEmptyInputAttrPath = NonEmptyInputAttrPath::append(inputAttrPathPrefix, id); + auto inputAttrPath = nonEmptyInputAttrPath.get(); auto inputAttrPathS = printInputAttrPath(inputAttrPath); debug("computing input '%s'", inputAttrPathS); @@ -561,11 +560,11 @@ LockedFlake lockFlake( /* Do we have an override for this input from one of the ancestors? */ - auto i = overrides.find(inputAttrPath); + auto i = overrides.find(nonEmptyInputAttrPath); bool hasOverride = i != overrides.end(); - bool hasCliOverride = explicitCliOverrides.contains(inputAttrPath); + bool hasCliOverride = explicitCliOverrides.contains(nonEmptyInputAttrPath); if (hasOverride) - overridesUsed.insert(inputAttrPath); + overridesUsed.insert(nonEmptyInputAttrPath); auto input = hasOverride ? i->second.input : input2; /* Resolve relative 'path:' inputs relative to @@ -603,7 +602,7 @@ LockedFlake lockFlake( if (auto relativePath = input.ref->input.isRelative()) { return SourcePath{ overriddenSourcePath.accessor, - CanonPath(*relativePath, overriddenSourcePath.path.parent().value())}; + CanonPath(relativePath->string(), overriddenSourcePath.path.parent().value())}; } else return std::nullopt; }; @@ -624,7 +623,7 @@ LockedFlake lockFlake( updatesUsed.insert(inputAttrPath); - if (oldNode && !lockFlags.inputUpdates.count(inputAttrPath)) + if (oldNode && !lockFlags.inputUpdates.count(nonEmptyInputAttrPath)) if (auto oldLock2 = get(oldNode->inputs, id)) if (auto oldLock3 = std::get_if<0>(&*oldLock2)) oldLock = *oldLock3; @@ -647,10 +646,10 @@ LockedFlake lockFlake( /* If we have this input in updateInputs, then we must fetch the flake to update it. */ - auto lb = lockFlags.inputUpdates.lower_bound(inputAttrPath); + auto lb = lockFlags.inputUpdates.lower_bound(nonEmptyInputAttrPath); - auto mustRefetch = lb != lockFlags.inputUpdates.end() && lb->size() > inputAttrPath.size() - && std::equal(inputAttrPath.begin(), inputAttrPath.end(), lb->begin()); + auto mustRefetch = lb != lockFlags.inputUpdates.end() && lb->get().size() > inputAttrPath.size() + && std::equal(inputAttrPath.begin(), inputAttrPath.end(), lb->get().begin()); FlakeInputs fakeInputs; @@ -672,8 +671,8 @@ LockedFlake lockFlake( // It is possible that the flake has changed, // so we must confirm all the follows that are in the lock file are also in the // flake. - auto overridePath(inputAttrPath); - overridePath.push_back(i.first); + auto overridePath = + NonEmptyInputAttrPath::append(nonEmptyInputAttrPath, i.first); auto o = overrides.find(overridePath); // If the override disappeared, we have to refetch the flake, // since some of the inputs may not be present in the lock file. @@ -727,7 +726,7 @@ LockedFlake lockFlake( nuked the next time we update the lock file. That is, overrides are sticky unless you use --no-write-lock-file. */ - auto inputIsOverride = explicitCliOverrides.contains(inputAttrPath); + auto inputIsOverride = explicitCliOverrides.contains(nonEmptyInputAttrPath); auto ref = (input2.ref && inputIsOverride) ? *input2.ref : *input.ref; /* Warn against the use of indirect flakerefs @@ -890,11 +889,11 @@ LockedFlake lockFlake( auto s = chomp(diff); if (lockFileExists) { if (s.empty()) - warn("updating lock file %s", outputLockFilePath); + warn("updating lock file %s", PathFmt(outputLockFilePath)); else - warn("updating lock file %s:\n%s", outputLockFilePath, s); + warn("updating lock file %s:\n%s", PathFmt(outputLockFilePath), s); } else - warn("creating lock file %s: \n%s", outputLockFilePath, s); + warn("creating lock file %s: \n%s", PathFmt(outputLockFilePath), s); std::optional commitMessage = std::nullopt; diff --git a/src/libflake/flakeref.cc b/src/libflake/flakeref.cc index d186db8ac858..4fc97ac53a98 100644 --- a/src/libflake/flakeref.cc +++ b/src/libflake/flakeref.cc @@ -114,7 +114,7 @@ std::pair parsePathFlakeRefWithFragment( auto succeeds = std::regex_match(url, match, pathFlakeRegex); if (!succeeds) throw Error("invalid flakeref '%s'", url); - auto path = match[1].str(); + std::filesystem::path path = match[1].str(); auto query = decodeQuery(match[3].str(), /*lenient=*/true); auto fragment = percentDecode(match[5].str()); @@ -123,60 +123,62 @@ std::pair parsePathFlakeRefWithFragment( to 'baseDir'). If so, search upward to the root of the repo (i.e. the directory containing .git). */ - path = absPath(path, baseDir->string(), true); + path = absPath(path, get(baseDir), true); if (isFlake) { if (!S_ISDIR(lstat(path).st_mode)) { - if (baseNameOf(path) == "flake.nix") { + if (path.filename() == "flake.nix") { // Be gentle with people who accidentally write `/foo/bar/flake.nix` instead of `/foo/bar` + auto parentPath = path.parent_path(); warn( - "Path '%s' should point at the directory containing the 'flake.nix' file, not the file itself. " - "Pretending that you meant '%s'", - path, - dirOf(path)); - path = dirOf(path); + "Path %s should point at the directory containing the 'flake.nix' file, not the file itself. " + "Pretending that you meant %s", + PathFmt(path), + PathFmt(parentPath)); + path = parentPath; } else { - throw BadURL("path '%s' is not a flake (because it's not a directory)", path); + throw BadURL("path %s is not a flake (because it's not a directory)", PathFmt(path)); } } - if (!allowMissing && !pathExists(path + "/flake.nix")) { - notice("path '%s' does not contain a 'flake.nix', searching up", path); + if (!allowMissing && !pathExists(path / "flake.nix")) { + notice("path %s does not contain a 'flake.nix', searching up", PathFmt(path)); // Save device to detect filesystem boundary dev_t device = lstat(path).st_dev; bool found = false; while (path != "/") { - if (pathExists(path + "/flake.nix")) { + if (pathExists(path / "flake.nix")) { found = true; break; - } else if (pathExists(path + "/.git")) + } else if (pathExists(path / ".git")) throw Error( - "path '%s' is not part of a flake (neither it nor its parent directories contain a 'flake.nix' file)", - path); + "path %s is not part of a flake (neither it nor its parent directories contain a 'flake.nix' file)", + PathFmt(path)); else { if (lstat(path).st_dev != device) - throw Error("unable to find a flake before encountering filesystem boundary at '%s'", path); + throw Error( + "unable to find a flake before encountering filesystem boundary at %s", PathFmt(path)); } - path = dirOf(path); + path = path.parent_path(); } if (!found) throw BadURL("could not find a flake.nix file"); } - if (!allowMissing && !pathExists(path + "/flake.nix")) - throw BadURL("path '%s' is not a flake (because it doesn't contain a 'flake.nix' file)", path); + if (!allowMissing && !pathExists(path / "flake.nix")) + throw BadURL("path %s is not a flake (because it doesn't contain a 'flake.nix' file)", PathFmt(path)); auto flakeRoot = path; std::string subdir; while (flakeRoot != "/") { - if (pathExists(flakeRoot + "/.git")) { + if (pathExists(flakeRoot / ".git")) { auto parsedURL = ParsedURL{ .scheme = "git+file", .authority = ParsedURL::Authority{}, - .path = splitString>(flakeRoot, "/"), + .path = pathToUrlPath(flakeRoot), .query = query, .fragment = fragment, }; @@ -187,19 +189,19 @@ std::pair parsePathFlakeRefWithFragment( parsedURL.query.insert_or_assign("dir", subdir); } - if (pathExists(flakeRoot + "/.git/shallow")) + if (pathExists(flakeRoot / ".git" / "shallow")) parsedURL.query.insert_or_assign("shallow", "1"); return fromParsedURL(fetchSettings, std::move(parsedURL), isFlake); } - subdir = std::string(baseNameOf(flakeRoot)) + (subdir.empty() ? "" : "/" + subdir); - flakeRoot = dirOf(flakeRoot); + subdir = flakeRoot.filename().string() + (subdir.empty() ? "" : "/" + subdir); + flakeRoot = flakeRoot.parent_path(); } } } else { - if (!preserveRelativePaths && !isAbsolute(path)) + if (!preserveRelativePaths && !path.is_absolute()) throw BadURL("flake reference '%s' is not an absolute path", url); } @@ -207,8 +209,8 @@ std::pair parsePathFlakeRefWithFragment( fetchSettings, { .scheme = "path", - .authority = ParsedURL::Authority{}, - .path = splitString>(path, "/"), + .authority = path.is_absolute() ? std::optional{ParsedURL::Authority{}} : std::nullopt, + .path = pathToUrlPath(path), .query = query, .fragment = fragment, }, @@ -252,9 +254,9 @@ std::optional> parseURLFlakeRef( auto parsed = parseURL(url, /*lenient=*/true); if (baseDir && (parsed.scheme == "path" || parsed.scheme == "git+file")) { /* Here we know that the path must not contain encoded '/' or NUL bytes. */ - auto path = renderUrlPathEnsureLegal(parsed.path); - if (!isAbsolute(path)) - parsed.path = splitString>(absPath(path, baseDir->string()), "/"); + auto path = urlPathToPath(parsed.path); + if (!path.is_absolute()) + parsed.path = pathToUrlPath(absPath(path, get(baseDir))); } return fromParsedURL(fetchSettings, std::move(parsed), isFlake); } catch (BadURL &) { diff --git a/src/libflake/include/nix/flake/flake.hh b/src/libflake/include/nix/flake/flake.hh index 301db4f60137..7156261ceb50 100644 --- a/src/libflake/include/nix/flake/flake.hh +++ b/src/libflake/include/nix/flake/flake.hh @@ -218,13 +218,13 @@ struct LockFlags /** * Flake inputs to be overridden. */ - std::map inputOverrides; + std::map inputOverrides; /** * Flake inputs to be updated. This means that any existing lock * for those inputs will be ignored. */ - std::set inputUpdates; + std::set inputUpdates; /** * Whether to require a locked input. diff --git a/src/libflake/include/nix/flake/flakeref.hh b/src/libflake/include/nix/flake/flakeref.hh index 629afab03b5a..1f39d62ebbf9 100644 --- a/src/libflake/include/nix/flake/flakeref.hh +++ b/src/libflake/include/nix/flake/flakeref.hh @@ -50,8 +50,10 @@ struct FlakeRef /** * sub-path within the fetched input that represents this input + * + * @todo Should probably use `CanonPath` instead of `std::string`? */ - Path subdir; + std::string subdir; bool operator==(const FlakeRef & other) const = default; @@ -60,7 +62,7 @@ struct FlakeRef return std::tie(input, subdir) < std::tie(other.input, other.subdir); } - FlakeRef(fetchers::Input && input, const Path & subdir) + FlakeRef(fetchers::Input && input, const std::string & subdir) : input(std::move(input)) , subdir(subdir) { diff --git a/src/libflake/include/nix/flake/lockfile.hh b/src/libflake/include/nix/flake/lockfile.hh index 1ca7cc3dd305..27232b20a669 100644 --- a/src/libflake/include/nix/flake/lockfile.hh +++ b/src/libflake/include/nix/flake/lockfile.hh @@ -14,6 +14,80 @@ namespace nix::flake { typedef std::vector InputAttrPath; +/** + * A non-empty input attribute path. + * + * Input attribute paths identify inputs in a flake. An empty path would + * refer to the flake itself rather than an input, which contradicts the + * purpose of operations like override or update. + */ +class NonEmptyInputAttrPath +{ + InputAttrPath path; + + explicit NonEmptyInputAttrPath(InputAttrPath && p) + : path(std::move(p)) + { + assert(!path.empty()); + } + +public: + /** + * Parse and validate a non-empty input attribute path. + * Returns std::nullopt if the path is empty. + */ + static std::optional parse(std::string_view s); + + /** + * Construct from an already-parsed path. + * Returns std::nullopt if the path is empty. + */ + static std::optional make(InputAttrPath path); + + /** + * Append an element to a path, creating a non-empty path. + * This is always safe because adding an element guarantees non-emptiness. + */ + static NonEmptyInputAttrPath append(const InputAttrPath & prefix, const FlakeId & element) + { + InputAttrPath path = prefix; + path.push_back(element); + return NonEmptyInputAttrPath{std::move(path)}; + } + + const InputAttrPath & get() const + { + return path; + } + + operator const InputAttrPath &() const + { + return path; + } + + /** + * Get the final component of the path (the input name). + * For a path like "a/b/c", returns "c". + */ + const FlakeId & inputName() const + { + return path.back(); + } + + /** + * Get the parent path (all components except the last). + * For a path like "a/b/c", returns "a/b". + */ + InputAttrPath parent() const + { + InputAttrPath result = path; + result.pop_back(); + return result; + } + + auto operator<=>(const NonEmptyInputAttrPath & other) const = default; +}; + struct LockedNode; /** diff --git a/src/libflake/lockfile.cc b/src/libflake/lockfile.cc index b287db5b8e59..dc5b79ffe510 100644 --- a/src/libflake/lockfile.cc +++ b/src/libflake/lockfile.cc @@ -315,6 +315,19 @@ InputAttrPath parseInputAttrPath(std::string_view s) return path; } +std::optional NonEmptyInputAttrPath::parse(std::string_view s) +{ + auto path = parseInputAttrPath(s); + return make(std::move(path)); +} + +std::optional NonEmptyInputAttrPath::make(InputAttrPath path) +{ + if (path.empty()) + return std::nullopt; + return NonEmptyInputAttrPath{std::move(path)}; +} + std::map LockFile::getAllInputs() const { std::set> done; diff --git a/src/libmain/common-args.cc b/src/libmain/common-args.cc index 6055ec0e7524..42b814ee5b8b 100644 --- a/src/libmain/common-args.cc +++ b/src/libmain/common-args.cc @@ -48,7 +48,7 @@ MixCommonArgs::MixCommonArgs(const std::string & programName) globalConfig.set(name, value); } catch (UsageError & e) { if (!getRoot().completions) - warn(e.what()); + logWarning(e.info()); } }}, .completer = diff --git a/src/libmain/include/nix/main/shared.hh b/src/libmain/include/nix/main/shared.hh index 800018290f69..19be9a04cf39 100644 --- a/src/libmain/include/nix/main/shared.hh +++ b/src/libmain/include/nix/main/shared.hh @@ -13,21 +13,18 @@ namespace nix { -int handleExceptions(const std::string & programName, std::function fun); - /** * Don't forget to call initPlugins() after settings are initialized! * @param loadConfig Whether to load configuration from `nix.conf`, `NIX_CONFIG`, etc. May be disabled for unit tests. */ void initNix(bool loadConfig = true); -void parseCmdLine( - int argc, char ** argv, std::function parseArg); +void parseCmdLine(int argc, char ** argv, fun parseArg); void parseCmdLine( const std::string & programName, const Strings & args, - std::function parseArg); + fun parseArg); std::string version(); @@ -58,11 +55,10 @@ N getIntArg(const std::string & opt, Strings::iterator & i, const Strings::itera struct LegacyArgs : public MixCommonArgs, public RootArgs { - std::function parseArg; + fun parseArg; LegacyArgs( - const std::string & programName, - std::function parseArg); + const std::string & programName, fun parseArg); bool processFlag(Strings::iterator & pos, Strings::iterator end) override; @@ -93,19 +89,7 @@ extern volatile ::sig_atomic_t blockInt; struct GCResults; -struct PrintFreed -{ - bool show; - const GCResults & results; - - PrintFreed(bool show, const GCResults & results) - : show(show) - , results(results) - { - } - - ~PrintFreed(); -}; +void printFreed(bool dryRun, const GCResults & results); #ifndef _WIN32 /** @@ -128,7 +112,7 @@ void detectStackOverflow(); * limited stack space and a potentially a corrupted heap, all while the failed * thread is blocked indefinitely. All functions called must be reentrant. */ -extern std::function stackOverflowHandler; +extern fun stackOverflowHandler; /** * The default, robust implementation of stackOverflowHandler. diff --git a/src/libmain/plugin.cc b/src/libmain/plugin.cc index 4755ba24bfad..ff8c8201184e 100644 --- a/src/libmain/plugin.cc +++ b/src/libmain/plugin.cc @@ -83,8 +83,8 @@ void initPlugins() checkInterrupt(); pluginFiles.emplace_back(ent.path()); } - } catch (SysError & e) { - if (e.errNo != ENOTDIR) + } catch (SystemError & e) { + if (!e.is(std::errc::not_a_directory)) throw; pluginFiles.emplace_back(pluginFile); } @@ -95,7 +95,7 @@ void initPlugins() #ifndef _WIN32 // TODO implement via DLL loading on Windows void * handle = dlopen(file.c_str(), RTLD_LAZY | RTLD_LOCAL); if (!handle) - throw Error("could not dynamically open plugin file '%s': %s", file, dlerror()); + throw Error("could not dynamically open plugin file %s: %s", PathFmt(file), dlerror()); /* Older plugins use a statically initialized object to run their code. Newer plugins can also export nix_plugin_entry() */ @@ -103,7 +103,7 @@ void initPlugins() if (nix_plugin_entry) nix_plugin_entry(); #else - throw Error("could not dynamically open plugin file '%s'", file); + throw Error("could not dynamically open plugin file %s", PathFmt(file)); #endif } } diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 1c4ede093556..e917af1831e0 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -1,7 +1,9 @@ #include "nix/store/globals.hh" #include "nix/util/current-process.hh" +#include "nix/util/executable-path.hh" #include "nix/main/shared.hh" #include "nix/store/store-api.hh" +#include "nix/store/store-open.hh" #include "nix/store/gc-store.hh" #include "nix/main/loggers.hh" #include "nix/main/progress-bar.hh" @@ -15,9 +17,12 @@ #include #include #include -#include #include #include + +#ifndef _WIN32 +# include +#endif #ifdef __linux__ # include #endif @@ -209,8 +214,7 @@ void initNix(bool loadConfig) } LegacyArgs::LegacyArgs( - const std::string & programName, - std::function parseArg) + const std::string & programName, fun parseArg) : MixCommonArgs(programName) , parseArg(parseArg) { @@ -232,13 +236,13 @@ LegacyArgs::LegacyArgs( .longName = "keep-going", .shortName = 'k', .description = "Keep going after a build fails.", - .handler = {&(bool &) settings.keepGoing, true}, + .handler = {&(bool &) settings.getWorkerSettings().keepGoing, true}, }); addFlag({ .longName = "fallback", .description = "Build from source if substitution fails.", - .handler = {&(bool &) settings.tryFallback, true}, + .handler = {&(bool &) settings.getWorkerSettings().tryFallback, true}, }); auto intSettingAlias = @@ -275,7 +279,7 @@ LegacyArgs::LegacyArgs( .longName = "store", .description = "The URL of the Nix store to use.", .labels = {"store-uri"}, - .handler = {&(std::string &) settings.storeUri}, + .handler = {[](std::string s) { settings.storeUri = StoreReference::parse(s); }}, }); } @@ -301,8 +305,7 @@ bool LegacyArgs::processArgs(const Strings & args, bool finish) return true; } -void parseCmdLine( - int argc, char ** argv, std::function parseArg) +void parseCmdLine(int argc, char ** argv, fun parseArg) { parseCmdLine(std::string(baseNameOf(argv[0])), argvToStrings(argc, argv), parseArg); } @@ -310,7 +313,7 @@ void parseCmdLine( void parseCmdLine( const std::string & programName, const Strings & args, - std::function parseArg) + fun parseArg) { LegacyArgs(programName, parseArg).parseCmdline(args); } @@ -332,52 +335,16 @@ void printVersion(const std::string & programName) std::cout << "System type: " << settings.thisSystem << "\n"; std::cout << "Additional system types: " << concatStringsSep(", ", settings.extraPlatforms.get()) << "\n"; std::cout << "Features: " << concatStringsSep(", ", cfg) << "\n"; - std::cout << "System configuration file: " << (settings.nixConfDir / "nix.conf") << "\n"; - std::cout << "User configuration files: " << concatStringsSep(":", settings.nixUserConfFiles) << "\n"; - std::cout << "Store directory: " << settings.nixStore << "\n"; + std::cout << "System configuration file: " << nixConfFile() << "\n"; + std::cout << "User configuration files: " + << os_string_to_string(ExecutablePath{.directories = nixUserConfFiles()}.render()) << "\n"; + std::cout << "Store directory: " << resolveStoreConfig(StoreReference{settings.storeUri.get()})->storeDir + << "\n"; std::cout << "State directory: " << settings.nixStateDir << "\n"; - std::cout << "Data directory: " << settings.nixDataDir << "\n"; } throw Exit(); } -int handleExceptions(const std::string & programName, std::function fun) -{ - ReceiveInterrupts receiveInterrupts; // FIXME: need better place for this - - ErrorInfo::programName = baseNameOf(programName); - - auto doLog = [&](BaseError & e) { - try { - logError(e.info()); - } catch (...) { - printError(ANSI_RED "error:" ANSI_NORMAL " Exception while printing an exception."); - } - }; - - std::string error = ANSI_RED "error:" ANSI_NORMAL " "; - try { - fun(); - } catch (Exit & e) { - return e.status; - } catch (UsageError & e) { - doLog(e); - printError("\nTry '%1% --help' for more information.", programName); - return 1; - } catch (BaseError & e) { - doLog(e); - return e.info().status; - } catch (std::bad_alloc & e) { - printError(error + "out of memory"); - return 1; - } catch (std::exception & e) { - printError(error + e.what()); - return 1; - } - - return 0; -} - RunPager::RunPager() { if (!isatty(STDOUT_FILENO)) @@ -432,9 +399,12 @@ RunPager::~RunPager() } } -PrintFreed::~PrintFreed() +void printFreed(bool dryRun, const GCResults & results) { - if (show) + /* bytesFreed cannot be reliably computed without actually deleting store paths because of hardlinking. */ + if (dryRun) + std::cout << fmt("%d store paths would be deleted\n", results.paths.size()); + else std::cout << fmt("%d store paths deleted, %s freed\n", results.paths.size(), renderSize(results.bytesFreed)); } diff --git a/src/libmain/unix/stack.cc b/src/libmain/unix/stack.cc index 458693407270..bec0d389f5ed 100644 --- a/src/libmain/unix/stack.cc +++ b/src/libmain/unix/stack.cc @@ -68,12 +68,12 @@ void detectStackOverflow() #endif } -std::function stackOverflowHandler(defaultStackOverflowHandler); +fun stackOverflowHandler(defaultStackOverflowHandler); void defaultStackOverflowHandler(siginfo_t * info, void * ctx) { char msg[] = "error: stack overflow (possible infinite recursion)\n"; - [[gnu::unused]] auto res = write(2, msg, strlen(msg)); + [[gnu::unused]] auto res = ::write(2, msg, strlen(msg)); _exit(1); // maybe abort instead? } diff --git a/src/libstore-c/nix_api_store.cc b/src/libstore-c/nix_api_store.cc index 4133d769f218..f0d8aeab3aa1 100644 --- a/src/libstore-c/nix_api_store.cc +++ b/src/libstore-c/nix_api_store.cc @@ -9,6 +9,7 @@ #include "nix/store/path.hh" #include "nix/store/store-api.hh" #include "nix/store/store-open.hh" +#include "nix/store/store-reference.hh" #include "nix/store/build-result.hh" #include "nix/store/local-fs-store.hh" #include "nix/util/base-nix-32.hh" @@ -47,14 +48,14 @@ Store * nix_store_open(nix_c_context * context, const char * uri, const char *** if (uri_str.empty()) return new Store{nix::openStore()}; - if (!params) - return new Store{nix::openStore(uri_str)}; + auto storeRef = nix::StoreReference::parse(uri_str); - nix::Store::Config::Params params_map; - for (size_t i = 0; params[i] != nullptr; i++) { - params_map[params[i][0]] = params[i][1]; + if (params) { + for (size_t i = 0; params[i] != nullptr; i++) { + storeRef.params[params[i][0]] = params[i][1]; + } } - return new Store{nix::openStore(uri_str, params_map)}; + return new Store{nix::openStore(std::move(storeRef))}; } NIXC_CATCH_ERRS_NULL } @@ -115,7 +116,7 @@ nix_err nix_store_real_path( context->last_err_code = NIX_OK; try { auto store2 = store->ptr.dynamic_pointer_cast(); - auto res = store2 ? store2->toRealPath(path->path) : store->ptr->printStorePath(path->path); + auto res = store2 ? store2->toRealPath(path->path).string() : store->ptr->printStorePath(path->path); return call_nix_get_string_callback(res, callback, user_data); } NIXC_CATCH_ERRS @@ -182,10 +183,8 @@ nix_err nix_store_realise( assert(results.size() == 1); // Check if any builds failed - for (auto & result : results) { - if (auto * failureP = result.tryGetFailure()) - failureP->rethrow(); - } + for (auto & result : results) + result.tryThrowBuildError(); if (callback) { for (const auto & result : results) { @@ -301,14 +300,12 @@ nix_err nix_derivation_make_outputs( context->last_err_code = NIX_OK; try { auto drv = nix::Derivation::parseJsonAndValidate(*store->ptr, nlohmann::json::parse(json)); - auto hashesModulo = hashDerivationModulo(*store->ptr, drv, true); for (auto & output : drv.outputs) { - nix::Hash h = hashesModulo.hashes.at(output.first); - auto outPath = store->ptr->makeOutputPath(output.first, h, drv.name); + auto outPath = output.second.path(*store->ptr, drv.name, output.first); - if (callback) { - callback(userdata, output.first.c_str(), store->ptr->printStorePath(outPath).c_str()); + if (callback && outPath) { + callback(userdata, output.first.c_str(), store->ptr->printStorePath(*outPath).c_str()); } } } @@ -334,7 +331,12 @@ StorePath * nix_add_derivation(nix_c_context * context, Store * store, nix_deriv if (context) context->last_err_code = NIX_OK; try { - auto ret = nix::writeDerivation(*store->ptr, derivation->drv, nix::NoRepair); + /* Quite dubious that users would want this to silently suceed + without actually writing the derivation if this setting is + set, but it was that way already, so we are doing this for + back-compat for now. */ + auto ret = nix::settings.readOnlyMode ? nix::computeStorePath(*store->ptr, derivation->drv) + : store->ptr->writeDerivation(derivation->drv, nix::NoRepair); return new StorePath{ret}; } diff --git a/src/libstore-test-support/https-store.cc b/src/libstore-test-support/https-store.cc new file mode 100644 index 000000000000..79548f61f3fe --- /dev/null +++ b/src/libstore-test-support/https-store.cc @@ -0,0 +1,144 @@ +#include "nix/store/tests/https-store.hh" +#include "nix/util/os-string.hh" + +#include + +namespace nix::testing { + +void TestHttpBinaryCacheStore::init() +{ + BinaryCacheStore::init(); +} + +ref TestHttpBinaryCacheStoreConfig::openTestStore(ref fileTransfer) const +{ + auto store = make_ref( + ref{// FIXME we shouldn't actually need a mutable config + std::const_pointer_cast(shared_from_this())}, + fileTransfer); + store->init(); + return store; +} + +void HttpsBinaryCacheStoreTest::openssl(Strings args) +{ + runProgram("openssl", /*lookupPath=*/true, toOsStrings(std::move(args))); +} + +void HttpsBinaryCacheStoreTest::SetUp() +{ + LibStoreNetworkTest::SetUp(); + +#ifdef _WIN32 + GTEST_SKIP() << "HTTPS store tests are not supported on Windows"; +#endif + + tmpDir = createTempDir(); + cacheDir = tmpDir / "cache"; + delTmpDir = std::make_unique(tmpDir); + + localCacheStore = + make_ref(cacheDir, LocalBinaryCacheStoreConfig::Params{})->openStore(); + + caCert = tmpDir / "ca.crt"; + caKey = tmpDir / "ca.key"; + serverCert = tmpDir / "server.crt"; + serverKey = tmpDir / "server.key"; + clientCert = tmpDir / "client.crt"; + clientKey = tmpDir / "client.key"; + + // clang-format off + openssl({"ecparam", "-genkey", "-name", "prime256v1", "-out", caKey.string()}); + openssl({"req", "-new", "-x509", "-days", "1", "-key", caKey.string(), "-out", caCert.string(), "-subj", "/CN=TestCA"}); + auto serverExtFile = tmpDir / "server.ext"; + writeFile(serverExtFile, "subjectAltName=DNS:localhost,IP:127.0.0.1"); + openssl({"ecparam", "-genkey", "-name", "prime256v1", "-out", serverKey.string()}); + openssl({"req", "-new", "-key", serverKey.string(), "-out", (tmpDir / "server.csr").string(), "-subj", "/CN=localhost", "-addext", "subjectAltName=DNS:localhost,IP:127.0.0.1"}); + openssl({"x509", "-req", "-in", (tmpDir / "server.csr").string(), "-CA", caCert.string(), "-CAkey", caKey.string(), "-CAcreateserial", "-out", serverCert.string(), "-days", "1", "-extfile", serverExtFile.string()}); + openssl({"ecparam", "-genkey", "-name", "prime256v1", "-out", clientKey.string()}); + openssl({"req", "-new", "-key", clientKey.string(), "-out", (tmpDir / "client.csr").string(), "-subj", "/CN=TestClient"}); + openssl({"x509", "-req", "-in", (tmpDir / "client.csr").string(), "-CA", caCert.string(), "-CAkey", caKey.string(), "-CAcreateserial", "-out", clientCert.string(), "-days", "1"}); + // clang-format on + +#ifndef _WIN32 /* FIXME: Can't yet start processes on windows */ + auto args = serverArgs(); + serverPid = startProcess( + [&] { + if (chdir(cacheDir.c_str()) == -1) + _exit(1); + std::vector argv; + argv.push_back(const_cast("openssl")); + for (auto & a : args) + argv.push_back(const_cast(a.c_str())); + argv.push_back(nullptr); + execvp("openssl", argv.data()); + _exit(1); + }, + {.dieWithParent = true}); +#endif + + /* As an optimization, sleep for a bit to allow the server to come up to avoid retrying when connecting. + This won't make the tests fail, but does make them run faster. We don't need to overcomplicate by waiting + for the port explicitly - this is enough. */ + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + + /* Create custom FileTransferSettings with our test CA certificate. + This avoids mutating global settings. */ + testFileTransferSettings = std::make_unique(); + testFileTransferSettings->caFile = caCert; + testFileTransfer = makeFileTransfer(*testFileTransferSettings); +} + +void HttpsBinaryCacheStoreTest::TearDown() +{ + serverPid.kill(); + delTmpDir.reset(); + testFileTransferSettings.reset(); +} + +std::vector HttpsBinaryCacheStoreTest::serverArgs() +{ + return { + "s_server", + "-accept", + std::to_string(port), + "-cert", + serverCert.string(), + "-key", + serverKey.string(), + "-WWW", /* Serve from current directory. */ + "-quiet", + }; +} + +std::vector HttpsBinaryCacheStoreMtlsTest::serverArgs() +{ + auto args = HttpsBinaryCacheStoreTest::serverArgs(); + /* With the -Verify option the client must supply a certificate or an error occurs, which is not the + case with -verify. */ + args.insert(args.end(), {"-CAfile", caCert.string(), "-Verify", "1", "-verify_return_error"}); + return args; +} + +ref HttpsBinaryCacheStoreTest::makeConfig() +{ + auto res = make_ref( + ParsedURL{ + .scheme = "https", + .authority = + ParsedURL::Authority{ + .host = "localhost", + .port = port, + }, + }, + TestHttpBinaryCacheStoreConfig::Params{}); + res->pathInfoCacheSize = 0; /* We don't want any caching in tests. */ + return res; +} + +ref HttpsBinaryCacheStoreTest::openStore(ref config) +{ + return config->openTestStore(ref{testFileTransfer}); +} + +} // namespace nix::testing diff --git a/src/libstore-test-support/include/nix/store/tests/https-store.hh b/src/libstore-test-support/include/nix/store/tests/https-store.hh new file mode 100644 index 000000000000..3aa2cd3455e9 --- /dev/null +++ b/src/libstore-test-support/include/nix/store/tests/https-store.hh @@ -0,0 +1,99 @@ +#pragma once +///@file + +#include +#include + +#include "nix/store/tests/libstore-network.hh" +#include "nix/store/http-binary-cache-store.hh" +#include "nix/store/store-api.hh" +#include "nix/store/globals.hh" +#include "nix/store/local-binary-cache-store.hh" +#include "nix/util/file-system.hh" +#include "nix/util/processes.hh" + +namespace nix::testing { + +class TestHttpBinaryCacheStoreConfig; + +/** + * Test shim for testing. We don't want to use the on-disk narinfo cache in unit + * tests. + */ +class TestHttpBinaryCacheStore : public HttpBinaryCacheStore +{ +public: + TestHttpBinaryCacheStore(const TestHttpBinaryCacheStore &) = delete; + TestHttpBinaryCacheStore(TestHttpBinaryCacheStore &&) = delete; + TestHttpBinaryCacheStore & operator=(const TestHttpBinaryCacheStore &) = delete; + TestHttpBinaryCacheStore & operator=(TestHttpBinaryCacheStore &&) = delete; + + TestHttpBinaryCacheStore(ref config, ref fileTransfer) + : Store{*config} + , BinaryCacheStore{*config} + , HttpBinaryCacheStore(config, fileTransfer) + { + diskCache = nullptr; /* Disable caching, we'll be creating a new binary cache for each test. */ + } + + void init() override; +}; + +class TestHttpBinaryCacheStoreConfig : public HttpBinaryCacheStoreConfig +{ +public: + TestHttpBinaryCacheStoreConfig(ParsedURL url, const Store::Config::Params & params) + : StoreConfig(params) + , HttpBinaryCacheStoreConfig(url, params) + { + } + + ref openTestStore(ref fileTransfer) const; +}; + +class HttpsBinaryCacheStoreTest : public virtual LibStoreNetworkTest +{ + std::unique_ptr delTmpDir; + +public: + static void SetUpTestSuite() + { + initLibStore(/*loadConfig=*/false); + } + +protected: + std::filesystem::path tmpDir, cacheDir; + std::filesystem::path caCert, caKey, serverCert, serverKey; + std::filesystem::path clientCert, clientKey; + Pid serverPid; + uint16_t port = 8443; + std::shared_ptr localCacheStore; + + /** + * Custom FileTransferSettings with the test CA certificate. + * This is used instead of modifying global settings. + */ + std::unique_ptr testFileTransferSettings; + + /** + * FileTransfer instance using our test settings. + * Initialized in SetUp(). + */ + std::shared_ptr testFileTransfer; + + static void openssl(Strings args); + void SetUp() override; + void TearDown() override; + + virtual std::vector serverArgs(); + ref makeConfig(); + ref openStore(ref config); +}; + +class HttpsBinaryCacheStoreMtlsTest : public HttpsBinaryCacheStoreTest +{ +protected: + std::vector serverArgs() override; +}; + +} // namespace nix::testing diff --git a/src/libstore-test-support/include/nix/store/tests/libstore-network.hh b/src/libstore-test-support/include/nix/store/tests/libstore-network.hh new file mode 100644 index 000000000000..ab03ace56182 --- /dev/null +++ b/src/libstore-test-support/include/nix/store/tests/libstore-network.hh @@ -0,0 +1,39 @@ +#pragma once +/// @file + +#include + +namespace nix::testing { + +/** + * Whether to run network tests. This is global so that the test harness can + * enable this by default if we can run tests in isolation. + */ +extern bool networkTestsAvailable; + +/** + * Set up network tests and, if on linux, create a new network namespace for + * tests with a loopback interface. This is to avoid binding to ports in the + * host's namespace. + */ +void setupNetworkTests(); + +class LibStoreNetworkTest : public virtual ::testing::Test +{ +protected: + void SetUp() override + { + if (networkTestsAvailable) + return; + static bool warned = false; + if (!warned) { + warned = true; + GTEST_SKIP() + << "Network tests not enabled by default without user namespaces, use NIX_TEST_FORCE_NETWORK_TESTS=1 to override"; + } else { + GTEST_SKIP(); + } + } +}; + +} // namespace nix::testing diff --git a/src/libstore-test-support/include/nix/store/tests/meson.build b/src/libstore-test-support/include/nix/store/tests/meson.build index 33524de3851a..8d844d24e7d8 100644 --- a/src/libstore-test-support/include/nix/store/tests/meson.build +++ b/src/libstore-test-support/include/nix/store/tests/meson.build @@ -4,6 +4,8 @@ include_dirs = [ include_directories('../../..') ] headers = files( 'derived-path.hh', + 'https-store.hh', + 'libstore-network.hh', 'libstore.hh', 'nix_api_store.hh', 'outputs-spec.hh', diff --git a/src/libstore-test-support/include/nix/store/tests/nix_api_store.hh b/src/libstore-test-support/include/nix/store/tests/nix_api_store.hh index a35d2b1eede6..bb9e5a3038fd 100644 --- a/src/libstore-test-support/include/nix/store/tests/nix_api_store.hh +++ b/src/libstore-test-support/include/nix/store/tests/nix_api_store.hh @@ -50,8 +50,7 @@ protected: #else // resolve any symlinks in i.e. on macOS /tmp -> /private/tmp // because this is not allowed for a nix store. - auto tmpl = - nix::absPath(std::filesystem::path(nix::defaultTempDir()) / "tests_nix-store.XXXXXX", std::nullopt, true); + auto tmpl = nix::absPath(nix::defaultTempDir() / "tests_nix-store.XXXXXX", nullptr, true); nixDir = mkdtemp((char *) tmpl.c_str()); #endif diff --git a/src/libstore-test-support/include/nix/store/tests/protocol.hh b/src/libstore-test-support/include/nix/store/tests/protocol.hh index 0f774df0ec0b..563c8cfb6c4e 100644 --- a/src/libstore-test-support/include/nix/store/tests/protocol.hh +++ b/src/libstore-test-support/include/nix/store/tests/protocol.hh @@ -21,14 +21,14 @@ class ProtoTest : public CharacterizationTest } public: - Path storeDir = "/nix/store"; + std::string storeDir = "/nix/store"; StoreDirConfig store{storeDir}; /** * Golden test for `T` JSON reading */ template - void readJsonTest(PathView testStem, const T & expected) + void readJsonTest(std::string_view testStem, const T & expected) { nix::readJsonTest(*this, testStem, expected); } @@ -37,7 +37,7 @@ public: * Golden test for `T` JSON write */ template - void writeJsonTest(PathView testStem, const T & decoded) + void writeJsonTest(std::string_view testStem, const T & decoded) { nix::writeJsonTest(*this, testStem, decoded); } @@ -51,7 +51,7 @@ public: * Golden test for `T` reading */ template - void readProtoTest(PathView testStem, typename Proto::Version version, T expected) + void readProtoTest(std::string_view testStem, typename Proto::Version version, T expected) { CharacterizationTest::readTest(std::string{testStem + ".bin"}, [&](const auto & encoded) { T got = ({ @@ -72,7 +72,7 @@ public: * Golden test for `T` write */ template - void writeProtoTest(PathView testStem, typename Proto::Version version, const T & decoded) + void writeProtoTest(std::string_view testStem, typename Proto::Version version, const T & decoded) { CharacterizationTest::writeTest(std::string{testStem + ".bin"}, [&]() { StringSink to; @@ -88,25 +88,38 @@ public: } }; -#define VERSIONED_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, VERSION, VALUE) \ - TEST_F(FIXTURE, NAME##_read) \ - { \ - readProtoTest(STEM, VERSION, VALUE); \ - } \ - TEST_F(FIXTURE, NAME##_write) \ - { \ - writeProtoTest(STEM, VERSION, VALUE); \ +#define VERSIONED_READ_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, VERSION, VALUE) \ + TEST_F(FIXTURE, NAME##_read) \ + { \ + readProtoTest(STEM, VERSION, VALUE); \ } -#define VERSIONED_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, VERSION, VALUE) \ - VERSIONED_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, VERSION, VALUE) \ - TEST_F(FIXTURE, NAME##_json_read) \ - { \ - readJsonTest(STEM, VALUE); \ - } \ - TEST_F(FIXTURE, NAME##_json_write) \ - { \ - writeJsonTest(STEM, VALUE); \ +#define VERSIONED_WRITE_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, VERSION, VALUE) \ + TEST_F(FIXTURE, NAME##_write) \ + { \ + writeProtoTest(STEM, VERSION, VALUE); \ } +#define VERSIONED_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, VERSION, VALUE) \ + VERSIONED_READ_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, (VERSION), VALUE) \ + VERSIONED_WRITE_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, (VERSION), VALUE) + +#define VERSIONED_READ_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, VERSION, VALUE) \ + VERSIONED_READ_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, (VERSION), VALUE) \ + TEST_F(FIXTURE, NAME##_json_read) \ + { \ + readJsonTest(STEM, VALUE); \ + } + +#define VERSIONED_WRITE_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, VERSION, VALUE) \ + VERSIONED_WRITE_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, (VERSION), VALUE) \ + TEST_F(FIXTURE, NAME##_json_write) \ + { \ + writeJsonTest(STEM, VALUE); \ + } + +#define VERSIONED_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, VERSION, VALUE) \ + VERSIONED_READ_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, (VERSION), VALUE) \ + VERSIONED_WRITE_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, (VERSION), VALUE) + } // namespace nix diff --git a/src/libstore-test-support/libstore-network.cc b/src/libstore-test-support/libstore-network.cc new file mode 100644 index 000000000000..8aa047bdd609 --- /dev/null +++ b/src/libstore-test-support/libstore-network.cc @@ -0,0 +1,60 @@ +#include "nix/store/tests/libstore-network.hh" +#include "nix/util/error.hh" +#include "nix/util/environment-variables.hh" + +#ifdef __linux__ +# include "nix/util/file-system.hh" +# include "nix/util/linux-namespaces.hh" +# include +# include +# include +# include +#endif + +namespace nix::testing { + +bool networkTestsAvailable = false; + +#ifdef __linux__ + +static void enterNetworkNamespace() +{ + auto uid = ::getuid(); + auto gid = ::getgid(); + + if (::unshare(CLONE_NEWUSER | CLONE_NEWNET) == -1) + throw SysError("setting up a private network namespace for tests"); + + std::filesystem::path procSelf = "/proc/self"; + writeFile(procSelf / "setgroups", "deny"); + writeFile(procSelf / "uid_map", fmt("%d %d 1", uid, uid)); + writeFile(procSelf / "gid_map", fmt("%d %d 1", gid, gid)); + + AutoCloseFD fd(::socket(PF_INET, SOCK_DGRAM, IPPROTO_IP)); + if (!fd) + throw SysError("cannot open IP socket for loopback interface"); + + struct ::ifreq ifr = {}; + strcpy(ifr.ifr_name, "lo"); + ifr.ifr_flags = IFF_UP | IFF_LOOPBACK | IFF_RUNNING; + if (::ioctl(fd.get(), SIOCSIFFLAGS, &ifr) == -1) + throw SysError("cannot set loopback interface flags"); +} + +#endif + +void setupNetworkTests() +try { + networkTestsAvailable = getEnvOs(OS_STR("NIX_TEST_FORCE_NETWORK_TESTS")).has_value(); + +#ifdef __linux__ + if (!networkTestsAvailable && userNamespacesSupported()) { + enterNetworkNamespace(); + networkTestsAvailable = true; + } +#endif +} catch (SystemError & e) { + /* Ignore any set up errors. */ +} + +} // namespace nix::testing diff --git a/src/libstore-test-support/meson.build b/src/libstore-test-support/meson.build index d3d925d26848..c508eeaecc70 100644 --- a/src/libstore-test-support/meson.build +++ b/src/libstore-test-support/meson.build @@ -28,10 +28,15 @@ subdir('nix-meson-build-support/subprojects') rapidcheck = dependency('rapidcheck') deps_public += rapidcheck +gtest = dependency('gtest') +deps_public += gtest + subdir('nix-meson-build-support/common') sources = files( 'derived-path.cc', + 'https-store.cc', + 'libstore-network.cc', 'outputs-spec.cc', 'path.cc', 'test-main.cc', diff --git a/src/libstore-test-support/package.nix b/src/libstore-test-support/package.nix index 2561dd791eb7..bd5a4c0aa7a0 100644 --- a/src/libstore-test-support/package.nix +++ b/src/libstore-test-support/package.nix @@ -7,6 +7,7 @@ nix-store-c, rapidcheck, + gtest, # Configuration Options @@ -39,6 +40,7 @@ mkMesonLibrary (finalAttrs: { nix-store nix-store-c rapidcheck + gtest ]; mesonFlags = [ diff --git a/src/libstore-test-support/test-main.cc b/src/libstore-test-support/test-main.cc index 0b9072dc08f6..7c4a56ecc266 100644 --- a/src/libstore-test-support/test-main.cc +++ b/src/libstore-test-support/test-main.cc @@ -15,10 +15,10 @@ int testMainForBuidingPre(int argc, char ** argv) } // Disable build hook. We won't be testing remote builds in these unit tests. If we do, fix the above build hook. - settings.buildHook = {}; + settings.getWorkerSettings().buildHook = {}; // No substituters, unless a test specifically requests. - settings.substituters = {}; + settings.getWorkerSettings().substituters = {}; #ifdef __linux__ // should match the conditional around sandboxBuildDir declaration. @@ -31,13 +31,13 @@ int testMainForBuidingPre(int argc, char ** argv) // sandboxBuildDir = /build // However, we have a rule that the store dir must not be inside the storeDir, so we need to pick a different // sandboxBuildDir. - settings.sandboxBuildDir = "/test-build-dir-instead-of-usual-build-dir"; + settings.getLocalSettings().sandboxBuildDir = "/test-build-dir-instead-of-usual-build-dir"; #endif #ifdef __APPLE__ // Avoid this error, when already running in a sandbox: // sandbox-exec: sandbox_apply: Operation not permitted - settings.sandboxMode = smDisabled; + settings.getLocalSettings().sandboxMode = smDisabled; setEnv("_NIX_TEST_NO_SANDBOX", "1"); #endif diff --git a/src/libstore-tests/build-result.cc b/src/libstore-tests/build-result.cc index 85e799c2a737..b7e8f83f9e7c 100644 --- a/src/libstore-tests/build-result.cc +++ b/src/libstore-tests/build-result.cc @@ -1,6 +1,7 @@ #include #include "nix/store/build-result.hh" +#include "nix/util/tests/characterization.hh" #include "nix/util/tests/json-characterization.hh" namespace nix { @@ -44,22 +45,22 @@ INSTANTIATE_TEST_SUITE_P( std::pair{ "not-deterministic", BuildResult{ - .inner{BuildResult::Failure{ + .inner{BuildResult::Failure{{ .status = BuildResult::Failure::NotDeterministic, - .errorMsg = "no idea why", + .msg = HintFmt("no idea why"), .isNonDeterministic = false, // Note: This field is separate from the status - }}, + }}}, .timesBuilt = 1, }, }, std::pair{ "output-rejected", BuildResult{ - .inner{BuildResult::Failure{ + .inner{BuildResult::Failure{{ .status = BuildResult::Failure::OutputRejected, - .errorMsg = "no idea why", + .msg = HintFmt("no idea why"), .isNonDeterministic = false, - }}, + }}}, .timesBuilt = 3, .startTime = 30, .stopTime = 50, diff --git a/src/libstore-tests/common-protocol.cc b/src/libstore-tests/common-protocol.cc index fa676eb7f4e1..d63cbeeeed2c 100644 --- a/src/libstore-tests/common-protocol.cc +++ b/src/libstore-tests/common-protocol.cc @@ -21,7 +21,7 @@ class CommonProtoTest : public ProtoTest * Golden test for `T` reading */ template - void readProtoTest(PathView testStem, const T & expected) + void readProtoTest(std::string_view testStem, const T & expected) { CharacterizationTest::readTest(std::string{testStem + ".bin"}, [&](const auto & encoded) { T got = ({ @@ -37,7 +37,7 @@ class CommonProtoTest : public ProtoTest * Golden test for `T` write */ template - void writeProtoTest(PathView testStem, const T & decoded) + void writeProtoTest(std::string_view testStem, const T & decoded) { CharacterizationTest::writeTest(std::string{testStem + ".bin"}, [&]() -> std::string { StringSink to; @@ -47,24 +47,30 @@ class CommonProtoTest : public ProtoTest } }; -#define CHARACTERIZATION_TEST(NAME, STEM, VALUE) \ - TEST_F(CommonProtoTest, NAME##_read) \ - { \ - readProtoTest(STEM, VALUE); \ - } \ - TEST_F(CommonProtoTest, NAME##_write) \ - { \ - writeProtoTest(STEM, VALUE); \ - } \ - TEST_F(CommonProtoTest, NAME##_json_read) \ - { \ - readJsonTest(STEM, VALUE); \ - } \ - TEST_F(CommonProtoTest, NAME##_json_write) \ - { \ - writeJsonTest(STEM, VALUE); \ +#define READ_CHARACTERIZATION_TEST(NAME, STEM, VALUE) \ + TEST_F(CommonProtoTest, NAME##_read) \ + { \ + readProtoTest(STEM, VALUE); \ + } \ + TEST_F(CommonProtoTest, NAME##_json_read) \ + { \ + readJsonTest(STEM, VALUE); \ } +#define WRITE_CHARACTERIZATION_TEST(NAME, STEM, VALUE) \ + TEST_F(CommonProtoTest, NAME##_write) \ + { \ + writeProtoTest(STEM, VALUE); \ + } \ + TEST_F(CommonProtoTest, NAME##_json_write) \ + { \ + writeJsonTest(STEM, VALUE); \ + } + +#define CHARACTERIZATION_TEST(NAME, STEM, VALUE) \ + READ_CHARACTERIZATION_TEST(NAME, STEM, VALUE) \ + WRITE_CHARACTERIZATION_TEST(NAME, STEM, VALUE) + CHARACTERIZATION_TEST( string, "string", @@ -132,7 +138,11 @@ CHARACTERIZATION_TEST( Realisation{ { .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - .signatures = {"asdf", "qwer"}, + .signatures = + { + Signature{.keyName = "asdf", .sig = std::string(64, '\0')}, + Signature{.keyName = "qwer", .sig = std::string(64, '\0')}, + }, }, { .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), @@ -141,23 +151,17 @@ CHARACTERIZATION_TEST( }, })) -CHARACTERIZATION_TEST( +READ_CHARACTERIZATION_TEST( realisation_with_deps, "realisation-with-deps", (std::tuple{ Realisation{ { .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - .signatures = {"asdf", "qwer"}, - .dependentRealisations = + .signatures = { - { - DrvOutput{ - .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), - .outputName = "quux", - }, - StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - }, + Signature{.keyName = "asdf", .sig = std::string(64, '\0')}, + Signature{.keyName = "qwer", .sig = std::string(64, '\0')}, }, }, { diff --git a/src/libstore-tests/data/common-protocol/realisation-with-deps.bin b/src/libstore-tests/data/common-protocol/realisation-with-deps.bin index 54a78b64ebcf..ed728c886cf1 100644 Binary files a/src/libstore-tests/data/common-protocol/realisation-with-deps.bin and b/src/libstore-tests/data/common-protocol/realisation-with-deps.bin differ diff --git a/src/libstore-tests/data/common-protocol/realisation-with-deps.json b/src/libstore-tests/data/common-protocol/realisation-with-deps.json index 77148d14ca48..790f87edd5d0 100644 --- a/src/libstore-tests/data/common-protocol/realisation-with-deps.json +++ b/src/libstore-tests/data/common-protocol/realisation-with-deps.json @@ -6,8 +6,8 @@ "id": "sha256:15e3c560894cbb27085cf65b5a2ecb18488c999497f4531b6907a7581ce6d527!baz", "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo", "signatures": [ - "asdf", - "qwer" + "asdf:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", + "qwer:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" ] } ] diff --git a/src/libstore-tests/data/common-protocol/realisation.bin b/src/libstore-tests/data/common-protocol/realisation.bin index 3a0b2b2d8e39..44885cbdafda 100644 Binary files a/src/libstore-tests/data/common-protocol/realisation.bin and b/src/libstore-tests/data/common-protocol/realisation.bin differ diff --git a/src/libstore-tests/data/common-protocol/realisation.json b/src/libstore-tests/data/common-protocol/realisation.json index f9ff09dbb63d..034d620306f0 100644 --- a/src/libstore-tests/data/common-protocol/realisation.json +++ b/src/libstore-tests/data/common-protocol/realisation.json @@ -10,8 +10,8 @@ "id": "sha256:15e3c560894cbb27085cf65b5a2ecb18488c999497f4531b6907a7581ce6d527!baz", "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo", "signatures": [ - "asdf", - "qwer" + "asdf:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", + "qwer:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" ] } ] diff --git a/src/libstore-tests/data/nar-info/json-1/impure.json b/src/libstore-tests/data/nar-info/json-1/impure.json index c6fafe13daa6..a05e54897dd7 100644 --- a/src/libstore-tests/data/nar-info/json-1/impure.json +++ b/src/libstore-tests/data/nar-info/json-1/impure.json @@ -12,8 +12,8 @@ ], "registrationTime": 23423, "signatures": [ - "asdf", - "qwer" + "asdf:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", + "qwer:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" ], "storeDir": "/nix/store", "ultimate": true, diff --git a/src/libstore-tests/data/nar-info/json-2/impure.json b/src/libstore-tests/data/nar-info/json-2/impure.json index b7b9f511827c..9af755ffd403 100644 --- a/src/libstore-tests/data/nar-info/json-2/impure.json +++ b/src/libstore-tests/data/nar-info/json-2/impure.json @@ -15,8 +15,8 @@ ], "registrationTime": 23423, "signatures": [ - "asdf", - "qwer" + "asdf:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", + "qwer:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" ], "storeDir": "/nix/store", "ultimate": true, diff --git a/src/libstore-tests/data/path-info/json-1/impure.json b/src/libstore-tests/data/path-info/json-1/impure.json index 04d1dedc2af3..dce4c8685c48 100644 --- a/src/libstore-tests/data/path-info/json-1/impure.json +++ b/src/libstore-tests/data/path-info/json-1/impure.json @@ -9,8 +9,8 @@ ], "registrationTime": 23423, "signatures": [ - "asdf", - "qwer" + "asdf:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", + "qwer:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" ], "storeDir": "/nix/store", "ultimate": true, diff --git a/src/libstore-tests/data/path-info/json-2/impure.json b/src/libstore-tests/data/path-info/json-2/impure.json index bed67610b1b7..2131f28c1ce1 100644 --- a/src/libstore-tests/data/path-info/json-2/impure.json +++ b/src/libstore-tests/data/path-info/json-2/impure.json @@ -12,8 +12,8 @@ ], "registrationTime": 23423, "signatures": [ - "asdf", - "qwer" + "asdf:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", + "qwer:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" ], "storeDir": "/nix/store", "ultimate": true, diff --git a/src/libstore-tests/data/realisation/with-signature.json b/src/libstore-tests/data/realisation/with-signature.json index a28848cb02bd..3270f1cdebf4 100644 --- a/src/libstore-tests/data/realisation/with-signature.json +++ b/src/libstore-tests/data/realisation/with-signature.json @@ -3,6 +3,6 @@ "id": "sha256:ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad!foo", "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv", "signatures": [ - "asdfasdfasdf" + "asdf:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" ] } diff --git a/src/libstore-tests/data/serve-protocol/realisation-with-deps.bin b/src/libstore-tests/data/serve-protocol/realisation-with-deps.bin index 54a78b64ebcf..ed728c886cf1 100644 Binary files a/src/libstore-tests/data/serve-protocol/realisation-with-deps.bin and b/src/libstore-tests/data/serve-protocol/realisation-with-deps.bin differ diff --git a/src/libstore-tests/data/serve-protocol/realisation-with-deps.json b/src/libstore-tests/data/serve-protocol/realisation-with-deps.json index 77148d14ca48..790f87edd5d0 100644 --- a/src/libstore-tests/data/serve-protocol/realisation-with-deps.json +++ b/src/libstore-tests/data/serve-protocol/realisation-with-deps.json @@ -6,8 +6,8 @@ "id": "sha256:15e3c560894cbb27085cf65b5a2ecb18488c999497f4531b6907a7581ce6d527!baz", "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo", "signatures": [ - "asdf", - "qwer" + "asdf:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", + "qwer:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" ] } ] diff --git a/src/libstore-tests/data/serve-protocol/realisation.bin b/src/libstore-tests/data/serve-protocol/realisation.bin index 3a0b2b2d8e39..44885cbdafda 100644 Binary files a/src/libstore-tests/data/serve-protocol/realisation.bin and b/src/libstore-tests/data/serve-protocol/realisation.bin differ diff --git a/src/libstore-tests/data/serve-protocol/realisation.json b/src/libstore-tests/data/serve-protocol/realisation.json index f9ff09dbb63d..034d620306f0 100644 --- a/src/libstore-tests/data/serve-protocol/realisation.json +++ b/src/libstore-tests/data/serve-protocol/realisation.json @@ -10,8 +10,8 @@ "id": "sha256:15e3c560894cbb27085cf65b5a2ecb18488c999497f4531b6907a7581ce6d527!baz", "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo", "signatures": [ - "asdf", - "qwer" + "asdf:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", + "qwer:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" ] } ] diff --git a/src/libstore-tests/data/serve-protocol/unkeyed-valid-path-info-2.4.bin b/src/libstore-tests/data/serve-protocol/unkeyed-valid-path-info-2.4.bin index 521b5c423c68..cd52e3477da2 100644 Binary files a/src/libstore-tests/data/serve-protocol/unkeyed-valid-path-info-2.4.bin and b/src/libstore-tests/data/serve-protocol/unkeyed-valid-path-info-2.4.bin differ diff --git a/src/libstore-tests/data/serve-protocol/unkeyed-valid-path-info-2.4.json b/src/libstore-tests/data/serve-protocol/unkeyed-valid-path-info-2.4.json index 801f20400026..9c1fa3134d49 100644 --- a/src/libstore-tests/data/serve-protocol/unkeyed-valid-path-info-2.4.json +++ b/src/libstore-tests/data/serve-protocol/unkeyed-valid-path-info-2.4.json @@ -27,8 +27,8 @@ ], "registrationTime": null, "signatures": [ - "fake-sig-1", - "fake-sig-2" + "fake-sig-1:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", + "fake-sig-2:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" ], "storeDir": "/nix/store", "ultimate": false, diff --git a/src/libstore-tests/data/store-reference/local_shorthand_2.txt b/src/libstore-tests/data/store-reference/local_shorthand_path_unix.txt similarity index 100% rename from src/libstore-tests/data/store-reference/local_shorthand_2.txt rename to src/libstore-tests/data/store-reference/local_shorthand_path_unix.txt diff --git a/src/libstore-tests/data/store-reference/local_shorthand_path_windows.txt b/src/libstore-tests/data/store-reference/local_shorthand_path_windows.txt new file mode 100644 index 000000000000..e3c0179e3002 --- /dev/null +++ b/src/libstore-tests/data/store-reference/local_shorthand_path_windows.txt @@ -0,0 +1 @@ +C:\foo\bar\baz?trusted=true \ No newline at end of file diff --git a/src/libstore-tests/data/worker-protocol/realisation-with-deps.bin b/src/libstore-tests/data/worker-protocol/realisation-with-deps.bin index 54a78b64ebcf..ed728c886cf1 100644 Binary files a/src/libstore-tests/data/worker-protocol/realisation-with-deps.bin and b/src/libstore-tests/data/worker-protocol/realisation-with-deps.bin differ diff --git a/src/libstore-tests/data/worker-protocol/realisation-with-deps.json b/src/libstore-tests/data/worker-protocol/realisation-with-deps.json index 77148d14ca48..790f87edd5d0 100644 --- a/src/libstore-tests/data/worker-protocol/realisation-with-deps.json +++ b/src/libstore-tests/data/worker-protocol/realisation-with-deps.json @@ -6,8 +6,8 @@ "id": "sha256:15e3c560894cbb27085cf65b5a2ecb18488c999497f4531b6907a7581ce6d527!baz", "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo", "signatures": [ - "asdf", - "qwer" + "asdf:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", + "qwer:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" ] } ] diff --git a/src/libstore-tests/data/worker-protocol/realisation.bin b/src/libstore-tests/data/worker-protocol/realisation.bin index 3a0b2b2d8e39..44885cbdafda 100644 Binary files a/src/libstore-tests/data/worker-protocol/realisation.bin and b/src/libstore-tests/data/worker-protocol/realisation.bin differ diff --git a/src/libstore-tests/data/worker-protocol/realisation.json b/src/libstore-tests/data/worker-protocol/realisation.json index f9ff09dbb63d..034d620306f0 100644 --- a/src/libstore-tests/data/worker-protocol/realisation.json +++ b/src/libstore-tests/data/worker-protocol/realisation.json @@ -10,8 +10,8 @@ "id": "sha256:15e3c560894cbb27085cf65b5a2ecb18488c999497f4531b6907a7581ce6d527!baz", "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo", "signatures": [ - "asdf", - "qwer" + "asdf:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", + "qwer:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" ] } ] diff --git a/src/libstore-tests/data/worker-protocol/valid-path-info-1.16.bin b/src/libstore-tests/data/worker-protocol/valid-path-info-1.16.bin index a72de6bd62ab..1d817ebbcb5e 100644 Binary files a/src/libstore-tests/data/worker-protocol/valid-path-info-1.16.bin and b/src/libstore-tests/data/worker-protocol/valid-path-info-1.16.bin differ diff --git a/src/libstore-tests/data/worker-protocol/valid-path-info-1.16.json b/src/libstore-tests/data/worker-protocol/valid-path-info-1.16.json index f980d842174a..2c377145a10c 100644 --- a/src/libstore-tests/data/worker-protocol/valid-path-info-1.16.json +++ b/src/libstore-tests/data/worker-protocol/valid-path-info-1.16.json @@ -24,8 +24,8 @@ ], "registrationTime": 23423, "signatures": [ - "fake-sig-1", - "fake-sig-2" + "fake-sig-1:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", + "fake-sig-2:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" ], "storeDir": "/nix/store", "ultimate": false, diff --git a/src/libstore-tests/data/worker-substitution/ca-drv/store-after.json b/src/libstore-tests/data/worker-substitution/ca-drv/store-after.json new file mode 100644 index 000000000000..7f0f62f876c0 --- /dev/null +++ b/src/libstore-tests/data/worker-substitution/ca-drv/store-after.json @@ -0,0 +1,58 @@ +{ + "buildTrace": { + "gnRuK+wfbXqRPzgO5MyiBebXrV10Kzv+tkZCEuPm7pY=": { + "out": { + "dependentRealisations": {}, + "outPath": "hrva7l0gsk67wffmks761mv4ks4vzsx7-test-ca-drv-out", + "signatures": [] + } + } + }, + "config": { + "store": "/nix/store" + }, + "contents": { + "hrva7l0gsk67wffmks761mv4ks4vzsx7-test-ca-drv-out": { + "contents": { + "contents": "I am the output of a CA derivation", + "executable": false, + "type": "regular" + }, + "info": { + "ca": { + "hash": "sha256-l0+gmYB0AK65UWuoSh7AbVRI4rAc5/VGqzBGTHgMsiU=", + "method": "nar" + }, + "deriver": null, + "narHash": "sha256-l0+gmYB0AK65UWuoSh7AbVRI4rAc5/VGqzBGTHgMsiU=", + "narSize": 152, + "references": [], + "registrationTime": null, + "signatures": [], + "storeDir": "/nix/store", + "ultimate": false, + "version": 2 + } + } + }, + "derivations": { + "vvyyj6h5ilinsv4q48q5y5vn7s3hxmhl-test-ca-drv.drv": { + "args": [], + "builder": "", + "env": {}, + "inputs": { + "drvs": {}, + "srcs": [] + }, + "name": "test-ca-drv", + "outputs": { + "out": { + "hashAlgo": "sha256", + "method": "nar" + } + }, + "system": "", + "version": 4 + } + } +} diff --git a/src/libstore-tests/data/worker-substitution/ca-drv/store-before.json b/src/libstore-tests/data/worker-substitution/ca-drv/store-before.json new file mode 100644 index 000000000000..6d64d23270ca --- /dev/null +++ b/src/libstore-tests/data/worker-substitution/ca-drv/store-before.json @@ -0,0 +1,27 @@ +{ + "buildTrace": {}, + "config": { + "store": "/nix/store" + }, + "contents": {}, + "derivations": { + "vvyyj6h5ilinsv4q48q5y5vn7s3hxmhl-test-ca-drv.drv": { + "args": [], + "builder": "", + "env": {}, + "inputs": { + "drvs": {}, + "srcs": [] + }, + "name": "test-ca-drv", + "outputs": { + "out": { + "hashAlgo": "sha256", + "method": "nar" + } + }, + "system": "", + "version": 4 + } + } +} diff --git a/src/libstore-tests/data/worker-substitution/ca-drv/substituter.json b/src/libstore-tests/data/worker-substitution/ca-drv/substituter.json new file mode 100644 index 000000000000..93f4fb22a620 --- /dev/null +++ b/src/libstore-tests/data/worker-substitution/ca-drv/substituter.json @@ -0,0 +1,39 @@ +{ + "buildTrace": { + "gnRuK+wfbXqRPzgO5MyiBebXrV10Kzv+tkZCEuPm7pY=": { + "out": { + "dependentRealisations": {}, + "outPath": "hrva7l0gsk67wffmks761mv4ks4vzsx7-test-ca-drv-out", + "signatures": [] + } + } + }, + "config": { + "store": "/nix/store" + }, + "contents": { + "hrva7l0gsk67wffmks761mv4ks4vzsx7-test-ca-drv-out": { + "contents": { + "contents": "I am the output of a CA derivation", + "executable": false, + "type": "regular" + }, + "info": { + "ca": { + "hash": "sha256-l0+gmYB0AK65UWuoSh7AbVRI4rAc5/VGqzBGTHgMsiU=", + "method": "nar" + }, + "deriver": null, + "narHash": "sha256-l0+gmYB0AK65UWuoSh7AbVRI4rAc5/VGqzBGTHgMsiU=", + "narSize": 152, + "references": [], + "registrationTime": null, + "signatures": [], + "storeDir": "/nix/store", + "ultimate": false, + "version": 2 + } + } + }, + "derivations": {} +} diff --git a/src/libstore-tests/data/worker-substitution/issue-11928/store-after.json b/src/libstore-tests/data/worker-substitution/issue-11928/store-after.json new file mode 100644 index 000000000000..617984a27eb6 --- /dev/null +++ b/src/libstore-tests/data/worker-substitution/issue-11928/store-after.json @@ -0,0 +1,83 @@ +{ + "buildTrace": { + "8vEkprm3vQ3BE6JLB8XKfU+AdAwEFOMI/skzyj3pr5I=": { + "out": { + "dependentRealisations": {}, + "outPath": "px7apdw6ydm9ynjy5g0bpdcylw3xz2kj-root-drv-out", + "signatures": [] + } + } + }, + "config": { + "store": "/nix/store" + }, + "contents": { + "px7apdw6ydm9ynjy5g0bpdcylw3xz2kj-root-drv-out": { + "contents": { + "contents": "I am the root output. I don't reference anything because the other derivation's output is just needed at build time.", + "executable": false, + "type": "regular" + }, + "info": { + "ca": { + "hash": "sha256-0mlhg9y1FGb7YsHAsNOmtuW44b8TfoPaNPK6SjVYe5s=", + "method": "nar" + }, + "deriver": null, + "narHash": "sha256-0mlhg9y1FGb7YsHAsNOmtuW44b8TfoPaNPK6SjVYe5s=", + "narSize": 232, + "references": [], + "registrationTime": null, + "signatures": [], + "storeDir": "/nix/store", + "ultimate": false, + "version": 2 + } + } + }, + "derivations": { + "11yvkl84ashq63ilwc2mi4va41z2disw-root-drv.drv": { + "args": [], + "builder": "", + "env": {}, + "inputs": { + "drvs": { + "vy7j6m6p5y0327fhk3zxn12hbpzkh6lp-dep-drv.drv": { + "dynamicOutputs": {}, + "outputs": [ + "out" + ] + } + }, + "srcs": [] + }, + "name": "root-drv", + "outputs": { + "out": { + "hashAlgo": "sha256", + "method": "nar" + } + }, + "system": "", + "version": 4 + }, + "vy7j6m6p5y0327fhk3zxn12hbpzkh6lp-dep-drv.drv": { + "args": [], + "builder": "", + "env": {}, + "inputs": { + "drvs": {}, + "srcs": [] + }, + "name": "dep-drv", + "outputs": { + "out": { + "hashAlgo": "sha256", + "method": "nar" + } + }, + "system": "", + "version": 4 + } + } +} diff --git a/src/libstore-tests/data/worker-substitution/issue-11928/store-before.json b/src/libstore-tests/data/worker-substitution/issue-11928/store-before.json new file mode 100644 index 000000000000..47a3b34beee4 --- /dev/null +++ b/src/libstore-tests/data/worker-substitution/issue-11928/store-before.json @@ -0,0 +1,52 @@ +{ + "buildTrace": {}, + "config": { + "store": "/nix/store" + }, + "contents": {}, + "derivations": { + "11yvkl84ashq63ilwc2mi4va41z2disw-root-drv.drv": { + "args": [], + "builder": "", + "env": {}, + "inputs": { + "drvs": { + "vy7j6m6p5y0327fhk3zxn12hbpzkh6lp-dep-drv.drv": { + "dynamicOutputs": {}, + "outputs": [ + "out" + ] + } + }, + "srcs": [] + }, + "name": "root-drv", + "outputs": { + "out": { + "hashAlgo": "sha256", + "method": "nar" + } + }, + "system": "", + "version": 4 + }, + "vy7j6m6p5y0327fhk3zxn12hbpzkh6lp-dep-drv.drv": { + "args": [], + "builder": "", + "env": {}, + "inputs": { + "drvs": {}, + "srcs": [] + }, + "name": "dep-drv", + "outputs": { + "out": { + "hashAlgo": "sha256", + "method": "nar" + } + }, + "system": "", + "version": 4 + } + } +} diff --git a/src/libstore-tests/data/worker-substitution/issue-11928/substituter.json b/src/libstore-tests/data/worker-substitution/issue-11928/substituter.json new file mode 100644 index 000000000000..a63d6243fae3 --- /dev/null +++ b/src/libstore-tests/data/worker-substitution/issue-11928/substituter.json @@ -0,0 +1,68 @@ +{ + "buildTrace": { + "8vEkprm3vQ3BE6JLB8XKfU+AdAwEFOMI/skzyj3pr5I=": { + "out": { + "dependentRealisations": {}, + "outPath": "px7apdw6ydm9ynjy5g0bpdcylw3xz2kj-root-drv-out", + "signatures": [] + } + }, + "gnRuK+wfbXqRPzgO5MyiBebXrV10Kzv+tkZCEuPm7pY=": { + "out": { + "dependentRealisations": {}, + "outPath": "w0yjpwh59kpbyc7hz9jgmi44r9br908i-dep-drv-out", + "signatures": [] + } + } + }, + "config": { + "store": "/nix/store" + }, + "contents": { + "px7apdw6ydm9ynjy5g0bpdcylw3xz2kj-root-drv-out": { + "contents": { + "contents": "I am the root output. I don't reference anything because the other derivation's output is just needed at build time.", + "executable": false, + "type": "regular" + }, + "info": { + "ca": { + "hash": "sha256-0mlhg9y1FGb7YsHAsNOmtuW44b8TfoPaNPK6SjVYe5s=", + "method": "nar" + }, + "deriver": null, + "narHash": "sha256-0mlhg9y1FGb7YsHAsNOmtuW44b8TfoPaNPK6SjVYe5s=", + "narSize": 232, + "references": [], + "registrationTime": null, + "signatures": [], + "storeDir": "/nix/store", + "ultimate": false, + "version": 2 + } + }, + "w0yjpwh59kpbyc7hz9jgmi44r9br908i-dep-drv-out": { + "contents": { + "contents": "I am the dependency output", + "executable": false, + "type": "regular" + }, + "info": { + "ca": { + "hash": "sha256-HK2LBzSTtwuRjc44PH3Ac1JHHPKmfnAgNxz6I5mVgL8=", + "method": "nar" + }, + "deriver": null, + "narHash": "sha256-HK2LBzSTtwuRjc44PH3Ac1JHHPKmfnAgNxz6I5mVgL8=", + "narSize": 144, + "references": [], + "registrationTime": null, + "signatures": [], + "storeDir": "/nix/store", + "ultimate": false, + "version": 2 + } + } + }, + "derivations": {} +} diff --git a/src/libstore-tests/data/worker-substitution/single/substituter.json b/src/libstore-tests/data/worker-substitution/single/substituter.json new file mode 100644 index 000000000000..f22d4c7dfbf1 --- /dev/null +++ b/src/libstore-tests/data/worker-substitution/single/substituter.json @@ -0,0 +1,31 @@ +{ + "buildTrace": {}, + "config": { + "store": "/nix/store" + }, + "contents": { + "axqic2q30v0sqvcpiqxs139q8w6zd4n8-hello": { + "contents": { + "contents": "Hello, world!", + "executable": false, + "type": "regular" + }, + "info": { + "ca": { + "hash": "sha256-KShIJtIwWG1gMSpvPMt5drppc1h5WMwHWzVpNJiVqGI=", + "method": "nar" + }, + "deriver": null, + "narHash": "sha256-KShIJtIwWG1gMSpvPMt5drppc1h5WMwHWzVpNJiVqGI=", + "narSize": 128, + "references": [], + "registrationTime": null, + "signatures": [], + "storeDir": "/nix/store", + "ultimate": false, + "version": 2 + } + } + }, + "derivations": {} +} diff --git a/src/libstore-tests/data/worker-substitution/with-dep/store.json b/src/libstore-tests/data/worker-substitution/with-dep/store.json new file mode 100644 index 000000000000..3f2994dfb455 --- /dev/null +++ b/src/libstore-tests/data/worker-substitution/with-dep/store.json @@ -0,0 +1,55 @@ +{ + "buildTrace": {}, + "config": { + "store": "/nix/store" + }, + "contents": { + "4k79i02avcckr96r97lqnswn75fi1gv7-dependency": { + "contents": { + "contents": "I am a dependency", + "executable": false, + "type": "regular" + }, + "info": { + "ca": { + "hash": "sha256-miJnClL0Ai/HAmX1G/pz7P2TIaeFjP5D/VN1rhYf354=", + "method": "nar" + }, + "deriver": null, + "narHash": "sha256-miJnClL0Ai/HAmX1G/pz7P2TIaeFjP5D/VN1rhYf354=", + "narSize": 136, + "references": [], + "registrationTime": null, + "signatures": [], + "storeDir": "/nix/store", + "ultimate": false, + "version": 2 + } + }, + "k09ldq9fvxb6vfwq0cmv6j1jgqx08y1n-main": { + "contents": { + "contents": "I depend on /nix/store/4k79i02avcckr96r97lqnswn75fi1gv7-dependency", + "executable": false, + "type": "regular" + }, + "info": { + "ca": { + "hash": "sha256-CBfMK3HkqiXjpI8HNL1spWD/US4RnQHwI67Ojl50XoQ=", + "method": "nar" + }, + "deriver": null, + "narHash": "sha256-CBfMK3HkqiXjpI8HNL1spWD/US4RnQHwI67Ojl50XoQ=", + "narSize": 184, + "references": [ + "4k79i02avcckr96r97lqnswn75fi1gv7-dependency" + ], + "registrationTime": null, + "signatures": [], + "storeDir": "/nix/store", + "ultimate": false, + "version": 2 + } + } + }, + "derivations": {} +} diff --git a/src/libstore-tests/data/worker-substitution/with-dep/substituter.json b/src/libstore-tests/data/worker-substitution/with-dep/substituter.json new file mode 100644 index 000000000000..3f2994dfb455 --- /dev/null +++ b/src/libstore-tests/data/worker-substitution/with-dep/substituter.json @@ -0,0 +1,55 @@ +{ + "buildTrace": {}, + "config": { + "store": "/nix/store" + }, + "contents": { + "4k79i02avcckr96r97lqnswn75fi1gv7-dependency": { + "contents": { + "contents": "I am a dependency", + "executable": false, + "type": "regular" + }, + "info": { + "ca": { + "hash": "sha256-miJnClL0Ai/HAmX1G/pz7P2TIaeFjP5D/VN1rhYf354=", + "method": "nar" + }, + "deriver": null, + "narHash": "sha256-miJnClL0Ai/HAmX1G/pz7P2TIaeFjP5D/VN1rhYf354=", + "narSize": 136, + "references": [], + "registrationTime": null, + "signatures": [], + "storeDir": "/nix/store", + "ultimate": false, + "version": 2 + } + }, + "k09ldq9fvxb6vfwq0cmv6j1jgqx08y1n-main": { + "contents": { + "contents": "I depend on /nix/store/4k79i02avcckr96r97lqnswn75fi1gv7-dependency", + "executable": false, + "type": "regular" + }, + "info": { + "ca": { + "hash": "sha256-CBfMK3HkqiXjpI8HNL1spWD/US4RnQHwI67Ojl50XoQ=", + "method": "nar" + }, + "deriver": null, + "narHash": "sha256-CBfMK3HkqiXjpI8HNL1spWD/US4RnQHwI67Ojl50XoQ=", + "narSize": 184, + "references": [ + "4k79i02avcckr96r97lqnswn75fi1gv7-dependency" + ], + "registrationTime": null, + "signatures": [], + "storeDir": "/nix/store", + "ultimate": false, + "version": 2 + } + } + }, + "derivations": {} +} diff --git a/src/libstore-tests/derivation-advanced-attrs.cc b/src/libstore-tests/derivation-advanced-attrs.cc index 296ffed619b2..16988613c633 100644 --- a/src/libstore-tests/derivation-advanced-attrs.cc +++ b/src/libstore-tests/derivation-advanced-attrs.cc @@ -5,6 +5,7 @@ #include "nix/store/derivations.hh" #include "nix/store/derived-path.hh" #include "nix/store/derivation-options.hh" +#include "nix/store/globals.hh" #include "nix/store/parsed-derivations.hh" #include "nix/util/types.hh" #include "nix/util/json-utils.hh" @@ -192,9 +193,7 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_defaults) EXPECT_EQ(options, advancedAttributes_defaults); - EXPECT_EQ(options.canBuildLocally(*this->store, got), false); - EXPECT_EQ(options.willBuildLocally(*this->store, got), false); - EXPECT_EQ(options.substitutesAllowed(), true); + EXPECT_EQ(options.substitutesAllowed(settings.getWorkerSettings()), true); EXPECT_EQ(options.useUidRange(got), false); }); }; @@ -242,7 +241,7 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes) EXPECT_EQ(options, expected); - EXPECT_EQ(options.substitutesAllowed(), false); + EXPECT_EQ(options.substitutesAllowed(settings.getWorkerSettings()), false); EXPECT_EQ(options.useUidRange(got), true); }); }; @@ -334,9 +333,7 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_structuredAttrs_d EXPECT_EQ(options, advancedAttributes_structuredAttrs_defaults); - EXPECT_EQ(options.canBuildLocally(*this->store, got), false); - EXPECT_EQ(options.willBuildLocally(*this->store, got), false); - EXPECT_EQ(options.substitutesAllowed(), true); + EXPECT_EQ(options.substitutesAllowed(settings.getWorkerSettings()), true); EXPECT_EQ(options.useUidRange(got), false); }); }; @@ -401,9 +398,7 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_structuredAttrs) EXPECT_EQ(options, expected); - EXPECT_EQ(options.canBuildLocally(*this->store, got), false); - EXPECT_EQ(options.willBuildLocally(*this->store, got), false); - EXPECT_EQ(options.substitutesAllowed(), false); + EXPECT_EQ(options.substitutesAllowed(settings.getWorkerSettings()), false); EXPECT_EQ(options.useUidRange(got), true); }); }; diff --git a/src/libstore-tests/derivation-parser-bench.cc b/src/libstore-tests/derivation-parser-bench.cc index 1709eed1cdb8..f0aa721cb7a9 100644 --- a/src/libstore-tests/derivation-parser-bench.cc +++ b/src/libstore-tests/derivation-parser-bench.cc @@ -2,7 +2,7 @@ #include "nix/store/derivations.hh" #include "nix/store/store-api.hh" #include "nix/util/experimental-features.hh" -#include "nix/util/environment-variables.hh" +#include "nix/util/tests/test-data.hh" #include "nix/store/store-open.hh" #include #include @@ -50,11 +50,7 @@ static void BM_UnparseRealDerivationFile(benchmark::State & state, const std::st } // Register benchmarks for actual test derivation files if they exist -BENCHMARK_CAPTURE( - BM_ParseRealDerivationFile, hello, getEnvNonEmpty("_NIX_TEST_UNIT_DATA").value() + "/derivation/hello.drv"); -BENCHMARK_CAPTURE( - BM_ParseRealDerivationFile, firefox, getEnvNonEmpty("_NIX_TEST_UNIT_DATA").value() + "/derivation/firefox.drv"); -BENCHMARK_CAPTURE( - BM_UnparseRealDerivationFile, hello, getEnvNonEmpty("_NIX_TEST_UNIT_DATA").value() + "/derivation/hello.drv"); -BENCHMARK_CAPTURE( - BM_UnparseRealDerivationFile, firefox, getEnvNonEmpty("_NIX_TEST_UNIT_DATA").value() + "/derivation/firefox.drv"); +BENCHMARK_CAPTURE(BM_ParseRealDerivationFile, hello, (getUnitTestData() / "derivation/hello.drv").string()); +BENCHMARK_CAPTURE(BM_ParseRealDerivationFile, firefox, (getUnitTestData() / "derivation/firefox.drv").string()); +BENCHMARK_CAPTURE(BM_UnparseRealDerivationFile, hello, (getUnitTestData() / "derivation/hello.drv").string()); +BENCHMARK_CAPTURE(BM_UnparseRealDerivationFile, firefox, (getUnitTestData() / "derivation/firefox.drv").string()); diff --git a/src/libstore-tests/derivation/external-formats.cc b/src/libstore-tests/derivation/external-formats.cc index 056eeaa8a967..a9be99f996ed 100644 --- a/src/libstore-tests/derivation/external-formats.cc +++ b/src/libstore-tests/derivation/external-formats.cc @@ -23,17 +23,17 @@ TEST_F(DynDerivationTest, BadATerm_oldVersionDynDeps) FormatError); } -#define MAKE_OUTPUT_JSON_TEST_P(FIXTURE) \ - TEST_P(FIXTURE, from_json) \ - { \ - const auto & [name, expected] = GetParam(); \ - readJsonTest(Path{"output-"} + name, expected, mockXpSettings); \ - } \ - \ - TEST_P(FIXTURE, to_json) \ - { \ - const auto & [name, value] = GetParam(); \ - writeJsonTest("output-" + name, value); \ +#define MAKE_OUTPUT_JSON_TEST_P(FIXTURE) \ + TEST_P(FIXTURE, from_json) \ + { \ + const auto & [name, expected] = GetParam(); \ + readJsonTest(std::string{"output-"} + name, expected, mockXpSettings); \ + } \ + \ + TEST_P(FIXTURE, to_json) \ + { \ + const auto & [name, value] = GetParam(); \ + writeJsonTest(std::string{"output-"} + name, value); \ } struct DerivationOutputJsonTest : DerivationTest, diff --git a/src/libstore-tests/derivation/invariants.cc b/src/libstore-tests/derivation/invariants.cc index cacdca0cdc0e..115d5bc4bb1f 100644 --- a/src/libstore-tests/derivation/invariants.cc +++ b/src/libstore-tests/derivation/invariants.cc @@ -51,7 +51,7 @@ class FillInOutputPathsTest : public LibStoreTest, public JsonCharacterizationTe depDrv.fillInOutputPaths(*store); // Write the dependency to the store - return writeDerivation(*store, depDrv, NoRepair); + return store->writeDerivation(depDrv, NoRepair); } public: diff --git a/src/libstore-tests/dummy-store.cc b/src/libstore-tests/dummy-store.cc index 4a12dcf78c09..e3422fe585f8 100644 --- a/src/libstore-tests/dummy-store.cc +++ b/src/libstore-tests/dummy-store.cc @@ -73,7 +73,7 @@ TEST_P(DummyStoreJsonTest, from_json) using namespace nlohmann; /* Cannot use `readJsonTest` because need to dereference the stores for equality. */ - readTest(Path{name} + ".json", [&](const auto & encodedRaw) { + readTest(std::string{name} + ".json", [&](const auto & encodedRaw) { auto encoded = json::parse(encodedRaw); ref decoded = adl_serializer>::from_json(encoded); ASSERT_EQ(*decoded, *expected); diff --git a/src/libstore-tests/http-binary-cache-store.cc b/src/libstore-tests/http-binary-cache-store.cc index 4b3754a1fe40..8495db4b5abb 100644 --- a/src/libstore-tests/http-binary-cache-store.cc +++ b/src/libstore-tests/http-binary-cache-store.cc @@ -1,27 +1,37 @@ #include +#include #include "nix/store/http-binary-cache-store.hh" +#include "nix/store/tests/https-store.hh" +#include "nix/util/fs-sink.hh" namespace nix { +using Authority = ParsedURL::Authority; + TEST(HttpBinaryCacheStore, constructConfig) { - HttpBinaryCacheStoreConfig config{"http", "foo.bar.baz", {}}; + HttpBinaryCacheStoreConfig config{ + { + .scheme = "http", + .authority = Authority{.host = "foo.bar.baz"}, + }, + {}, + }; EXPECT_EQ(config.cacheUri.to_string(), "http://foo.bar.baz"); } TEST(HttpBinaryCacheStore, constructConfigNoTrailingSlash) { - HttpBinaryCacheStoreConfig config{"https", "foo.bar.baz/a/b/", {}}; - + HttpBinaryCacheStoreConfig config{parseURL("https://foo.bar.baz/a/b/"), {}}; EXPECT_EQ(config.cacheUri.to_string(), "https://foo.bar.baz/a/b"); } TEST(HttpBinaryCacheStore, constructConfigWithParams) { StoreConfig::Params params{{"compression", "xz"}}; - HttpBinaryCacheStoreConfig config{"https", "foo.bar.baz/a/b/", params}; + HttpBinaryCacheStoreConfig config{parseURL("https://foo.bar.baz/a/b/"), params}; EXPECT_EQ(config.cacheUri.to_string(), "https://foo.bar.baz/a/b"); EXPECT_EQ(config.getReference().params, params); } @@ -29,9 +39,79 @@ TEST(HttpBinaryCacheStore, constructConfigWithParams) TEST(HttpBinaryCacheStore, constructConfigWithParamsAndUrlWithParams) { StoreConfig::Params params{{"compression", "xz"}}; - HttpBinaryCacheStoreConfig config{"https", "foo.bar.baz/a/b?some-param=some-value", params}; + HttpBinaryCacheStoreConfig config{parseURL("https://foo.bar.baz/a/b?some-param=some-value"), params}; EXPECT_EQ(config.cacheUri.to_string(), "https://foo.bar.baz/a/b?some-param=some-value"); EXPECT_EQ(config.getReference().params, params); } +using testing::HttpsBinaryCacheStoreMtlsTest; +using testing::HttpsBinaryCacheStoreTest; + +using namespace std::string_view_literals; +using namespace std::string_literals; + +TEST_F(HttpsBinaryCacheStoreTest, queryPathInfo) +{ + auto store = openStore(makeConfig()); + StringSource dump{"test"sv}; + auto path = localCacheStore->addToStoreFromDump(dump, "test-name", FileSerialisationMethod::Flat); + EXPECT_NO_THROW(store->queryPathInfo(path)); +} + +TEST_F(HttpsBinaryCacheStoreMtlsTest, queryPathInfo) +{ + auto config = makeConfig(); + config->tlsCert = clientCert; + config->tlsKey = clientKey; + auto store = openStore(config); + StringSource dump{"test"sv}; + auto path = localCacheStore->addToStoreFromDump(dump, "test-name", FileSerialisationMethod::Flat); + EXPECT_NO_THROW(store->queryPathInfo(path)); +} + +TEST_F(HttpsBinaryCacheStoreMtlsTest, rejectsWithoutClientCert) +{ + testFileTransferSettings->tries = 1; + EXPECT_THROW(openStore(makeConfig()), Error); +} + +TEST_F(HttpsBinaryCacheStoreMtlsTest, rejectsWrongClientCert) +{ + auto wrongKey = tmpDir / "wrong.key"; + auto wrongCert = tmpDir / "wrong.crt"; + + // clang-format off + openssl({"ecparam", "-genkey", "-name", "prime256v1", "-out", wrongKey.string()}); + openssl({"req", "-new", "-x509", "-days", "1", "-key", wrongKey.string(), "-out", wrongCert.string(), "-subj", "/CN=WrongClient"}); + // clang-format on + + auto config = makeConfig(); + config->tlsCert = wrongCert; + config->tlsKey = wrongKey; + testFileTransferSettings->tries = 1; + EXPECT_THROW(openStore(config), Error); +} + +TEST_F(HttpsBinaryCacheStoreMtlsTest, doesNotSendCertOnRedirectToDifferentAuthority) +{ + StringSource dump{"test"sv}; + auto path = localCacheStore->addToStoreFromDump(dump, "test-name", FileSerialisationMethod::Flat); + + for (auto & entry : DirectoryIterator{cacheDir}) + if (entry.path().extension() == ".narinfo") { + auto content = readFile(entry.path()); + content = std::regex_replace(content, std::regex("URL: nar/"), fmt("URL: https://127.0.0.1:%d/nar/", port)); + writeFile(entry.path(), content); + } + + auto config = makeConfig(); + config->tlsCert = clientCert; + config->tlsKey = clientKey; + testFileTransferSettings->tries = 1; + auto store = openStore(config); + auto info = store->queryPathInfo(path); + NullSink null; + EXPECT_THROW(store->narFromPath(path, null), Error); +} + } // namespace nix diff --git a/src/libstore-tests/legacy-ssh-store.cc b/src/libstore-tests/legacy-ssh-store.cc index d60ecc424c54..35543cf40064 100644 --- a/src/libstore-tests/legacy-ssh-store.cc +++ b/src/libstore-tests/legacy-ssh-store.cc @@ -7,8 +7,7 @@ namespace nix { TEST(LegacySSHStore, constructConfig) { LegacySSHStoreConfig config( - "ssh", - "me@localhost:2222", + ParsedURL::Authority::parse("me@localhost:2222"), StoreConfig::Params{ { "remote-program", diff --git a/src/libstore-tests/local-binary-cache-store.cc b/src/libstore-tests/local-binary-cache-store.cc index 01f514e89aae..295976488b5f 100644 --- a/src/libstore-tests/local-binary-cache-store.cc +++ b/src/libstore-tests/local-binary-cache-store.cc @@ -6,8 +6,7 @@ namespace nix { TEST(LocalBinaryCacheStore, constructConfig) { - LocalBinaryCacheStoreConfig config{"local", "/foo/bar/baz", {}}; - + LocalBinaryCacheStoreConfig config{std::filesystem::path("/foo/bar/baz"), {}}; EXPECT_EQ(config.binaryCacheDir, "/foo/bar/baz"); } diff --git a/src/libstore-tests/local-overlay-store.cc b/src/libstore-tests/local-overlay-store.cc index 175e5d0f44e3..c207564255f3 100644 --- a/src/libstore-tests/local-overlay-store.cc +++ b/src/libstore-tests/local-overlay-store.cc @@ -7,7 +7,6 @@ namespace nix { TEST(LocalOverlayStore, constructConfig_rootQueryParam) { LocalOverlayStoreConfig config{ - "local-overlay", "", { { @@ -22,7 +21,7 @@ TEST(LocalOverlayStore, constructConfig_rootQueryParam) TEST(LocalOverlayStore, constructConfig_rootPath) { - LocalOverlayStoreConfig config{"local-overlay", "/foo/bar", {}}; + LocalOverlayStoreConfig config{"/foo/bar", {}}; EXPECT_EQ(config.rootDir.get(), std::optional{"/foo/bar"}); } diff --git a/src/libstore-tests/local-store.cc b/src/libstore-tests/local-store.cc index d008888974b8..554d42efc811 100644 --- a/src/libstore-tests/local-store.cc +++ b/src/libstore-tests/local-store.cc @@ -13,7 +13,6 @@ namespace nix { TEST(LocalStore, constructConfig_rootQueryParam) { LocalStoreConfig config{ - "local", "", { { @@ -28,14 +27,14 @@ TEST(LocalStore, constructConfig_rootQueryParam) TEST(LocalStore, constructConfig_rootPath) { - LocalStoreConfig config{"local", "/foo/bar", {}}; + LocalStoreConfig config{"/foo/bar", {}}; EXPECT_EQ(config.rootDir.get(), std::optional{"/foo/bar"}); } TEST(LocalStore, constructConfig_to_string) { - LocalStoreConfig config{"local", "", {}}; + LocalStoreConfig config{"", {}}; EXPECT_EQ(config.getReference().to_string(), "local"); } diff --git a/src/libstore-tests/machines.cc b/src/libstore-tests/machines.cc index e4186372de42..f0a334c1b613 100644 --- a/src/libstore-tests/machines.cc +++ b/src/libstore-tests/machines.cc @@ -13,10 +13,6 @@ using testing::Eq; using testing::Field; using testing::SizeIs; -namespace nix::fs { -using namespace std::filesystem; -} - using namespace nix; TEST(machines, getMachinesWithEmptyBuilders) @@ -31,7 +27,7 @@ TEST(machines, getMachinesUriOnly) ASSERT_THAT(actual, SizeIs(1)); EXPECT_THAT(actual[0], Field(&Machine::storeUri, Eq(StoreReference::parse("ssh://nix@scratchy.labs.cs.uu.nl")))); EXPECT_THAT(actual[0], Field(&Machine::systemTypes, ElementsAre("TEST_ARCH-TEST_OS"))); - EXPECT_THAT(actual[0], Field(&Machine::sshKey, SizeIs(0))); + EXPECT_THAT(actual[0], Field(&Machine::sshKey, Eq(std::filesystem::path{}))); EXPECT_THAT(actual[0], Field(&Machine::maxJobs, Eq(1))); EXPECT_THAT(actual[0], Field(&Machine::speedFactor, Eq(1))); EXPECT_THAT(actual[0], Field(&Machine::supportedFeatures, SizeIs(0))); @@ -53,7 +49,7 @@ TEST(machines, getMachinesDefaults) ASSERT_THAT(actual, SizeIs(1)); EXPECT_THAT(actual[0], Field(&Machine::storeUri, Eq(StoreReference::parse("ssh://nix@scratchy.labs.cs.uu.nl")))); EXPECT_THAT(actual[0], Field(&Machine::systemTypes, ElementsAre("TEST_ARCH-TEST_OS"))); - EXPECT_THAT(actual[0], Field(&Machine::sshKey, SizeIs(0))); + EXPECT_THAT(actual[0], Field(&Machine::sshKey, Eq(std::filesystem::path{}))); EXPECT_THAT(actual[0], Field(&Machine::maxJobs, Eq(1))); EXPECT_THAT(actual[0], Field(&Machine::speedFactor, Eq(1))); EXPECT_THAT(actual[0], Field(&Machine::supportedFeatures, SizeIs(0))); diff --git a/src/libstore-tests/main.cc b/src/libstore-tests/main.cc index ffe9816134f1..c45e3a7f384a 100644 --- a/src/libstore-tests/main.cc +++ b/src/libstore-tests/main.cc @@ -1,6 +1,7 @@ #include #include "nix/store/tests/test-main.hh" +#include "nix/store/tests/libstore-network.hh" using namespace nix; @@ -10,6 +11,7 @@ int main(int argc, char ** argv) if (res) return res; + nix::testing::setupNetworkTests(); ::testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); } diff --git a/src/libstore-tests/meson.build b/src/libstore-tests/meson.build index 58f624611a40..2d12b14d3cb5 100644 --- a/src/libstore-tests/meson.build +++ b/src/libstore-tests/meson.build @@ -85,6 +85,7 @@ sources = files( 'store-reference.cc', 'uds-remote-store.cc', 'worker-protocol.cc', + 'worker-substitution.cc', 'write-derivation.cc', ) @@ -123,6 +124,7 @@ if get_option('benchmarks') 'bench-main.cc', 'derivation-parser-bench.cc', 'ref-scan-bench.cc', + 'register-valid-paths-bench.cc', ) benchmark_exe = executable( diff --git a/src/libstore-tests/nar-info-disk-cache.cc b/src/libstore-tests/nar-info-disk-cache.cc index b925a4a1e04a..aebefc775675 100644 --- a/src/libstore-tests/nar-info-disk-cache.cc +++ b/src/libstore-tests/nar-info-disk-cache.cc @@ -2,6 +2,7 @@ #include #include +#include "nix/store/globals.hh" #include "nix/store/sqlite.hh" #include @@ -24,7 +25,8 @@ TEST(NarInfoDiskCacheImpl, create_and_read) SQLiteStmt getIds; { - auto cache = getTestNarInfoDiskCache(dbPath.string()); + auto cache = NarInfoDiskCache::getTest( + settings.getNarInfoDiskCacheSettings(), {.useWAL = settings.useSQLiteWAL}, dbPath); // Set up "background noise" and check that different caches receive different ids { @@ -48,7 +50,7 @@ TEST(NarInfoDiskCacheImpl, create_and_read) // We're going to pay special attention to the id field because we had a bug // that changed it. - db = SQLite(dbPath); + db = SQLite(dbPath, {.useWAL = settings.useSQLiteWAL}); getIds.create(db, "select id from BinaryCaches where url = 'http://foo'"); { @@ -73,7 +75,8 @@ TEST(NarInfoDiskCacheImpl, create_and_read) { // We can't clear the in-memory cache, so we use a new cache object. This is // more realistic anyway. - auto cache2 = getTestNarInfoDiskCache(dbPath.string()); + auto cache2 = NarInfoDiskCache::getTest( + settings.getNarInfoDiskCacheSettings(), {.useWAL = settings.useSQLiteWAL}, dbPath); { auto r = cache2->upToDateCacheExists("http://foo"); diff --git a/src/libstore-tests/nar-info.cc b/src/libstore-tests/nar-info.cc index 493ca2a8c37b..9b0f6018cdeb 100644 --- a/src/libstore-tests/nar-info.cc +++ b/src/libstore-tests/nar-info.cc @@ -15,7 +15,7 @@ class NarInfoTestV1 : public CharacterizationTest, public LibStoreTest { std::filesystem::path unitTestData = getUnitTestData() / "nar-info" / "json-1"; - std::filesystem::path goldenMaster(PathView testStem) const override + std::filesystem::path goldenMaster(std::string_view testStem) const override { return unitTestData / (testStem + ".json"); } @@ -25,7 +25,7 @@ class NarInfoTestV2 : public CharacterizationTest, public LibStoreTest { std::filesystem::path unitTestData = getUnitTestData() / "nar-info" / "json-2"; - std::filesystem::path goldenMaster(PathView testStem) const override + std::filesystem::path goldenMaster(std::string_view testStem) const override { return unitTestData / (testStem + ".json"); } @@ -59,7 +59,10 @@ static NarInfo makeNarInfo(const Store & store, bool includeImpureInfo) }; info.registrationTime = 23423; info.ultimate = true; - info.sigs = {"asdf", "qwer"}; + info.sigs = { + Signature{.keyName = "asdf", .sig = std::string(64, '\0')}, + Signature{.keyName = "qwer", .sig = std::string(64, '\0')}, + }; info.url = "nar/1w1fff338fvdw53sqgamddn1b2xgds473pv6y13gizdbqjv4i5p3.nar.xz"; info.compression = "xz"; diff --git a/src/libstore-tests/nix_api_store.cc b/src/libstore-tests/nix_api_store.cc index 0f6c795765d2..f35d6ddf36f3 100644 --- a/src/libstore-tests/nix_api_store.cc +++ b/src/libstore-tests/nix_api_store.cc @@ -8,6 +8,7 @@ #include "nix/store/tests/nix_api_store.hh" #include "nix/store/globals.hh" #include "nix/util/tests/string_callback.hh" +#include "nix/util/tests/test-data.hh" #include "nix/util/url.hh" #include "store-tests-config.hh" @@ -298,11 +299,11 @@ class NixApiStoreTestWithRealisedPath : public nix_api_store_test_base nix_api_store_test_base::SetUp(); nix::experimentalFeatureSettings.set("extra-experimental-features", "ca-derivations"); - nix::settings.substituters = {}; + nix::settings.getWorkerSettings().substituters = {}; store = open_local_store(); - std::filesystem::path unitTestData{getenv("_NIX_TEST_UNIT_DATA")}; + std::filesystem::path unitTestData = nix::getUnitTestData(); std::ifstream t{unitTestData / "derivation/ca/self-contained.json"}; std::stringstream buffer; buffer << t.rdbuf(); @@ -353,11 +354,11 @@ TEST_F(nix_api_store_test_base, build_from_json) { // FIXME get rid of these nix::experimentalFeatureSettings.set("extra-experimental-features", "ca-derivations"); - nix::settings.substituters = {}; + nix::settings.getWorkerSettings().substituters = {}; auto * store = open_local_store(); - std::filesystem::path unitTestData{getenv("_NIX_TEST_UNIT_DATA")}; + std::filesystem::path unitTestData = nix::getUnitTestData(); std::ifstream t{unitTestData / "derivation/ca/self-contained.json"}; std::stringstream buffer; @@ -400,11 +401,11 @@ TEST_F(nix_api_store_test_base, nix_store_realise_invalid_system) { // Test that nix_store_realise properly reports errors when the system is invalid nix::experimentalFeatureSettings.set("extra-experimental-features", "ca-derivations"); - nix::settings.substituters = {}; + nix::settings.getWorkerSettings().substituters = {}; auto * store = open_local_store(); - std::filesystem::path unitTestData{getenv("_NIX_TEST_UNIT_DATA")}; + std::filesystem::path unitTestData = nix::getUnitTestData(); std::ifstream t{unitTestData / "derivation/ca/self-contained.json"}; std::stringstream buffer; buffer << t.rdbuf(); @@ -445,11 +446,11 @@ TEST_F(nix_api_store_test_base, nix_store_realise_builder_fails) { // Test that nix_store_realise properly reports errors when the builder fails nix::experimentalFeatureSettings.set("extra-experimental-features", "ca-derivations"); - nix::settings.substituters = {}; + nix::settings.getWorkerSettings().substituters = {}; auto * store = open_local_store(); - std::filesystem::path unitTestData{getenv("_NIX_TEST_UNIT_DATA")}; + std::filesystem::path unitTestData = nix::getUnitTestData(); std::ifstream t{unitTestData / "derivation/ca/self-contained.json"}; std::stringstream buffer; buffer << t.rdbuf(); @@ -490,11 +491,11 @@ TEST_F(nix_api_store_test_base, nix_store_realise_builder_no_output) { // Test that nix_store_realise properly reports errors when builder succeeds but produces no output nix::experimentalFeatureSettings.set("extra-experimental-features", "ca-derivations"); - nix::settings.substituters = {}; + nix::settings.getWorkerSettings().substituters = {}; auto * store = open_local_store(); - std::filesystem::path unitTestData{getenv("_NIX_TEST_UNIT_DATA")}; + std::filesystem::path unitTestData = nix::getUnitTestData(); std::ifstream t{unitTestData / "derivation/ca/self-contained.json"}; std::stringstream buffer; buffer << t.rdbuf(); @@ -686,7 +687,7 @@ TEST_F(NixApiStoreTestWithRealisedPath, nix_store_realise_output_ordering) // This test uses a CA derivation with 10 outputs in randomized input order // to verify that the callback order is deterministic and alphabetical. nix::experimentalFeatureSettings.set("extra-experimental-features", "ca-derivations"); - nix::settings.substituters = {}; + nix::settings.getWorkerSettings().substituters = {}; auto * store = open_local_store(); @@ -870,7 +871,7 @@ TEST_F(NixApiStoreTestWithRealisedPath, nix_store_get_fs_closure_error_propagati */ static std::string load_json_from_test_data(const char * filename) { - std::filesystem::path unitTestData{getenv("_NIX_TEST_UNIT_DATA")}; + std::filesystem::path unitTestData = nix::getUnitTestData(); std::ifstream t{unitTestData / filename}; std::stringstream buffer; buffer << t.rdbuf(); @@ -958,7 +959,7 @@ TEST_F(nix_api_store_test, nix_derivation_clone) TEST_F(nix_api_store_test, nix_store_build_paths) { nix::experimentalFeatureSettings.set("extra-experimental-features", "ca-derivations"); - nix::settings.substituters = {}; + nix::settings.getWorkerSettings().substituters = {}; auto * store = open_local_store(); diff --git a/src/libstore-tests/package.nix b/src/libstore-tests/package.nix index ac547aca35e1..2bd0ab1a5db2 100644 --- a/src/libstore-tests/package.nix +++ b/src/libstore-tests/package.nix @@ -9,6 +9,7 @@ nix-store-c, nix-store-test-support, sqlite, + openssl, rapidcheck, gtest, @@ -75,7 +76,10 @@ mkMesonExecutable (finalAttrs: { runCommand "${finalAttrs.pname}-run" { meta.broken = !stdenv.hostPlatform.emulatorAvailable buildPackages; - buildInputs = [ writableTmpDirAsHomeHook ]; + nativeBuildInputs = [ + writableTmpDirAsHomeHook + openssl + ]; } ( '' diff --git a/src/libstore-tests/path-info.cc b/src/libstore-tests/path-info.cc index 6c0fd183bebf..97ad4b270ade 100644 --- a/src/libstore-tests/path-info.cc +++ b/src/libstore-tests/path-info.cc @@ -14,7 +14,7 @@ class PathInfoTestV1 : public CharacterizationTest, public LibStoreTest { std::filesystem::path unitTestData = getUnitTestData() / "path-info" / "json-1"; - std::filesystem::path goldenMaster(PathView testStem) const override + std::filesystem::path goldenMaster(std::string_view testStem) const override { return unitTestData / (testStem + ".json"); } @@ -24,7 +24,7 @@ class PathInfoTestV2 : public CharacterizationTest, public LibStoreTest { std::filesystem::path unitTestData = getUnitTestData() / "path-info" / "json-2"; - std::filesystem::path goldenMaster(PathView testStem) const override + std::filesystem::path goldenMaster(std::string_view testStem) const override { return unitTestData / (testStem + ".json"); } @@ -66,7 +66,10 @@ static ValidPathInfo makeFullKeyed(const Store & store, bool includeImpureInfo) }; info.registrationTime = 23423; info.ultimate = true; - info.sigs = {"asdf", "qwer"}; + info.sigs = { + Signature{.keyName = "asdf", .sig = std::string(64, '\0')}, + Signature{.keyName = "qwer", .sig = std::string(64, '\0')}, + }; } return info; } diff --git a/src/libstore-tests/realisation.cc b/src/libstore-tests/realisation.cc index d16049bc5b00..d2d7df80f59a 100644 --- a/src/libstore-tests/realisation.cc +++ b/src/libstore-tests/realisation.cc @@ -44,54 +44,47 @@ TEST_P(RealisationJsonTest, to_json) writeJsonTest(name, value); } +Realisation simple{ + { + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"}, + }, + { + .drvHash = Hash::parseExplicitFormatUnprefixed( + "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", + HashAlgorithm::SHA256, + HashFormat::Base16), + .outputName = "foo", + }, +}; + INSTANTIATE_TEST_SUITE_P( RealisationJSON, RealisationJsonTest, - ([] { - Realisation simple{ - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"}, - }, - { - .drvHash = Hash::parseExplicitFormatUnprefixed( - "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", - HashAlgorithm::SHA256, - HashFormat::Base16), - .outputName = "foo", - }, - }; - return ::testing::Values( - std::pair{ - "simple", - simple, - }, - std::pair{ - "with-signature", - [&] { - auto r = simple; - // FIXME actually sign properly - r.signatures = {"asdfasdfasdf"}; - return r; - }()}, - std::pair{ - "with-dependent-realisations", - [&] { - auto r = simple; - r.dependentRealisations = {{ - { - .drvHash = Hash::parseExplicitFormatUnprefixed( - "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", - HashAlgorithm::SHA256, - HashFormat::Base16), - .outputName = "foo", - }, - StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"}, - }}; - return r; - }(), - }); - } + ::testing::Values( + std::pair{ + "simple", + simple, + }, + std::pair{ + "with-signature", + [&] { + auto r = simple; + // FIXME actually sign properly + r.signatures = { + Signature{.keyName = "asdf", .sig = std::string(64, '\0')}, + }; + return r; + }(), + })); - ())); +/** + * We no longer have a notion of "dependent realisations", but we still + * want to parse old realisation files. So make this just be a read test + * (no write direction), accordingly. + */ +TEST_F(RealisationTest, dependent_realisations_from_json) +{ + readJsonTest("with-dependent-realisations", simple); +} } // namespace nix diff --git a/src/libstore-tests/register-valid-paths-bench.cc b/src/libstore-tests/register-valid-paths-bench.cc new file mode 100644 index 000000000000..1d795818369e --- /dev/null +++ b/src/libstore-tests/register-valid-paths-bench.cc @@ -0,0 +1,79 @@ +#include + +#include "nix/store/derivations.hh" +#include "nix/store/local-store.hh" +#include "nix/store/store-open.hh" +#include "nix/util/file-system.hh" +#include "nix/util/hash.hh" +#include "nix/util/tests/test-data.hh" + +#ifndef _WIN32 + +# include +# include + +using namespace nix; + +static void BM_RegisterValidPathsDerivations(benchmark::State & state) +{ + const int derivationCount = state.range(0); + + for (auto _ : state) { + state.PauseTiming(); + + auto tmpRoot = createTempDir(); + auto realStoreDir = tmpRoot / "nix/store"; + std::filesystem::create_directories(realStoreDir); + + std::shared_ptr store = openStore(fmt("local?root=%s", tmpRoot.string())); + auto localStore = std::dynamic_pointer_cast(store); + if (!localStore) + throw Error("expected local store"); + + ValidPathInfos infos; + for (int i = 0; i < derivationCount; ++i) { + std::string drvName = fmt("register-valid-paths-bench-%d", i); + auto drvPath = StorePath::random(drvName + ".drv"); + + Derivation drv; + drv.name = drvName; + drv.outputs.emplace("out", DerivationOutput{DerivationOutput::Deferred{}}); + drv.platform = "x86_64-linux"; + drv.builder = "foo"; + drv.env["out"] = ""; + drv.fillInOutputPaths(*localStore); + + auto drvContents = drv.unparse(*localStore, /*maskOutputs=*/false); + + /* Create an on-disk store object without registering it + in the SQLite DB. LocalFSStore::getFSAccessor(path, false) + allows reading store objects based on their filesystem + presence alone. */ + std::ofstream out(realStoreDir / std::string(drvPath.to_string()), std::ios::binary); + out.write(drvContents.data(), drvContents.size()); + if (!out) + throw SysError("writing derivation to store"); + + ValidPathInfo info{drvPath, UnkeyedValidPathInfo(*localStore, Hash::dummy)}; + info.narSize = drvContents.size(); + + infos.emplace(drvPath, std::move(info)); + } + + state.ResumeTiming(); + + localStore->registerValidPaths(infos); + + state.PauseTiming(); + localStore.reset(); + store.reset(); + std::filesystem::remove_all(tmpRoot); + state.ResumeTiming(); + } + + state.SetItemsProcessed(state.iterations() * derivationCount); +} + +BENCHMARK(BM_RegisterValidPathsDerivations)->Arg(10); + +#endif diff --git a/src/libstore-tests/s3-binary-cache-store.cc b/src/libstore-tests/s3-binary-cache-store.cc index 59090a589f06..9aa9b2dd1a3e 100644 --- a/src/libstore-tests/s3-binary-cache-store.cc +++ b/src/libstore-tests/s3-binary-cache-store.cc @@ -9,7 +9,7 @@ namespace nix { TEST(S3BinaryCacheStore, constructConfig) { - S3BinaryCacheStoreConfig config{"s3", "foobar", {}}; + S3BinaryCacheStoreConfig config{"foobar", {}}; // The bucket name is stored as the host part of the authority in cacheUri EXPECT_EQ( @@ -23,7 +23,7 @@ TEST(S3BinaryCacheStore, constructConfig) TEST(S3BinaryCacheStore, constructConfigWithRegion) { Store::Config::Params params{{"region", "eu-west-1"}}; - S3BinaryCacheStoreConfig config{"s3", "my-bucket", params}; + S3BinaryCacheStoreConfig config{"my-bucket", params}; EXPECT_EQ( config.cacheUri, @@ -37,7 +37,7 @@ TEST(S3BinaryCacheStore, constructConfigWithRegion) TEST(S3BinaryCacheStore, defaultSettings) { - S3BinaryCacheStoreConfig config{"s3", "test-bucket", {}}; + S3BinaryCacheStoreConfig config{"test-bucket", {}}; EXPECT_EQ( config.cacheUri, @@ -62,7 +62,7 @@ TEST(S3BinaryCacheStore, s3StoreConfigPreservesParameters) params["region"] = "eu-west-1"; params["endpoint"] = "custom.s3.com"; - S3BinaryCacheStoreConfig config("s3", "test-bucket", params); + S3BinaryCacheStoreConfig config("test-bucket", params); // The config should preserve S3-specific parameters EXPECT_EQ( @@ -99,7 +99,7 @@ TEST(S3BinaryCacheStore, parameterFiltering) params["want-mass-query"] = "true"; // Non-S3 store parameter params["priority"] = "10"; // Non-S3 store parameter - S3BinaryCacheStoreConfig config("s3", "test-bucket", params); + S3BinaryCacheStoreConfig config("test-bucket", params); // Only S3-specific params should be in cacheUri.query EXPECT_EQ( @@ -127,7 +127,7 @@ TEST(S3BinaryCacheStore, parameterFiltering) */ TEST(S3BinaryCacheStore, storageClassDefault) { - S3BinaryCacheStoreConfig config{"s3", "test-bucket", {}}; + S3BinaryCacheStoreConfig config{"test-bucket", {}}; EXPECT_EQ(config.storageClass.get(), std::nullopt); } @@ -136,7 +136,7 @@ TEST(S3BinaryCacheStore, storageClassConfiguration) StringMap params; params["storage-class"] = "GLACIER"; - S3BinaryCacheStoreConfig config("s3", "test-bucket", params); + S3BinaryCacheStoreConfig config("test-bucket", params); EXPECT_EQ(config.storageClass.get(), std::optional("GLACIER")); } diff --git a/src/libstore-tests/serve-protocol.cc b/src/libstore-tests/serve-protocol.cc index 258dbf049908..159abeb60827 100644 --- a/src/libstore-tests/serve-protocol.cc +++ b/src/libstore-tests/serve-protocol.cc @@ -25,7 +25,10 @@ struct ServeProtoTest : VersionedProtoTest * For serializers that don't care about the minimum version, we * used the oldest one: 2.5. */ - ServeProto::Version defaultVersion = 2 << 8 | 5; + ServeProto::Version defaultVersion = { + .major = 2, + .minor = 5, + }; }; VERSIONED_CHARACTERIZATION_TEST( @@ -109,7 +112,11 @@ VERSIONED_CHARACTERIZATION_TEST( Realisation{ { .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - .signatures = {"asdf", "qwer"}, + .signatures = + { + Signature{.keyName = "asdf", .sig = std::string(64, '\0')}, + Signature{.keyName = "qwer", .sig = std::string(64, '\0')}, + }, }, { .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), @@ -118,7 +125,7 @@ VERSIONED_CHARACTERIZATION_TEST( }, })) -VERSIONED_CHARACTERIZATION_TEST( +VERSIONED_READ_CHARACTERIZATION_TEST( ServeProtoTest, realisation_with_deps, "realisation-with-deps", @@ -127,16 +134,10 @@ VERSIONED_CHARACTERIZATION_TEST( Realisation{ { .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - .signatures = {"asdf", "qwer"}, - .dependentRealisations = + .signatures = { - { - DrvOutput{ - .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), - .outputName = "quux", - }, - StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - }, + Signature{.keyName = "asdf", .sig = std::string(64, '\0')}, + Signature{.keyName = "qwer", .sig = std::string(64, '\0')}, }, }, { @@ -146,66 +147,89 @@ VERSIONED_CHARACTERIZATION_TEST( }, })) -VERSIONED_CHARACTERIZATION_TEST(ServeProtoTest, buildResult_2_2, "build-result-2.2", 2 << 8 | 2, ({ - using namespace std::literals::chrono_literals; - std::tuple t{ - BuildResult{.inner{BuildResult::Failure{ - .status = BuildResult::Failure::OutputRejected, - .errorMsg = "no idea why", - }}}, - BuildResult{.inner{BuildResult::Failure{ - .status = BuildResult::Failure::NotDeterministic, - .errorMsg = "no idea why", - }}}, - BuildResult{.inner{BuildResult::Success{ - .status = BuildResult::Success::Built, - }}}, - }; - t; - })) - -VERSIONED_CHARACTERIZATION_TEST(ServeProtoTest, buildResult_2_3, "build-result-2.3", 2 << 8 | 3, ({ - using namespace std::literals::chrono_literals; - std::tuple t{ - BuildResult{.inner{BuildResult::Failure{ - .status = BuildResult::Failure::OutputRejected, - .errorMsg = "no idea why", - }}}, - BuildResult{ - .inner{BuildResult::Failure{ - .status = BuildResult::Failure::NotDeterministic, - .errorMsg = "no idea why", - .isNonDeterministic = true, - }}, - .timesBuilt = 3, - .startTime = 30, - .stopTime = 50, - }, - BuildResult{ - .inner{BuildResult::Success{ - .status = BuildResult::Success::Built, - }}, - .startTime = 30, - .stopTime = 50, - }, - }; - t; - })) - VERSIONED_CHARACTERIZATION_TEST( - ServeProtoTest, buildResult_2_6, "build-result-2.6", 2 << 8 | 6, ({ + ServeProtoTest, + buildResult_2_2, + "build-result-2.2", + (ServeProto::Version{ + .major = 2, + .minor = 2, + }), + ({ using namespace std::literals::chrono_literals; std::tuple t{ - BuildResult{.inner{BuildResult::Failure{ + BuildResult{.inner{BuildResult::Failure{{ .status = BuildResult::Failure::OutputRejected, - .errorMsg = "no idea why", + .msg = HintFmt("no idea why"), + }}}}, + BuildResult{.inner{BuildResult::Failure{{ + .status = BuildResult::Failure::NotDeterministic, + .msg = HintFmt("no idea why"), + }}}}, + BuildResult{.inner{BuildResult::Success{ + .status = BuildResult::Success::Built, }}}, + }; + t; + })) + +VERSIONED_CHARACTERIZATION_TEST( + ServeProtoTest, + buildResult_2_3, + "build-result-2.3", + (ServeProto::Version{ + .major = 2, + .minor = 3, + }), + ({ + using namespace std::literals::chrono_literals; + std::tuple t{ + BuildResult{.inner{BuildResult::Failure{{ + .status = BuildResult::Failure::OutputRejected, + .msg = HintFmt("no idea why"), + }}}}, BuildResult{ - .inner{BuildResult::Failure{ + .inner{BuildResult::Failure{{ .status = BuildResult::Failure::NotDeterministic, - .errorMsg = "no idea why", + .msg = HintFmt("no idea why"), .isNonDeterministic = true, + }}}, + .timesBuilt = 3, + .startTime = 30, + .stopTime = 50, + }, + BuildResult{ + .inner{BuildResult::Success{ + .status = BuildResult::Success::Built, }}, + .startTime = 30, + .stopTime = 50, + }, + }; + t; + })) + +VERSIONED_CHARACTERIZATION_TEST( + ServeProtoTest, + buildResult_2_6, + "build-result-2.6", + (ServeProto::Version{ + .major = 2, + .minor = 6, + }), + ({ + using namespace std::literals::chrono_literals; + std::tuple t{ + BuildResult{.inner{BuildResult::Failure{{ + .status = BuildResult::Failure::OutputRejected, + .msg = HintFmt("no idea why"), + }}}}, + BuildResult{ + .inner{BuildResult::Failure{{ + .status = BuildResult::Failure::NotDeterministic, + .msg = HintFmt("no idea why"), + .isNonDeterministic = true, + }}}, .timesBuilt = 3, .startTime = 30, .stopTime = 50, @@ -262,7 +286,10 @@ VERSIONED_CHARACTERIZATION_TEST( ServeProtoTest, unkeyedValidPathInfo_2_3, "unkeyed-valid-path-info-2.3", - 2 << 8 | 3, + (ServeProto::Version{ + .major = 2, + .minor = 3, + }), (std::tuple{ ({ UnkeyedValidPathInfo info{std::string{defaultStoreDir}, Hash::dummy}; @@ -288,7 +315,10 @@ VERSIONED_CHARACTERIZATION_TEST( ServeProtoTest, unkeyedValidPathInfo_2_4, "unkeyed-valid-path-info-2.4", - 2 << 8 | 4, + (ServeProto::Version{ + .major = 2, + .minor = 4, + }), (std::tuple{ ({ UnkeyedValidPathInfo info{ @@ -331,8 +361,8 @@ VERSIONED_CHARACTERIZATION_TEST( info.narSize = 34878; info.sigs = { - "fake-sig-1", - "fake-sig-2", + Signature{.keyName = "fake-sig-1", .sig = std::string(64, '\0')}, + Signature{.keyName = "fake-sig-2", .sig = std::string(64, '\0')}, }, static_cast(std::move(info)); }), @@ -342,7 +372,10 @@ VERSIONED_CHARACTERIZATION_TEST_NO_JSON( ServeProtoTest, build_options_2_1, "build-options-2.1", - 2 << 8 | 1, + (ServeProto::Version{ + .major = 2, + .minor = 1, + }), (ServeProto::BuildOptions{ .maxSilentTime = 5, .buildTimeout = 6, @@ -352,7 +385,10 @@ VERSIONED_CHARACTERIZATION_TEST_NO_JSON( ServeProtoTest, build_options_2_2, "build-options-2.2", - 2 << 8 | 2, + (ServeProto::Version{ + .major = 2, + .minor = 2, + }), (ServeProto::BuildOptions{ .maxSilentTime = 5, .buildTimeout = 6, @@ -363,7 +399,10 @@ VERSIONED_CHARACTERIZATION_TEST_NO_JSON( ServeProtoTest, build_options_2_3, "build-options-2.3", - 2 << 8 | 3, + (ServeProto::Version{ + .major = 2, + .minor = 3, + }), (ServeProto::BuildOptions{ .maxSilentTime = 5, .buildTimeout = 6, @@ -376,7 +415,10 @@ VERSIONED_CHARACTERIZATION_TEST_NO_JSON( ServeProtoTest, build_options_2_7, "build-options-2.7", - 2 << 8 | 7, + (ServeProto::Version{ + .major = 2, + .minor = 7, + }), (ServeProto::BuildOptions{ .maxSilentTime = 5, .buildTimeout = 6, diff --git a/src/libstore-tests/ssh-store.cc b/src/libstore-tests/ssh-store.cc index a156da52b711..2d4548d6ae21 100644 --- a/src/libstore-tests/ssh-store.cc +++ b/src/libstore-tests/ssh-store.cc @@ -9,8 +9,7 @@ namespace nix { TEST(SSHStore, constructConfig) { SSHStoreConfig config{ - "ssh-ng", - "me@localhost:2222", + ParsedURL::Authority::parse("me@localhost:2222"), StoreConfig::Params{ { "remote-program", @@ -35,8 +34,7 @@ TEST(SSHStore, constructConfig) TEST(MountedSSHStore, constructConfig) { MountedSSHStoreConfig config{ - "mounted-ssh", - "localhost", + {.host = "localhost"}, StoreConfig::Params{ { "remote-program", diff --git a/src/libstore-tests/store-reference.cc b/src/libstore-tests/store-reference.cc index 272d6732a855..e83fc97234fb 100644 --- a/src/libstore-tests/store-reference.cc +++ b/src/libstore-tests/store-reference.cc @@ -15,7 +15,7 @@ class StoreReferenceTest : public CharacterizationTest, public LibStoreTest { std::filesystem::path unitTestData = getUnitTestData() / "store-reference"; - std::filesystem::path goldenMaster(PathView testStem) const override + std::filesystem::path goldenMaster(std::string_view testStem) const override { return unitTestData / (testStem + ".txt"); } @@ -85,6 +85,20 @@ static StoreReference localExample_2{ }, }; +#ifdef _WIN32 +static StoreReference localExample_windows{ + .variant = + StoreReference::Specified{ + .scheme = "local", + .authority = "/C:/foo/bar/baz", + }, + .params = + { + {"trusted", "true"}, + }, +}; +#endif + static StoreReference localExample_3{ .variant = StoreReference::Specified{ @@ -108,7 +122,11 @@ URI_TEST_READ(local_3_no_percent, localExample_3) URI_TEST_READ(local_shorthand_1, localExample_1) -URI_TEST_READ(local_shorthand_2, localExample_2) +#ifndef _WIN32 +URI_TEST_READ(local_shorthand_path_unix, localExample_2) +#else +URI_TEST_READ(local_shorthand_path_windows, localExample_windows) +#endif URI_TEST( local_shorthand_3, diff --git a/src/libstore-tests/uds-remote-store.cc b/src/libstore-tests/uds-remote-store.cc index 415dfc4ac946..88af22dbb22d 100644 --- a/src/libstore-tests/uds-remote-store.cc +++ b/src/libstore-tests/uds-remote-store.cc @@ -6,26 +6,21 @@ namespace nix { TEST(UDSRemoteStore, constructConfig) { - UDSRemoteStoreConfig config{"unix", "/tmp/socket", {}}; + UDSRemoteStoreConfig config{"/tmp/socket", {}}; EXPECT_EQ(config.path, "/tmp/socket"); } -TEST(UDSRemoteStore, constructConfigWrongScheme) -{ - EXPECT_THROW(UDSRemoteStoreConfig("http", "/tmp/socket", {}), UsageError); -} - TEST(UDSRemoteStore, constructConfig_to_string) { - UDSRemoteStoreConfig config{"unix", "", {}}; + UDSRemoteStoreConfig config{"", {}}; EXPECT_EQ(config.getReference().to_string(), "daemon"); } TEST(UDSRemoteStore, constructConfigWithParams) { StoreConfig::Params params{{"max-connections", "1"}}; - UDSRemoteStoreConfig config{"unix", "/tmp/socket", params}; + UDSRemoteStoreConfig config{"/tmp/socket", params}; auto storeReference = config.getReference(); EXPECT_EQ(storeReference.to_string(), "unix:///tmp/socket?max-connections=1"); EXPECT_EQ(storeReference.render(/*withParams=*/false), "unix:///tmp/socket"); @@ -35,7 +30,7 @@ TEST(UDSRemoteStore, constructConfigWithParams) TEST(UDSRemoteStore, constructConfigWithParamsNoPath) { StoreConfig::Params params{{"max-connections", "1"}}; - UDSRemoteStoreConfig config{"unix", "", params}; + UDSRemoteStoreConfig config{"", params}; auto storeReference = config.getReference(); EXPECT_EQ(storeReference.to_string(), "daemon?max-connections=1"); EXPECT_EQ(storeReference.render(/*withParams=*/false), "daemon"); diff --git a/src/libstore-tests/worker-protocol.cc b/src/libstore-tests/worker-protocol.cc index 7416d732301f..aac5822d01f8 100644 --- a/src/libstore-tests/worker-protocol.cc +++ b/src/libstore-tests/worker-protocol.cc @@ -15,6 +15,80 @@ namespace nix { +TEST(WorkerProtoVersionNumber, ordering) +{ + using Number = WorkerProto::Version::Number; + EXPECT_LT((Number{1, 10}), (Number{1, 20})); + EXPECT_GT((Number{1, 30}), (Number{1, 20})); + EXPECT_EQ((Number{1, 10}), (Number{1, 10})); + EXPECT_LT((Number{0, 255}), (Number{1, 0})); +} + +TEST(WorkerProtoVersion, partialOrderingSameFeatures) +{ + using V = WorkerProto::Version; + V v1{.number = {1, 20}, .features = {"a", "b"}}; + V v2{.number = {1, 30}, .features = {"a", "b"}}; + + EXPECT_TRUE(v1 < v2); + EXPECT_TRUE(v2 > v1); + EXPECT_TRUE(v1 <= v2); + EXPECT_TRUE(v2 >= v1); + EXPECT_FALSE(v1 == v2); +} + +TEST(WorkerProtoVersion, partialOrderingSubsetFeatures) +{ + using V = WorkerProto::Version; + V fewer{.number = {1, 30}, .features = {"a"}}; + V more{.number = {1, 30}, .features = {"a", "b"}}; + + // fewer <= more: JUST the features are a subset + EXPECT_TRUE(fewer < more); + EXPECT_TRUE(fewer <= more); + EXPECT_FALSE(fewer > more); + EXPECT_TRUE(fewer != more); +} + +TEST(WorkerProtoVersion, partialOrderingUnordered) +{ + using V = WorkerProto::Version; + // Same number but incomparable features + V v1{.number = {1, 20}, .features = {"a", "c"}}; + V v2{.number = {1, 20}, .features = {"a", "b"}}; + + EXPECT_FALSE(v1 < v2); + EXPECT_FALSE(v1 > v2); + EXPECT_FALSE(v1 <= v2); + EXPECT_FALSE(v1 >= v2); + EXPECT_FALSE(v1 == v2); + EXPECT_TRUE(v1 != v2); +} + +TEST(WorkerProtoVersion, partialOrderingHigherNumberFewerFeatures) +{ + using V = WorkerProto::Version; + // Higher number but fewer features — unordered + V v1{.number = {1, 30}, .features = {"a"}}; + V v2{.number = {1, 20}, .features = {"a", "b"}}; + + EXPECT_FALSE(v1 < v2); + EXPECT_FALSE(v1 > v2); + EXPECT_FALSE(v1 == v2); +} + +TEST(WorkerProtoVersion, partialOrderingEmptyFeatures) +{ + using V = WorkerProto::Version; + V empty{.number = {1, 20}, .features = {}}; + V some{.number = {1, 30}, .features = {"a"}}; + + // empty features is a subset of everything + EXPECT_TRUE(empty < some); + EXPECT_TRUE(empty <= some); + EXPECT_TRUE(empty != some); +} + const char workerProtoDir[] = "worker-protocol"; static constexpr std::string_view defaultStoreDir = "/nix/store"; @@ -25,7 +99,13 @@ struct WorkerProtoTest : VersionedProtoTest * For serializers that don't care about the minimum version, we * used the oldest one: 1.10. */ - WorkerProto::Version defaultVersion = 1 << 8 | 10; + WorkerProto::Version defaultVersion = { + .number = + { + .major = 1, + .minor = 10, + }, + }; }; VERSIONED_CHARACTERIZATION_TEST( @@ -79,7 +159,13 @@ VERSIONED_CHARACTERIZATION_TEST( WorkerProtoTest, derivedPath_1_29, "derived-path-1.29", - 1 << 8 | 29, + (WorkerProto::Version{ + .number = + { + .major = 1, + .minor = 29, + }, + }), (std::tuple{ DerivedPath::Opaque{ .path = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, @@ -104,7 +190,13 @@ VERSIONED_CHARACTERIZATION_TEST( WorkerProtoTest, derivedPath_1_30, "derived-path-1.30", - 1 << 8 | 30, + (WorkerProto::Version{ + .number = + { + .major = 1, + .minor = 30, + }, + }), (std::tuple{ DerivedPath::Opaque{ .path = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, @@ -162,7 +254,11 @@ VERSIONED_CHARACTERIZATION_TEST( Realisation{ { .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - .signatures = {"asdf", "qwer"}, + .signatures = + { + Signature{.keyName = "asdf", .sig = std::string(64, '\0')}, + Signature{.keyName = "qwer", .sig = std::string(64, '\0')}, + }, }, { .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), @@ -171,7 +267,7 @@ VERSIONED_CHARACTERIZATION_TEST( }, })) -VERSIONED_CHARACTERIZATION_TEST( +VERSIONED_READ_CHARACTERIZATION_TEST( WorkerProtoTest, realisation_with_deps, "realisation-with-deps", @@ -180,16 +276,10 @@ VERSIONED_CHARACTERIZATION_TEST( Realisation{ { .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - .signatures = {"asdf", "qwer"}, - .dependentRealisations = + .signatures = { - { - DrvOutput{ - .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), - .outputName = "quux", - }, - StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - }, + Signature{.keyName = "asdf", .sig = std::string(64, '\0')}, + Signature{.keyName = "qwer", .sig = std::string(64, '\0')}, }, }, { @@ -199,36 +289,57 @@ VERSIONED_CHARACTERIZATION_TEST( }, })) -VERSIONED_CHARACTERIZATION_TEST(WorkerProtoTest, buildResult_1_27, "build-result-1.27", 1 << 8 | 27, ({ - using namespace std::literals::chrono_literals; - std::tuple t{ - BuildResult{.inner{BuildResult::Failure{ - .status = BuildResult::Failure::OutputRejected, - .errorMsg = "no idea why", - }}}, - BuildResult{.inner{BuildResult::Failure{ - .status = BuildResult::Failure::NotDeterministic, - .errorMsg = "no idea why", - }}}, - BuildResult{.inner{BuildResult::Success{ - .status = BuildResult::Success::Built, - }}}, - }; - t; - })) - VERSIONED_CHARACTERIZATION_TEST( - WorkerProtoTest, buildResult_1_28, "build-result-1.28", 1 << 8 | 28, ({ + WorkerProtoTest, + buildResult_1_27, + "build-result-1.27", + (WorkerProto::Version{ + .number = + { + .major = 1, + .minor = 27, + }, + }), + ({ using namespace std::literals::chrono_literals; std::tuple t{ - BuildResult{.inner{BuildResult::Failure{ + BuildResult{.inner{BuildResult::Failure{{ .status = BuildResult::Failure::OutputRejected, - .errorMsg = "no idea why", - }}}, - BuildResult{.inner{BuildResult::Failure{ + .msg = HintFmt("no idea why"), + }}}}, + BuildResult{.inner{BuildResult::Failure{{ .status = BuildResult::Failure::NotDeterministic, - .errorMsg = "no idea why", + .msg = HintFmt("no idea why"), + }}}}, + BuildResult{.inner{BuildResult::Success{ + .status = BuildResult::Success::Built, }}}, + }; + t; + })) + +VERSIONED_CHARACTERIZATION_TEST( + WorkerProtoTest, + buildResult_1_28, + "build-result-1.28", + (WorkerProto::Version{ + .number = + { + .major = 1, + .minor = 28, + }, + }), + ({ + using namespace std::literals::chrono_literals; + std::tuple t{ + BuildResult{.inner{BuildResult::Failure{{ + .status = BuildResult::Failure::OutputRejected, + .msg = HintFmt("no idea why"), + }}}}, + BuildResult{.inner{BuildResult::Failure{{ + .status = BuildResult::Failure::NotDeterministic, + .msg = HintFmt("no idea why"), + }}}}, BuildResult{.inner{BuildResult::Success{ .status = BuildResult::Success::Built, .builtOutputs = @@ -264,19 +375,29 @@ VERSIONED_CHARACTERIZATION_TEST( })) VERSIONED_CHARACTERIZATION_TEST( - WorkerProtoTest, buildResult_1_29, "build-result-1.29", 1 << 8 | 29, ({ + WorkerProtoTest, + buildResult_1_29, + "build-result-1.29", + (WorkerProto::Version{ + .number = + { + .major = 1, + .minor = 29, + }, + }), + ({ using namespace std::literals::chrono_literals; std::tuple t{ - BuildResult{.inner{BuildResult::Failure{ + BuildResult{.inner{BuildResult::Failure{{ .status = BuildResult::Failure::OutputRejected, - .errorMsg = "no idea why", - }}}, + .msg = HintFmt("no idea why"), + }}}}, BuildResult{ - .inner{BuildResult::Failure{ + .inner{BuildResult::Failure{{ .status = BuildResult::Failure::NotDeterministic, - .errorMsg = "no idea why", + .msg = HintFmt("no idea why"), .isNonDeterministic = true, - }}, + }}}, .timesBuilt = 3, .startTime = 30, .stopTime = 50, @@ -323,19 +444,29 @@ VERSIONED_CHARACTERIZATION_TEST( })) VERSIONED_CHARACTERIZATION_TEST( - WorkerProtoTest, buildResult_1_37, "build-result-1.37", 1 << 8 | 37, ({ + WorkerProtoTest, + buildResult_1_37, + "build-result-1.37", + (WorkerProto::Version{ + .number = + { + .major = 1, + .minor = 37, + }, + }), + ({ using namespace std::literals::chrono_literals; std::tuple t{ - BuildResult{.inner{BuildResult::Failure{ + BuildResult{.inner{BuildResult::Failure{{ .status = BuildResult::Failure::OutputRejected, - .errorMsg = "no idea why", - }}}, + .msg = HintFmt("no idea why"), + }}}}, BuildResult{ - .inner{BuildResult::Failure{ + .inner{BuildResult::Failure{{ .status = BuildResult::Failure::NotDeterministic, - .errorMsg = "no idea why", + .msg = HintFmt("no idea why"), .isNonDeterministic = true, - }}, + }}}, .timesBuilt = 3, .startTime = 30, .stopTime = 50, @@ -383,48 +514,65 @@ VERSIONED_CHARACTERIZATION_TEST( t; })) -VERSIONED_CHARACTERIZATION_TEST(WorkerProtoTest, keyedBuildResult_1_29, "keyed-build-result-1.29", 1 << 8 | 29, ({ - using namespace std::literals::chrono_literals; - std::tuple t{ - KeyedBuildResult{ - {.inner{BuildResult::Failure{ - .status = KeyedBuildResult::Failure::OutputRejected, - .errorMsg = "no idea why", - }}}, - /* .path = */ - DerivedPath::Opaque{ - StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-xxx"}, - }, - }, - KeyedBuildResult{ - { - .inner{BuildResult::Failure{ - .status = KeyedBuildResult::Failure::NotDeterministic, - .errorMsg = "no idea why", - .isNonDeterministic = true, - }}, - .timesBuilt = 3, - .startTime = 30, - .stopTime = 50, - }, - /* .path = */ - DerivedPath::Built{ - .drvPath = makeConstantStorePathRef( - StorePath{ - "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", - }), - .outputs = OutputsSpec::Names{"out"}, - }, - }, - }; - t; - })) +VERSIONED_CHARACTERIZATION_TEST( + WorkerProtoTest, + keyedBuildResult_1_29, + "keyed-build-result-1.29", + (WorkerProto::Version{ + .number = + { + .major = 1, + .minor = 29, + }, + }), + ({ + using namespace std::literals::chrono_literals; + std::tuple t{ + KeyedBuildResult{ + BuildResult{.inner{KeyedBuildResult::Failure{{ + .status = KeyedBuildResult::Failure::OutputRejected, + .msg = HintFmt("no idea why"), + }}}}, + /* .path = */ + DerivedPath::Opaque{ + StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-xxx"}, + }, + }, + KeyedBuildResult{ + BuildResult{ + .inner{KeyedBuildResult::Failure{{ + .status = KeyedBuildResult::Failure::NotDeterministic, + .msg = HintFmt("no idea why"), + .isNonDeterministic = true, + }}}, + .timesBuilt = 3, + .startTime = 30, + .stopTime = 50, + }, + /* .path = */ + DerivedPath::Built{ + .drvPath = makeConstantStorePathRef( + StorePath{ + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + }), + .outputs = OutputsSpec::Names{"out"}, + }, + }, + }; + t; + })) VERSIONED_CHARACTERIZATION_TEST( WorkerProtoTest, unkeyedValidPathInfo_1_15, "unkeyed-valid-path-info-1.15", - 1 << 8 | 15, + (WorkerProto::Version{ + .number = + { + .major = 1, + .minor = 15, + }, + }), (std::tuple{ ({ UnkeyedValidPathInfo info{ @@ -458,7 +606,13 @@ VERSIONED_CHARACTERIZATION_TEST( WorkerProtoTest, validPathInfo_1_15, "valid-path-info-1.15", - 1 << 8 | 15, + (WorkerProto::Version{ + .number = + { + .major = 1, + .minor = 15, + }, + }), (std::tuple{ ({ ValidPathInfo info{ @@ -507,7 +661,13 @@ VERSIONED_CHARACTERIZATION_TEST( WorkerProtoTest, validPathInfo_1_16, "valid-path-info-1.16", - 1 << 8 | 16, + (WorkerProto::Version{ + .number = + { + .major = 1, + .minor = 16, + }, + }), (std::tuple{ ({ ValidPathInfo info{ @@ -551,8 +711,8 @@ VERSIONED_CHARACTERIZATION_TEST( info.narSize = 34878; info.sigs = { - "fake-sig-1", - "fake-sig-2", + Signature{.keyName = "fake-sig-1", .sig = std::string(64, '\0')}, + Signature{.keyName = "fake-sig-2", .sig = std::string(64, '\0')}, }, info; }), @@ -662,7 +822,13 @@ VERSIONED_CHARACTERIZATION_TEST_NO_JSON( WorkerProtoTest, clientHandshakeInfo_1_30, "client-handshake-info_1_30", - 1 << 8 | 30, + (WorkerProto::Version{ + .number = + { + .major = 1, + .minor = 30, + }, + }), (std::tuple{ {}, })) @@ -671,7 +837,13 @@ VERSIONED_CHARACTERIZATION_TEST_NO_JSON( WorkerProtoTest, clientHandshakeInfo_1_33, "client-handshake-info_1_33", - 1 << 8 | 33, + (WorkerProto::Version{ + .number = + { + .major = 1, + .minor = 33, + }, + }), (std::tuple{ { .daemonNixVersion = std::optional{"foo"}, @@ -685,7 +857,13 @@ VERSIONED_CHARACTERIZATION_TEST_NO_JSON( WorkerProtoTest, clientHandshakeInfo_1_35, "client-handshake-info_1_35", - 1 << 8 | 35, + (WorkerProto::Version{ + .number = + { + .major = 1, + .minor = 35, + }, + }), (std::tuple{ { .daemonNixVersion = std::optional{"foo"}, @@ -712,13 +890,13 @@ TEST_F(WorkerProtoTest, handshake_log) FdSink out{toServer.writeSide.get()}; FdSource in0{toClient.readSide.get()}; TeeSource in{in0, toClientLog}; - clientResult = std::get<0>(WorkerProto::BasicClientConnection::handshake(out, in, defaultVersion, {})); + clientResult = WorkerProto::BasicClientConnection::handshake(out, in, defaultVersion); }); { FdSink out{toClient.writeSide.get()}; FdSource in{toServer.readSide.get()}; - WorkerProto::BasicServerConnection::handshake(out, in, defaultVersion, {}); + WorkerProto::BasicServerConnection::handshake(out, in, defaultVersion); }; thread.join(); @@ -733,23 +911,43 @@ TEST_F(WorkerProtoTest, handshake_features) toClient.create(); toServer.create(); - std::tuple clientResult; + WorkerProto::Version clientResult; auto clientThread = std::thread([&]() { FdSink out{toServer.writeSide.get()}; FdSource in{toClient.readSide.get()}; - clientResult = WorkerProto::BasicClientConnection::handshake(out, in, 123, {"bar", "aap", "mies", "xyzzy"}); + clientResult = WorkerProto::BasicClientConnection::handshake( + out, + in, + WorkerProto::Version{ + .number = {.major = 1, .minor = 123}, + .features = {"bar", "aap", "mies", "xyzzy"}, + }); }); FdSink out{toClient.writeSide.get()}; FdSource in{toServer.readSide.get()}; - auto daemonResult = WorkerProto::BasicServerConnection::handshake(out, in, 456, {"foo", "bar", "xyzzy"}); + auto daemonResult = WorkerProto::BasicServerConnection::handshake( + out, + in, + WorkerProto::Version{ + .number = {.major = 1, .minor = 200}, + .features = {"foo", "bar", "xyzzy"}, + }); clientThread.join(); EXPECT_EQ(clientResult, daemonResult); - EXPECT_EQ(std::get<0>(clientResult), 123u); - EXPECT_EQ(std::get<1>(clientResult), WorkerProto::FeatureSet({"bar", "xyzzy"})); + EXPECT_EQ( + clientResult, + (WorkerProto::Version{ + .number = + { + .major = 1, + .minor = 123, + }, + .features = {"bar", "xyzzy"}, + })); } /// Has to be a `BufferedSink` for handshake. @@ -764,8 +962,7 @@ TEST_F(WorkerProtoTest, handshake_client_replay) NullBufferedSink nullSink; StringSource in{toClientLog}; - auto clientResult = - std::get<0>(WorkerProto::BasicClientConnection::handshake(nullSink, in, defaultVersion, {})); + auto clientResult = WorkerProto::BasicClientConnection::handshake(nullSink, in, defaultVersion); EXPECT_EQ(clientResult, defaultVersion); }); @@ -779,11 +976,10 @@ TEST_F(WorkerProtoTest, handshake_client_truncated_replay_throws) auto substring = toClientLog.substr(0, len); StringSource in{substring}; if (len < 8) { - EXPECT_THROW( - WorkerProto::BasicClientConnection::handshake(nullSink, in, defaultVersion, {}), EndOfFile); + EXPECT_THROW(WorkerProto::BasicClientConnection::handshake(nullSink, in, defaultVersion), EndOfFile); } else { // Not sure why cannot keep on checking for `EndOfFile`. - EXPECT_THROW(WorkerProto::BasicClientConnection::handshake(nullSink, in, defaultVersion, {}), Error); + EXPECT_THROW(WorkerProto::BasicClientConnection::handshake(nullSink, in, defaultVersion), Error); } } }); @@ -803,14 +999,13 @@ TEST_F(WorkerProtoTest, handshake_client_corrupted_throws) if (idx < 4 || idx == 9) { // magic bytes don't match - EXPECT_THROW(WorkerProto::BasicClientConnection::handshake(nullSink, in, defaultVersion, {}), Error); + EXPECT_THROW(WorkerProto::BasicClientConnection::handshake(nullSink, in, defaultVersion), Error); } else if (idx < 8 || idx >= 12) { // Number out of bounds EXPECT_THROW( - WorkerProto::BasicClientConnection::handshake(nullSink, in, defaultVersion, {}), - SerialisationError); + WorkerProto::BasicClientConnection::handshake(nullSink, in, defaultVersion), SerialisationError); } else { - auto ver = std::get<0>(WorkerProto::BasicClientConnection::handshake(nullSink, in, defaultVersion, {})); + auto ver = WorkerProto::BasicClientConnection::handshake(nullSink, in, defaultVersion); // `std::min` of this and the other version saves us EXPECT_EQ(ver, defaultVersion); } diff --git a/src/libstore-tests/worker-substitution.cc b/src/libstore-tests/worker-substitution.cc new file mode 100644 index 000000000000..3a049b7e7216 --- /dev/null +++ b/src/libstore-tests/worker-substitution.cc @@ -0,0 +1,450 @@ +#include +#include + +#include "nix/store/build/worker.hh" +#include "nix/store/derivations.hh" +#include "nix/store/dummy-store-impl.hh" +#include "nix/store/globals.hh" +#include "nix/util/memory-source-accessor.hh" + +#include "nix/store/tests/libstore.hh" +#include "nix/util/tests/json-characterization.hh" + +namespace nix { + +class WorkerSubstitutionTest : public LibStoreTest, public JsonCharacterizationTest> +{ + std::filesystem::path unitTestData = getUnitTestData() / "worker-substitution"; + +protected: + ref dummyStore; + ref substituter; + + WorkerSubstitutionTest() + : LibStoreTest([] { + auto config = make_ref(DummyStoreConfig::Params{}); + config->readOnly = false; + return config->openDummyStore(); + }()) + , dummyStore(store.dynamic_pointer_cast()) + , substituter([] { + auto config = make_ref(DummyStoreConfig::Params{}); + config->readOnly = false; + config->isTrusted = true; + return config->openDummyStore(); + }()) + { + } + +public: + std::filesystem::path goldenMaster(std::string_view testStem) const override + { + return unitTestData / testStem; + } + + static void SetUpTestSuite() + { + initLibStore(false); + } +}; + +TEST_F(WorkerSubstitutionTest, singleStoreObject) +{ + // Add a store path to the substituter + auto pathInSubstituter = substituter->addToStore( + "hello", + SourcePath{ + [] { + auto sc = make_ref(); + sc->root = MemorySourceAccessor::File{MemorySourceAccessor::File::Regular{ + .executable = false, + .contents = "Hello, world!", + }}; + return sc; + }(), + }, + ContentAddressMethod::Raw::NixArchive, + HashAlgorithm::SHA256); + + // Snapshot the substituter (has one store object) + checkpointJson("single/substituter", substituter); + + // Snapshot the destination store before (should be empty) + checkpointJson("../dummy-store/empty", dummyStore); + + // The path should not exist in the destination store yet + ASSERT_FALSE(dummyStore->isValidPath(pathInSubstituter)); + + // Create a worker with our custom substituter + Worker worker{*dummyStore, *dummyStore}; + + // Override the substituters to use our dummy store substituter + ref substituerAsStore = substituter; + worker.getSubstituters = [substituerAsStore]() -> std::list> { return {substituerAsStore}; }; + + // Create a substitution goal for the path + auto goal = worker.makePathSubstitutionGoal(pathInSubstituter); + + // Run the worker with -j0 semantics (no local builds, only substitution) + // The worker.run() takes a set of goals + Goals goals; + goals.insert(upcast_goal(goal)); + worker.run(goals); + + // Snapshot the destination store after (should match the substituter) + checkpointJson("single/substituter", dummyStore); + + // The path should now exist in the destination store + ASSERT_TRUE(dummyStore->isValidPath(pathInSubstituter)); + + // Verify the goal succeeded + ASSERT_EQ(upcast_goal(goal)->exitCode, Goal::ecSuccess); +} + +TEST_F(WorkerSubstitutionTest, singleRootStoreObjectWithSingleDepStoreObject) +{ + // First, add a dependency store path to the substituter + auto dependencyPath = substituter->addToStore( + "dependency", + SourcePath{ + [] { + auto sc = make_ref(); + sc->root = MemorySourceAccessor::File{MemorySourceAccessor::File::Regular{ + .executable = false, + .contents = "I am a dependency", + }}; + return sc; + }(), + }, + ContentAddressMethod::Raw::NixArchive, + HashAlgorithm::SHA256); + + // Now add a store path that references the dependency + auto mainPath = substituter->addToStore( + "main", + SourcePath{ + [&] { + auto sc = make_ref(); + // Include a reference to the dependency path in the contents + sc->root = MemorySourceAccessor::File{MemorySourceAccessor::File::Regular{ + .executable = false, + .contents = "I depend on " + substituter->printStorePath(dependencyPath), + }}; + return sc; + }(), + }, + ContentAddressMethod::Raw::NixArchive, + HashAlgorithm::SHA256, + StorePathSet{dependencyPath}); + + // Snapshot the substituter (has two store objects) + checkpointJson("with-dep/substituter", substituter); + + // Snapshot the destination store before (should be empty) + checkpointJson("../dummy-store/empty", dummyStore); + + // Neither path should exist in the destination store yet + ASSERT_FALSE(dummyStore->isValidPath(dependencyPath)); + ASSERT_FALSE(dummyStore->isValidPath(mainPath)); + + // Create a worker with our custom substituter + Worker worker{*dummyStore, *dummyStore}; + + // Override the substituters to use our dummy store substituter + ref substituterAsStore = substituter; + worker.getSubstituters = [substituterAsStore]() -> std::list> { return {substituterAsStore}; }; + + // Create a substitution goal for the main path only + // The worker should automatically substitute the dependency as well + auto goal = worker.makePathSubstitutionGoal(mainPath); + + // Run the worker + Goals goals; + goals.insert(upcast_goal(goal)); + worker.run(goals); + + // Snapshot the destination store after (should match the substituter) + checkpointJson("with-dep/substituter", dummyStore); + + // Both paths should now exist in the destination store + ASSERT_TRUE(dummyStore->isValidPath(dependencyPath)); + ASSERT_TRUE(dummyStore->isValidPath(mainPath)); + + // Verify the goal succeeded + ASSERT_EQ(upcast_goal(goal)->exitCode, Goal::ecSuccess); +} + +TEST_F(WorkerSubstitutionTest, floatingDerivationOutput) +{ + // Enable CA derivations experimental feature + experimentalFeatureSettings.set("extra-experimental-features", "ca-derivations"); + + // Create a CA floating output derivation + Derivation drv; + drv.name = "test-ca-drv"; + drv.outputs = { + { + "out", + DerivationOutput{DerivationOutput::CAFloating{ + .method = ContentAddressMethod::Raw::NixArchive, + .hashAlgo = HashAlgorithm::SHA256, + }}, + }, + }; + + // Write the derivation to the destination store + auto drvPath = dummyStore->writeDerivation(drv); + + // Snapshot the destination store before + checkpointJson("ca-drv/store-before", dummyStore); + + // Compute the hash modulo of the derivation + // For CA floating derivations, the kind is Deferred since outputs aren't known until build + auto hashModulo = hashDerivationModulo(*dummyStore, drv, true); + ASSERT_EQ(hashModulo.kind, DrvHash::Kind::Deferred); + auto drvHash = hashModulo.hashes.at("out"); + + // Create the output store object + auto outputPath = substituter->addToStore( + "test-ca-drv-out", + SourcePath{ + [] { + auto sc = make_ref(); + sc->root = MemorySourceAccessor::File{MemorySourceAccessor::File::Regular{ + .executable = false, + .contents = "I am the output of a CA derivation", + }}; + return sc; + }(), + }, + ContentAddressMethod::Raw::NixArchive, + HashAlgorithm::SHA256); + + // Add the realisation (build trace) to the substituter + substituter->buildTrace.insert_or_assign( + drvHash, + std::map{ + { + "out", + UnkeyedRealisation{ + .outPath = outputPath, + }, + }, + }); + + // Snapshot the substituter + checkpointJson("ca-drv/substituter", substituter); + + // The realisation should not exist in the destination store yet + DrvOutput drvOutput{drvHash, "out"}; + ASSERT_FALSE(dummyStore->queryRealisation(drvOutput)); + + // Create a worker with our custom substituter + Worker worker{*dummyStore, *dummyStore}; + + // Override the substituters to use our dummy store substituter + ref substituterAsStore = substituter; + worker.getSubstituters = [substituterAsStore]() -> std::list> { return {substituterAsStore}; }; + + // Create a derivation goal for the CA derivation output + // The worker should substitute the output rather than building + auto goal = worker.makeDerivationGoal(drvPath, drv, "out", bmNormal, true); + + // Run the worker + Goals goals; + goals.insert(upcast_goal(goal)); + worker.run(goals); + + // Snapshot the destination store after + checkpointJson("ca-drv/store-after", dummyStore); + + // The output path should now exist in the destination store + ASSERT_TRUE(dummyStore->isValidPath(outputPath)); + + // The realisation should now exist in the destination store + auto realisation = dummyStore->queryRealisation(drvOutput); + ASSERT_TRUE(realisation); + ASSERT_EQ(realisation->outPath, outputPath); + + // Verify the goal succeeded + ASSERT_EQ(upcast_goal(goal)->exitCode, Goal::ecSuccess); + + // Disable CA derivations experimental feature + experimentalFeatureSettings.set("extra-experimental-features", ""); +} + +/** + * Test for issue #11928: substituting a CA derivation output should not + * require fetching the output of an input derivation when that output + * is not referenced. + */ +TEST_F(WorkerSubstitutionTest, floatingDerivationOutputWithDepDrv) +{ + // Enable CA derivations experimental feature + experimentalFeatureSettings.set("extra-experimental-features", "ca-derivations"); + + // Create the dependency CA floating derivation + Derivation depDrv; + depDrv.name = "dep-drv"; + depDrv.outputs = { + { + "out", + DerivationOutput{DerivationOutput::CAFloating{ + .method = ContentAddressMethod::Raw::NixArchive, + .hashAlgo = HashAlgorithm::SHA256, + }}, + }, + }; + + // Write the dependency derivation to the destination store + auto depDrvPath = dummyStore->writeDerivation(depDrv); + + // Compute the hash modulo for the dependency derivation + auto depHashModulo = hashDerivationModulo(*dummyStore, depDrv, true); + ASSERT_EQ(depHashModulo.kind, DrvHash::Kind::Deferred); + auto depDrvHash = depHashModulo.hashes.at("out"); + + // Create the output store object for the dependency in the substituter + auto depOutputPath = substituter->addToStore( + "dep-drv-out", + SourcePath{ + [] { + auto sc = make_ref(); + sc->root = MemorySourceAccessor::File{MemorySourceAccessor::File::Regular{ + .executable = false, + .contents = "I am the dependency output", + }}; + return sc; + }(), + }, + ContentAddressMethod::Raw::NixArchive, + HashAlgorithm::SHA256); + + // Add the realisation for the dependency to the substituter + substituter->buildTrace.insert_or_assign( + depDrvHash, + std::map{ + { + "out", + UnkeyedRealisation{ + .outPath = depOutputPath, + }, + }, + }); + + // Create the root CA floating derivation that depends on depDrv + Derivation rootDrv; + rootDrv.name = "root-drv"; + rootDrv.outputs = { + { + "out", + DerivationOutput{DerivationOutput::CAFloating{ + .method = ContentAddressMethod::Raw::NixArchive, + .hashAlgo = HashAlgorithm::SHA256, + }}, + }, + }; + // Add the dependency derivation as an input + rootDrv.inputDrvs = {.map = {{depDrvPath, {.value = {"out"}}}}}; + + // Write the root derivation to the destination store + auto rootDrvPath = dummyStore->writeDerivation(rootDrv); + + // Snapshot the destination store before + checkpointJson("issue-11928/store-before", dummyStore); + + // Compute the hash modulo for the root derivation + auto rootHashModulo = hashDerivationModulo(*dummyStore, rootDrv, true); + ASSERT_EQ(rootHashModulo.kind, DrvHash::Kind::Deferred); + auto rootDrvHash = rootHashModulo.hashes.at("out"); + + // Create the output store object for the root derivation + // Note: it does NOT reference the dependency's output + auto rootOutputPath = substituter->addToStore( + "root-drv-out", + SourcePath{ + [] { + auto sc = make_ref(); + sc->root = MemorySourceAccessor::File{MemorySourceAccessor::File::Regular{ + .executable = false, + .contents = + "I am the root output. " + "I don't reference anything because the other derivation's output is just needed at build time.", + }}; + return sc; + }(), + }, + ContentAddressMethod::Raw::NixArchive, + HashAlgorithm::SHA256); + + // The DrvOutputs for both derivations + DrvOutput depDrvOutput{depDrvHash, "out"}; + DrvOutput rootDrvOutput{rootDrvHash, "out"}; + + // Add the realisation for the root derivation to the substituter + substituter->buildTrace.insert_or_assign( + rootDrvHash, + std::map{ + { + "out", + UnkeyedRealisation{ + .outPath = rootOutputPath, + }, + }, + }); + + // Snapshot the substituter + // Note: it has realisations for both drvs, but only the root's output store object + checkpointJson("issue-11928/substituter", substituter); + + // The realisations should not exist in the destination store yet + ASSERT_FALSE(dummyStore->queryRealisation(depDrvOutput)); + ASSERT_FALSE(dummyStore->queryRealisation(rootDrvOutput)); + + // Create a worker with our custom substituter + Worker worker{*dummyStore, *dummyStore}; + + // Override the substituters to use our dummy store substituter + ref substituterAsStore = substituter; + worker.getSubstituters = [substituterAsStore]() -> std::list> { return {substituterAsStore}; }; + + // Create a derivation goal for the root derivation output + // The worker should substitute the output rather than building + auto goal = worker.makeDerivationGoal(rootDrvPath, rootDrv, "out", bmNormal, false); + + // Run the worker + Goals goals; + goals.insert(upcast_goal(goal)); + worker.run(goals); + + // Snapshot the destination store after + checkpointJson("issue-11928/store-after", dummyStore); + + // The root output path should now exist in the destination store + ASSERT_TRUE(dummyStore->isValidPath(rootOutputPath)); + + // The root realisation should now exist in the destination store + auto rootRealisation = dummyStore->queryRealisation(rootDrvOutput); + ASSERT_TRUE(rootRealisation); + ASSERT_EQ(rootRealisation->outPath, rootOutputPath); + + // #11928: The dependency's REALISATION should be fetched, because + // it is needed to resolve the underlying derivation. Currently the + // realisation is not fetched (bug). Once fixed: Change + // depRealisation ASSERT_FALSE to ASSERT_TRUE and uncomment the + // ASSERT_EQ + auto depRealisation = dummyStore->queryRealisation(depDrvOutput); + ASSERT_FALSE(depRealisation); + // ASSERT_EQ(depRealisation->outPath, depOutputPath); + + // The dependency's OUTPUT is correctly not fetched (not referenced by root output) + ASSERT_FALSE(dummyStore->isValidPath(depOutputPath)); + + // Verify the goal succeeded + ASSERT_EQ(upcast_goal(goal)->exitCode, Goal::ecSuccess); + + // Disable CA derivations experimental feature + experimentalFeatureSettings.set("extra-experimental-features", ""); +} + +} // namespace nix diff --git a/src/libstore-tests/write-derivation.cc b/src/libstore-tests/write-derivation.cc index c320f92faf31..c68753823d19 100644 --- a/src/libstore-tests/write-derivation.cc +++ b/src/libstore-tests/write-derivation.cc @@ -44,12 +44,12 @@ TEST_F(WriteDerivationTest, addToStoreFromDumpCalledOnce) { auto drv = makeSimpleDrv(); - auto path1 = writeDerivation(*store, drv, NoRepair); + auto path1 = store->writeDerivation(drv, NoRepair); config->readOnly = true; - auto path2 = writeDerivation(*store, drv, NoRepair); + auto path2 = computeStorePath(*store, drv); EXPECT_EQ(path1, path2); EXPECT_THAT( - [&] { writeDerivation(*store, drv, Repair); }, + [&] { store->writeDerivation(drv, Repair); }, ::testing::ThrowsMessage( testing::HasSubstrIgnoreANSIMatcher("operation 'writeDerivation' is not supported by store 'dummy://'"))); } diff --git a/src/libstore/active-builds.cc b/src/libstore/active-builds.cc index 838f188d8912..d165b6d7d4eb 100644 --- a/src/libstore/active-builds.cc +++ b/src/libstore/active-builds.cc @@ -102,13 +102,16 @@ ActiveBuild adl_serializer::from_json(const json & j) auto type = j.at("type").get(); if (type != "build") throw Error("invalid active build JSON: expected type 'build' but got '%s'", type); + std::optional cgroup; + if (!j.at("cgroup").is_null()) + cgroup = j.at("cgroup").get(); return ActiveBuild{ .nixPid = j.at("nixPid").get(), .clientPid = j.at("clientPid").get>(), .clientUid = j.at("clientUid").get>(), .mainPid = j.at("mainPid").get(), .mainUser = j.at("mainUser").get(), - .cgroup = j.at("cgroup").get>(), + .cgroup = std::move(cgroup), .startTime = (time_t) j.at("startTime").get(), .derivation = StorePath{getString(j.at("derivation"))}, }; @@ -123,7 +126,7 @@ void adl_serializer::to_json(json & j, const ActiveBuild & build) {"clientUid", build.clientUid}, {"mainPid", build.mainPid}, {"mainUser", build.mainUser}, - {"cgroup", build.cgroup}, + {"cgroup", build.cgroup ? nlohmann::json(*build.cgroup) : nlohmann::json(nullptr)}, {"startTime", (double) build.startTime}, {"derivation", build.derivation.to_string()}, }; diff --git a/src/libstore/async-path-writer.cc b/src/libstore/async-path-writer.cc index dce38b526478..ede52a146aaf 100644 --- a/src/libstore/async-path-writer.cc +++ b/src/libstore/async-path-writer.cc @@ -76,7 +76,6 @@ struct AsyncPathWriterImpl : AsyncPathWriter std::string name, StorePathSet references, RepairFlag repair, - bool readOnly, std::shared_ptr provenance) override { auto hash = hashString(HashAlgorithm::SHA256, contents); @@ -88,23 +87,21 @@ struct AsyncPathWriterImpl : AsyncPathWriter .references = references, }); - if (!readOnly) { - auto state(state_.lock()); - std::promise promise; - state->futures.insert_or_assign(storePath, promise.get_future()); - state->items.push_back( - Item{ - .storePath = storePath, - .contents = std::move(contents), - .name = std::move(name), - .hash = hash, - .references = std::move(references), - .repair = repair, - .provenance = provenance, - .promise = std::move(promise), - }); - wakeupCV.notify_all(); - } + auto state(state_.lock()); + std::promise promise; + state->futures.insert_or_assign(storePath, promise.get_future()); + state->items.push_back( + Item{ + .storePath = storePath, + .contents = std::move(contents), + .name = std::move(name), + .hash = hash, + .references = std::move(references), + .repair = repair, + .provenance = provenance, + .promise = std::move(promise), + }); + wakeupCV.notify_all(); return storePath; } diff --git a/src/libstore/aws-creds.cc b/src/libstore/aws-creds.cc index 9b0ddefdca85..b471962897bb 100644 --- a/src/libstore/aws-creds.cc +++ b/src/libstore/aws-creds.cc @@ -10,7 +10,7 @@ # include # include -// C library headers for SSO provider support +// C library headers for SSO, STS WebIdentity, and ECS credential providers # include // C library headers for custom logging @@ -28,7 +28,7 @@ namespace nix { AwsAuthError::AwsAuthError(int errorCode) - : Error("AWS authentication error: '%s' (%d)", aws_error_str(errorCode), errorCode) + : CloneableError("AWS authentication error: '%s' (%d)", aws_error_str(errorCode), errorCode) , errorCode(errorCode) { } @@ -170,6 +170,53 @@ static std::shared_ptr createSSOProvider( return createWrappedProvider(aws_credentials_provider_new_sso(allocator, &options), allocator); } +/** + * Create an STS WebIdentity credentials provider using the C library directly. + * This reads AWS_WEB_IDENTITY_TOKEN_FILE, AWS_ROLE_ARN, AWS_ROLE_SESSION_NAME, + * and AWS_REGION from the environment (falling back to the profile config). + * Used by EKS IRSA, GitHub Actions OIDC, and other sts:AssumeRoleWithWebIdentity flows. + * Returns nullptr if the required parameters can't be resolved. + */ +static std::shared_ptr createSTSWebIdentityProvider( + const std::string & profileName, + Aws::Crt::Io::ClientBootstrap * bootstrap, + Aws::Crt::Io::TlsContext * tlsContext, + Aws::Crt::Allocator * allocator = Aws::Crt::ApiAllocator()) +{ + aws_credentials_provider_sts_web_identity_options options; + AWS_ZERO_STRUCT(options); + + options.bootstrap = bootstrap->GetUnderlyingHandle(); + options.tls_ctx = tlsContext ? tlsContext->GetUnderlyingHandle() : nullptr; + if (!profileName.empty()) { + options.profile_name_override = aws_byte_cursor_from_c_str(profileName.c_str()); + } + + return createWrappedProvider(aws_credentials_provider_new_sts_web_identity(allocator, &options), allocator); +} + +/** + * Create an ECS container credentials provider using the C library directly. + * This reads AWS_CONTAINER_CREDENTIALS_RELATIVE_URI or + * AWS_CONTAINER_CREDENTIALS_FULL_URI (plus the optional + * AWS_CONTAINER_AUTHORIZATION_TOKEN / _TOKEN_FILE) from the environment. + * Used by ECS tasks and EKS Pod Identity. + * Returns nullptr if neither URI env var is set. + */ +static std::shared_ptr createECSProvider( + Aws::Crt::Io::ClientBootstrap * bootstrap, + Aws::Crt::Io::TlsContext * tlsContext, + Aws::Crt::Allocator * allocator = Aws::Crt::ApiAllocator()) +{ + aws_credentials_provider_ecs_environment_options options; + AWS_ZERO_STRUCT(options); + + options.bootstrap = bootstrap->GetUnderlyingHandle(); + options.tls_ctx = tlsContext ? tlsContext->GetUnderlyingHandle() : nullptr; + + return createWrappedProvider(aws_credentials_provider_new_ecs_from_environment(allocator, &options), allocator); +} + static AwsCredentials getCredentialsFromProvider(std::shared_ptr provider) { if (!provider || !provider->IsValid()) { @@ -223,13 +270,14 @@ class AwsCredentialProviderImpl : public AwsCredentialProvider // This ensures AWS logs respect Nix's verbosity settings and are formatted consistently. initialiseAwsLogger(); - // Create a shared TLS context for SSO (required for HTTPS connections) + // Create a shared TLS context for SSO, STS WebIdentity, and ECS providers (required for HTTPS) auto allocator = Aws::Crt::ApiAllocator(); auto tlsCtxOptions = Aws::Crt::Io::TlsContextOptions::InitDefaultClient(allocator); tlsContext = std::make_shared(tlsCtxOptions, Aws::Crt::Io::TlsMode::CLIENT, allocator); if (!tlsContext || !*tlsContext) { - warn("failed to create TLS context for AWS SSO; SSO authentication will be unavailable"); + warn( + "failed to create TLS context for AWS credential providers; SSO, STS WebIdentity, and ECS container authentication will be unavailable"); tlsContext = nullptr; } @@ -273,19 +321,20 @@ AwsCredentialProviderImpl::createProviderForProfile(const std::string & profile) debug("[pid=%d] creating new AWS credential provider for profile '%s'", getpid(), profileDisplayName); - // Build a custom credential chain: Environment → SSO → Profile → IMDS + // Build a custom credential chain: Environment → SSO → Profile → STS WebIdentity → ECS → IMDS // This works for both default and named profiles, ensuring consistent behavior // including SSO support and proper TLS context for STS-based role assumption. Aws::Crt::Auth::CredentialsProviderChainConfig chainConfig; auto allocator = Aws::Crt::ApiAllocator(); - auto addProviderToChain = [&](std::string_view name, auto createProvider) { + auto addProviderToChain = [&](std::string_view name, auto createProvider) -> bool { if (auto provider = createProvider()) { chainConfig.Providers.push_back(provider); debug("Added AWS %s Credential Provider to chain for profile '%s'", name, profileDisplayName); - } else { - debug("Skipped AWS %s Credential Provider for profile '%s'", name, profileDisplayName); + return true; } + debug("Skipped AWS %s Credential Provider for profile '%s'", name, profileDisplayName); + return false; }; // 1. Environment variables (highest priority) @@ -311,12 +360,37 @@ AwsCredentialProviderImpl::createProviderForProfile(const std::string & profile) return Aws::Crt::Auth::CredentialsProvider::CreateCredentialsProviderProfile(profileConfig, allocator); }); - // 4. IMDS provider (for EC2 instances, lowest priority) - addProviderToChain("IMDS", [&]() { - Aws::Crt::Auth::CredentialsProviderImdsConfig imdsConfig; - imdsConfig.Bootstrap = bootstrap; - return Aws::Crt::Auth::CredentialsProvider::CreateCredentialsProviderImds(imdsConfig, allocator); - }); + // 4. STS WebIdentity (AWS_WEB_IDENTITY_TOKEN_FILE + AWS_ROLE_ARN — EKS IRSA, GitHub Actions OIDC) + // 5. ECS container metadata (AWS_CONTAINER_CREDENTIALS_RELATIVE_URI — ECS tasks, EKS Pod Identity) + // ECS and IMDS are mutually exclusive per both the aws-c-auth default chain and the + // pre-2.33 aws-sdk-cpp DefaultAWSCredentialsProviderChain: when container credential + // env vars are set, IMDS is skipped so a transient ECS endpoint failure can't silently + // fall through to the (typically broader) EC2 instance profile. + bool ecsAdded = false; + if (tlsContext) { + addProviderToChain("STS WebIdentity", [&]() { + return createSTSWebIdentityProvider(profile, bootstrap, tlsContext.get(), allocator); + }); + ecsAdded = + addProviderToChain("ECS", [&]() { return createECSProvider(bootstrap, tlsContext.get(), allocator); }); + } else { + debug( + "Skipped AWS STS WebIdentity and ECS Credential Providers for profile '%s': TLS context unavailable", + profileDisplayName); + } + + // 6. IMDS provider (for EC2 instances, lowest priority) — only if ECS didn't claim the slot + if (!ecsAdded) { + addProviderToChain("IMDS", [&]() { + Aws::Crt::Auth::CredentialsProviderImdsConfig imdsConfig; + imdsConfig.Bootstrap = bootstrap; + return Aws::Crt::Auth::CredentialsProvider::CreateCredentialsProviderImds(imdsConfig, allocator); + }); + } else { + debug( + "Skipped AWS IMDS Credential Provider for profile '%s': ECS provider is active (mutually exclusive)", + profileDisplayName); + } return Aws::Crt::Auth::CredentialsProvider::CreateCredentialsProviderChain(chainConfig, allocator); } diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 51ef2fc7faad..f95d8a866194 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -27,12 +27,12 @@ namespace nix { BinaryCacheStore::BinaryCacheStore(Config & config) : config{config} { - if (config.secretKeyFile != "") - signers.push_back(std::make_unique(SecretKey{readFile(config.secretKeyFile)})); + if (!config.secretKeyFile.get().empty()) + signers.push_back(std::make_unique(SecretKey{readFile(config.secretKeyFile.get())})); if (config.secretKeyFiles != "") { std::stringstream ss(config.secretKeyFiles); - Path keyPath; + std::string keyPath; while (std::getline(ss, keyPath, ',')) { signers.push_back(std::make_unique(SecretKey{readFile(keyPath)})); } @@ -137,7 +137,7 @@ void BinaryCacheStore::writeNarInfo(ref narInfo) } ref BinaryCacheStore::addToStoreCommon( - Source & narSource, RepairFlag repair, CheckSigsFlag checkSigs, std::function mkInfo) + Source & narSource, RepairFlag repair, CheckSigsFlag checkSigs, fun mkInfo) { auto fdTemp = createAnonymousTempFile(); @@ -147,7 +147,7 @@ ref BinaryCacheStore::addToStoreCommon( write the compressed NAR to disk), into a HashSink (to get the NAR hash), and into a NarAccessor (to get the NAR listing). */ HashSink fileHashSink{HashAlgorithm::SHA256}; - std::shared_ptr narAccessor; + std::shared_ptr narAccessor; HashSink narHashSink{HashAlgorithm::SHA256}; { FdSink fileSink(fdTemp.get()); @@ -156,7 +156,7 @@ ref BinaryCacheStore::addToStoreCommon( config.compression, teeSinkCompressed, config.parallelCompression, config.compressionLevel); TeeSink teeSinkUncompressed{*compressionSink, narHashSink}; TeeSource teeSource{narSource, teeSinkUncompressed}; - narAccessor = makeNarAccessor(teeSource); + narAccessor = makeNarAccessor(parseNarListing(teeSource)); compressionSink->finish(); fileSink.flush(); } @@ -165,18 +165,18 @@ ref BinaryCacheStore::addToStoreCommon( auto info = mkInfo(narHashSink.finish()); auto narInfo = make_ref(info); - narInfo->compression = config.compression; + narInfo->compression = config.compression.to_string(); // FIXME: Make NarInfo use CompressionAlgo auto [fileHash, fileSize] = fileHashSink.finish(); narInfo->fileHash = fileHash; narInfo->fileSize = fileSize; narInfo->url = "nar/" + narInfo->fileHash->to_string(HashFormat::Nix32, false) + ".nar" - + (config.compression == "xz" ? ".xz" - : config.compression == "bzip2" ? ".bz2" - : config.compression == "zstd" ? ".zst" - : config.compression == "lzip" ? ".lzip" - : config.compression == "lz4" ? ".lz4" - : config.compression == "br" ? ".br" - : ""); + + (config.compression == CompressionAlgo::xz ? ".xz" + : config.compression == CompressionAlgo::bzip2 ? ".bz2" + : config.compression == CompressionAlgo::zstd ? ".zst" + : config.compression == CompressionAlgo::lzip ? ".lzip" + : config.compression == CompressionAlgo::lz4 ? ".lz4" + : config.compression == CompressionAlgo::brotli ? ".br" + : ""); auto duration = std::chrono::duration_cast(now2 - now1).count(); printMsg( @@ -205,7 +205,7 @@ ref BinaryCacheStore::addToStoreCommon( if (config.writeNARListing) { nlohmann::json j = { {"version", 1}, - {"root", listNarDeep(*narAccessor, CanonPath::root)}, + {"root", narAccessor->getListing()}, }; upsertFile(std::string(info.path.hashPart()) + ".ls", j.dump(), "application/json"); @@ -572,7 +572,7 @@ std::shared_ptr BinaryCacheStore::getFSAccessor(const StorePath return getRemoteFSAccessor(requireValidPath)->accessObject(storePath); } -void BinaryCacheStore::addSignatures(const StorePath & storePath, const StringSet & sigs) +void BinaryCacheStore::addSignatures(const StorePath & storePath, const std::set & sigs) { /* Note: this is inherently racy since there is no locking on binary caches. In particular, with S3 this unreliable, even diff --git a/src/libstore/build-result.cc b/src/libstore/build-result.cc index 19080e0f19ac..9b854f18cbad 100644 --- a/src/libstore/build-result.cc +++ b/src/libstore/build-result.cc @@ -6,15 +6,65 @@ namespace nix { +void ExitStatusFlags::updateFromStatus(BuildResult::Failure::Status status) +{ +// Allow selecting a subset of enum values +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wswitch-enum" + switch (status) { + case BuildResult::Failure::TimedOut: + timedOut = true; + break; + case BuildResult::Failure::HashMismatch: + hashMismatch = true; + break; + case BuildResult::Failure::NotDeterministic: + checkMismatch = true; + break; + case BuildResult::Failure::PermanentFailure: + // Also considered a permenant failure, it seems + case BuildResult::Failure::InputRejected: + permanentFailure = true; + break; + default: + break; + } +#pragma GCC diagnostic pop +} + +unsigned int ExitStatusFlags::failingExitStatus() const +{ + bool buildFailure = permanentFailure || timedOut || hashMismatch; + + /* Any of the 4 booleans we track */ + bool problemWithSpecialExitCode = checkMismatch || buildFailure; + + unsigned int mask = 0; + if (problemWithSpecialExitCode) { + mask |= 0b1100000; + if (buildFailure) { + mask |= 0b0100; // 100 + if (timedOut) + mask |= 0b0001; // 101 + if (hashMismatch) + mask |= 0b0010; // 102 + } + if (checkMismatch) + mask |= 0b1000; // 104 + } + + /* We still (per the function docs) only call this function in the + failure case, so the default should not be 0, but 1, indicating + "some other kind of error. */ + return mask ? mask : 1; +} + bool BuildResult::operator==(const BuildResult &) const noexcept = default; std::strong_ordering BuildResult::operator<=>(const BuildResult &) const noexcept = default; bool BuildResult::Success::operator==(const BuildResult::Success &) const noexcept = default; std::strong_ordering BuildResult::Success::operator<=>(const BuildResult::Success &) const noexcept = default; -bool BuildResult::Failure::operator==(const BuildResult::Failure &) const noexcept = default; -std::strong_ordering BuildResult::Failure::operator<=>(const BuildResult::Failure &) const noexcept = default; - static constexpr std::array, 4> successStatusStrings{{ #define ENUM_ENTRY(e) {BuildResult::Success::e, #e} ENUM_ENTRY(Built), @@ -24,7 +74,7 @@ static constexpr std::array(const BuildError & other) const noexcept +{ + if (auto cmp = status <=> other.status; cmp != 0) + return cmp; + if (auto cmp = isNonDeterministic <=> other.isNonDeterministic; cmp != 0) + return cmp; + return message() <=> other.message(); +} + } // namespace nix namespace nlohmann { @@ -105,15 +169,15 @@ void adl_serializer::to_json(json & res, const BuildResult & br) overloaded{ [&](const BuildResult::Success & success) { res["success"] = true; - res["status"] = BuildResult::Success::statusToString(success.status); + res["status"] = successStatusToString(success.status); res["builtOutputs"] = success.builtOutputs; if (success.provenance) res["provenance"] = success.provenance->to_json(); }, [&](const BuildResult::Failure & failure) { res["success"] = false; - res["status"] = BuildResult::Failure::statusToString(failure.status); - res["errorMsg"] = failure.errorMsg; + res["status"] = failureStatusToString(failure.status); + res["errorMsg"] = failure.message(); res["isNonDeterministic"] = failure.isNonDeterministic; if (failure.provenance) res["provenance"] = failure.provenance->to_json(); @@ -157,12 +221,11 @@ BuildResult adl_serializer::from_json(const json & _json) s.provenance = provenanceFromJson(optionalValueAt(json, "provenance")); br.inner = std::move(s); } else { - BuildResult::Failure f; - f.status = failureStatusFromString(statusStr); - f.errorMsg = getString(valueAt(json, "errorMsg")); - f.isNonDeterministic = getBoolean(valueAt(json, "isNonDeterministic")); - f.provenance = provenanceFromJson(optionalValueAt(json, "provenance")); - br.inner = std::move(f); + br.inner = BuildResult::Failure{ + {.status = failureStatusFromString(statusStr), + .msg = HintFmt(getString(valueAt(json, "errorMsg"))), + .isNonDeterministic = getBoolean(valueAt(json, "isNonDeterministic")), + .provenance = provenanceFromJson(optionalValueAt(json, "provenance"))}}; } return br; diff --git a/src/libstore/build/build-log.cc b/src/libstore/build/build-log.cc new file mode 100644 index 000000000000..85b8877d6590 --- /dev/null +++ b/src/libstore/build/build-log.cc @@ -0,0 +1,48 @@ +#include "nix/store/build/build-log.hh" + +namespace nix { + +BuildLog::BuildLog(size_t maxTailLines, ref act) + : maxTailLines(maxTailLines) + , act(std::move(act)) +{ +} + +void BuildLog::operator()(std::string_view data) +{ + for (auto c : data) + if (c == '\r') + currentLogLinePos = 0; + else if (c == '\n') + flushLine(); + else { + if (currentLogLinePos >= currentLogLine.size()) + currentLogLine.resize(currentLogLinePos + 1); + currentLogLine[currentLogLinePos++] = c; + } +} + +void BuildLog::flush() +{ + if (!currentLogLine.empty()) + flushLine(); +} + +void BuildLog::flushLine() +{ + // Truncate to actual content (currentLogLinePos may be less than size due to \r) + currentLogLine.resize(currentLogLinePos); + + if (!handleJSONLogMessage(currentLogLine, *act, builderActivities, "the derivation builder", false)) { + // Line was not handled as JSON, emit and add to tail + act->result(resBuildLogLine, currentLogLine); + logTail.push_back(currentLogLine); + if (logTail.size() > maxTailLines) + logTail.pop_front(); + } + + currentLogLine.clear(); + currentLogLinePos = 0; +} + +} // namespace nix diff --git a/src/libstore/build/derivation-building-goal.cc b/src/libstore/build/derivation-building-goal.cc index 871504d5ed83..685624bb6790 100644 --- a/src/libstore/build/derivation-building-goal.cc +++ b/src/libstore/build/derivation-building-goal.cc @@ -4,7 +4,9 @@ # include "nix/store/build/hook-instance.hh" # include "nix/store/build/derivation-builder.hh" #endif +#include "nix/util/fun.hh" #include "nix/util/processes.hh" +#include "nix/util/environment-variables.hh" #include "nix/util/config-global.hh" #include "nix/store/build/worker.hh" #include "nix/util/util.hh" @@ -14,6 +16,7 @@ #include "nix/store/local-store.hh" // TODO remove, along with remaining downcasts #include "nix/store/globals.hh" +#include #include #include #include @@ -40,45 +43,13 @@ DerivationBuildingGoal::DerivationBuildingGoal( worker.store.addTempRoot(this->drvPath); } -DerivationBuildingGoal::~DerivationBuildingGoal() -{ - /* Careful: we should never ever throw an exception from a - destructor. */ -#ifndef _WIN32 // TODO enable `DerivationBuilder` on Windows - if (builder) - builder.reset(); -#endif - try { - closeLogFile(); - } catch (...) { - ignoreExceptionInDestructor(); - } -} +DerivationBuildingGoal::~DerivationBuildingGoal() = default; std::string DerivationBuildingGoal::key() { return "dd$" + std::string(drvPath.name()) + "$" + worker.store.printStorePath(drvPath); } -void DerivationBuildingGoal::killChild() -{ -#ifndef _WIN32 // TODO enable build hook on Windows - hook.reset(); -#endif -#ifndef _WIN32 // TODO enable `DerivationBuilder` on Windows - if (builder && builder->killChild()) - worker.childTerminated(this); -#endif -} - -void DerivationBuildingGoal::timedOut(Error && ex) -{ - killChild(); - // We're not inside a coroutine, hence we can't use co_return here. - // Thus we ignore the return value. - [[maybe_unused]] Done _ = doneFailure({BuildResult::Failure::TimedOut, std::move(ex)}); -} - std::string showKnownOutputs(const StoreDirConfig & store, const Derivation & drv) { std::string msg; @@ -95,7 +66,11 @@ std::string showKnownOutputs(const StoreDirConfig & store, const Derivation & dr } static void runPostBuildHook( - const StoreDirConfig & store, Logger & logger, const StorePath & drvPath, const StorePathSet & outputPaths); + const WorkerSettings & workerSettings, + const StoreDirConfig & store, + Logger & logger, + const StorePath & drvPath, + const StorePathSet & outputPaths); /* At least one of the output paths could not be produced using a substitute. So we have to build instead. */ @@ -119,7 +94,7 @@ Goal::Co DerivationBuildingGoal::gaveUpOnSubstitution(bool storeDerivation) for (auto & i : drv->inputSrcs) { if (worker.store.isValidPath(i)) continue; - if (!settings.useSubstitutes) + if (!worker.settings.useSubstitutes) throw Error( "dependency '%s' of '%s' does not exist, and substitution is disabled", worker.store.printStorePath(i), @@ -151,9 +126,11 @@ Goal::Co DerivationBuildingGoal::gaveUpOnSubstitution(bool storeDerivation) assert(drv->inputDrvs.map.empty()); /* Store the resolved derivation, as part of the record of what we're actually building */ - writeDerivation(worker.store, *drv); + worker.store.writeDerivation(*drv); } + StorePathSet inputPaths; + { /* If we get this far, we know no dynamic drvs inputs */ @@ -187,16 +164,107 @@ Goal::Co DerivationBuildingGoal::gaveUpOnSubstitution(bool storeDerivation) /* Second, the input sources. */ worker.store.computeFSClosure(drv->inputSrcs, inputPaths); - debug("added input paths %s", worker.store.showPaths(inputPaths)); + debug("added input paths %s", concatMapStringsSep(", ", inputPaths, [&](auto & p) { + return "'" + worker.store.printStorePath(p) + "'"; + })); /* Okay, try to build. Note that here we don't wait for a build slot to become available, since we don't need one if there is a build hook. */ co_await yield(); - co_return tryToBuild(); + co_return tryToBuild(std::move(inputPaths)); } -Goal::Co DerivationBuildingGoal::tryToBuild() +/** + * RAII wrapper for build log file. + * Constructor opens the log file, destructor closes it. + */ +struct LogFile +{ + AutoCloseFD fd; + std::shared_ptr fileSink, sink; + + LogFile(Store & store, const StorePath & drvPath, const LogFileSettings & logSettings); + ~LogFile(); +}; + +struct LocalBuildRejection +{ + bool maxJobsZero = false; + + struct NoLocalStore + {}; + + /** + * We have a local store, but we don't have an external derivation builder (which is fine), if we did, it'd be + * fine because we would not care about platforms and features then. Since we don't, we either have the wrong + * platform, or we are missing some system features. + */ + struct WrongLocalStore + { + template + struct Pair + { + T derivation; + T localStore; + }; + + std::optional> badPlatform; + std::optional> missingFeatures; + }; + + std::variant rejection; +}; + +static BuildError reject(const LocalBuildRejection & rejection, std::string_view thingCannotBuild) +{ + if (std::get_if(&rejection.rejection)) + return BuildError( + BuildResult::Failure::InputRejected, + "Unable to build with a primary store that isn't a local store; " + "either pass a different '--store' or enable remote builds.\n\n" + "For more information check 'man nix.conf' and search for '/machines'."); + + auto & wrongStore = std::get(rejection.rejection); + + std::string msg = fmt("Cannot build '%s'.", Magenta(thingCannotBuild)); + + if (rejection.maxJobsZero) + msg += "\nReason: " ANSI_RED "local builds are disabled" ANSI_NORMAL + " (max-jobs = 0)" + "\nHint: set 'max-jobs' to a non-zero value to enable local builds, " + "or configure remote builders via 'builders'"; + + if (wrongStore.badPlatform) + msg += + fmt("\nReason: " ANSI_RED "platform mismatch" ANSI_NORMAL + "\nRequired system: '%s'" + "\nCurrent system: '%s'", + Magenta(wrongStore.badPlatform->derivation), + Magenta(wrongStore.badPlatform->localStore)); + + if (wrongStore.missingFeatures) + msg += + fmt("\nReason: " ANSI_RED "missing system features" ANSI_NORMAL + "\nRequired features: {%s}" + "\nAvailable features: {%s}", + concatStringsSep(", ", wrongStore.missingFeatures->derivation), + concatStringsSep(", ", wrongStore.missingFeatures->localStore)); + + if (wrongStore.badPlatform || wrongStore.missingFeatures) { + // since aarch64-darwin has Rosetta 2, this user can actually run x86_64-darwin on their + // hardware - we should tell them to run the command to install Rosetta + if (wrongStore.badPlatform && wrongStore.badPlatform->derivation == "x86_64-darwin" + && wrongStore.badPlatform->localStore == "aarch64-darwin") + msg += + fmt("\nNote: run `%s` to run programs for x86_64-darwin", + Magenta("/usr/sbin/softwareupdate --install-rosetta && launchctl stop org.nixos.nix-daemon")); + } + + return BuildError(BuildResult::Failure::InputRejected, std::move(msg)); +} + +Goal::Co DerivationBuildingGoal::tryToBuild(StorePathSet inputPaths) { auto drvOptions = [&] { DerivationOptions temp; @@ -257,56 +325,57 @@ Goal::Co DerivationBuildingGoal::tryToBuild() } checkPathValidity(initialOutputs); - auto started = [&]() { - auto msg = - fmt(buildMode == bmRepair ? "repairing outputs of '%s'" - : buildMode == bmCheck ? "checking outputs of '%s'" - : "building '%s'", - worker.store.printStorePath(drvPath)); -#ifndef _WIN32 // TODO enable build hook on Windows - if (hook) - msg += fmt(" on '%s'", hook->machineName); -#endif - act = std::make_unique( - *logger, - lvlInfo, - actBuild, - msg, - Logger::Fields{ - worker.store.printStorePath(drvPath), -#ifndef _WIN32 // TODO enable build hook on Windows - hook ? hook->machineName : -#endif - "", - 1, - 1}); - mcRunningBuilds = std::make_unique>(worker.runningBuilds); - worker.updateProgress(); - }; + auto localBuildResult = [&]() -> std::variant { + bool maxJobsZero = worker.settings.maxBuildJobs.get() == 0; - /** - * Activity that denotes waiting for a lock. - */ - std::unique_ptr actLock; + auto * localStoreP = dynamic_cast(&worker.store); + if (!localStoreP) + return LocalBuildRejection{.maxJobsZero = maxJobsZero, .rejection = LocalBuildRejection::NoLocalStore{}}; - /** - * Locks on (fixed) output paths. - */ - PathLocks outputLocks; + /** + * Now that we've decided we can't / won't do a remote build, check + * that we can in fact build locally. First see if there is an + * external builder for a "semi-local build". If there is, prefer to + * use that. If there is not, then check if we can do a "true" local + * build. + */ + auto * ext = settings.getLocalSettings().findExternalDerivationBuilderIfSupported(*drv); - bool useHook; + if (ext) + return LocalBuildCapability{*localStoreP, ext}; - const ExternalBuilder * externalBuilder = nullptr; + using WrongLocalStore = LocalBuildRejection::WrongLocalStore; - while (true) { - trace("trying to build"); + WrongLocalStore wrongStore; - /* Obtain locks on all output paths, if the paths are known a priori. + if (drv->platform != settings.thisSystem.get() && drv->platform != "wasm32-wasip1" + && !settings.extraPlatforms.get().count(drv->platform) && !drv->isBuiltin()) + wrongStore.badPlatform = WrongLocalStore::Pair{drv->platform, settings.thisSystem.get()}; - The locks are automatically released when we exit this function or Nix - crashes. If we can't acquire the lock, then continue; hopefully some - other goal can start a build, and if not, the main loop will sleep a few - seconds and then retry this goal. */ + { + auto required = drvOptions.getRequiredSystemFeatures(*drv); + auto & available = worker.store.config.systemFeatures.get(); + if (std::ranges::any_of(required, [&](const std::string & f) { return !available.count(f); })) + wrongStore.missingFeatures = WrongLocalStore::Pair{required, available}; + } + + if (maxJobsZero || wrongStore.badPlatform || wrongStore.missingFeatures) + return LocalBuildRejection{.maxJobsZero = maxJobsZero, .rejection = std::move(wrongStore)}; + + return LocalBuildCapability{*localStoreP, ext}; + }(); + + auto acquireResources = [&](bool & done, PathLocks & outputLocks) -> Goal::Co { + trace("trying to build"); + + /** + * Output paths to acquire locks on, if known a priori. + * + * The locks are automatically released when the caller's `PathLocks` goes + * out of scope, including on exception unwinding. If we can't acquire the lock, then + * continue; hopefully some other goal can start a build, and if not, the + * main loop will sleep a few seconds and then retry this goal. + */ std::set lockFiles; /* FIXME: Should lock something like the drv itself so we don't build same CA drv concurrently */ @@ -321,14 +390,21 @@ Goal::Co DerivationBuildingGoal::tryToBuild() for (auto & i : drv->outputsAndOptPaths(worker.store)) { if (i.second.second) lockFiles.insert(localStore->toRealPath(*i.second.second)); - else - lockFiles.insert(localStore->toRealPath(drvPath) + "." + i.first); + else { + auto lockPath = localStore->toRealPath(drvPath); + lockPath += "." + i.first; + lockFiles.insert(std::move(lockPath)); + } } } if (!outputLocks.lockPaths(lockFiles, "", false)) { Activity act( - *logger, lvlWarn, actBuildWaiting, fmt("waiting for lock on %s", Magenta(showPaths(lockFiles)))); + *logger, + lvlWarn, + actBuildWaiting, + fmt("waiting for lock on %s", + Magenta(concatMapStringsSep(", ", lockFiles, [](auto & p) { return "'" + p.string() + "'"; })))); /* Wait then try locking again, repeat until success (returned boolean is true). */ @@ -350,7 +426,8 @@ Goal::Co DerivationBuildingGoal::tryToBuild() debug("skipping build of derivation '%s', someone beat us to it", worker.store.printStorePath(drvPath)); outputLocks.setDeletion(true); outputLocks.unlock(); - co_return doneSuccess(BuildResult::Success::AlreadyValid, std::move(validOutputs)); + done = true; + co_return Return{}; } /* If any of the outputs already exist but are not valid, delete @@ -365,198 +442,390 @@ Goal::Co DerivationBuildingGoal::tryToBuild() } } - /* Don't do a remote build if the derivation has the attribute - `preferLocalBuild' set. Also, check and repair modes are only - supported for local builds. */ - bool buildLocally = (buildMode != bmNormal || drvOptions.willBuildLocally(worker.store, *drv)) - && settings.maxBuildJobs.get() != 0; + co_return Return{}; + }; - if (buildLocally) { - useHook = false; - } else { - switch (tryBuildHook(initialOutputs, drvOptions)) { + auto tryHookLoop = [&](bool & valid) -> Goal::Co { + { + PathLocks outputLocks; + co_await acquireResources(valid, outputLocks); + if (valid) + co_return doneSuccess(BuildResult::Success::AlreadyValid, checkPathValidity(initialOutputs).second); + + switch (tryBuildHook(drvOptions)) { case rpAccept: /* Yes, it has started doing so. Wait until we get EOF from the hook. */ - useHook = true; - break; + valid = true; + co_return buildWithHook( + std::move(inputPaths), std::move(initialOutputs), std::move(drvOptions), std::move(outputLocks)); + case rpDecline: + // We should do it ourselves. + co_return Return{}; case rpPostpone: /* Not now; wait until at least one child finishes or the wake-up timeout expires. */ - if (!actLock) - actLock = std::make_unique( - *logger, - lvlWarn, - actBuildWaiting, - fmt("waiting for a machine to build '%s'", Magenta(worker.store.printStorePath(drvPath)))); - outputLocks.unlock(); + break; + } + } + + PathLocks outputLocks; + { + // First attempt was postponed. Retry in a loop with an activity + // that lives until accept or decline. + Activity act( + *logger, + lvlWarn, + actBuildWaiting, + fmt("waiting for a machine to build '%s'", Magenta(worker.store.printStorePath(drvPath)))); + + while (true) { co_await waitForAWhile(); - continue; - case rpDecline: - /* We should do it ourselves. - - Now that we've decided we can't / won't do a remote build, check - that we can in fact build locally. First see if there is an - external builder for a "semi-local build". If there is, prefer to - use that. If there is not, then check if we can do a "true" local - build. */ - - externalBuilder = settings.findExternalDerivationBuilderIfSupported(*drv); - - if (!externalBuilder && !drvOptions.canBuildLocally(worker.store, *drv)) { - auto msg = - fmt("Cannot build '%s'.\n" - "Reason: " ANSI_RED "required system or feature not available" ANSI_NORMAL - "\n" - "Required system: '%s' with features {%s}\n" - "Current system: '%s' with features {%s}", - Magenta(worker.store.printStorePath(drvPath)), - Magenta(drv->platform), - concatStringsSep(", ", drvOptions.getRequiredSystemFeatures(*drv)), - Magenta(settings.thisSystem), - concatStringsSep(", ", worker.store.Store::config.systemFeatures)); - - // since aarch64-darwin has Rosetta 2, this user can actually run x86_64-darwin on their hardware - - // we should tell them to run the command to install Darwin 2 - if (drv->platform == "x86_64-darwin" && settings.thisSystem == "aarch64-darwin") - msg += fmt( - "\nNote: run `%s` to run programs for x86_64-darwin", - Magenta( - "/usr/sbin/softwareupdate --install-rosetta && launchctl stop org.nixos.nix-daemon")); - -#ifndef _WIN32 // TODO enable `DerivationBuilder` on Windows - builder.reset(); -#endif + co_await acquireResources(valid, outputLocks); + if (valid) + break; + + switch (tryBuildHook(drvOptions)) { + case rpAccept: + /* Yes, it has started doing so. Wait until we get + EOF from the hook. */ + break; + case rpPostpone: + /* Not now; wait until at least one child finishes or + the wake-up timeout expires. */ outputLocks.unlock(); - worker.permanentFailure = true; - co_return doneFailure({BuildResult::Failure::InputRejected, std::move(msg)}); + continue; + case rpDecline: + // We should do it ourselves. + co_return Return{}; } - useHook = false; + break; } } - break; - } - actLock.reset(); + if (valid) { + co_return doneSuccess(BuildResult::Success::AlreadyValid, checkPathValidity(initialOutputs).second); + } else { + co_return buildWithHook( + std::move(inputPaths), std::move(initialOutputs), std::move(drvOptions), std::move(outputLocks)); + } + }; - /* Get the provenance of the derivation, if available. */ - std::shared_ptr provenance; - if (auto info = worker.evalStore.maybeQueryPathInfo(drvPath)) - provenance = info->provenance; + auto tryBuildLocally = [&](bool & valid) -> Goal::Co { + if (auto * cap = std::get_if(&localBuildResult)) { + PathLocks outputLocks; + co_await acquireResources(valid, outputLocks); + if (valid) + co_return doneSuccess(BuildResult::Success::AlreadyValid, checkPathValidity(initialOutputs).second); - if (useHook) { - buildResult.startTime = time(0); // inexact - started(); - co_await Suspend{}; + valid = true; + co_return buildLocally( + *cap, std::move(inputPaths), std::move(initialOutputs), std::move(drvOptions), std::move(outputLocks)); + } -#ifndef _WIN32 - assert(hook); -#endif + co_return Return{}; + }; - trace("hook build done"); + if (buildMode != bmNormal) { + // Check and repair modes operate on the state of this store specifically, + // so they must always build locally. + bool valid = false; + co_await tryBuildLocally(valid); + if (valid) + co_return Return{}; + } else if (drvOptions.preferLocalBuild) { + // Local is preferred, so try it first. If it's not available, fall back to the hook. + { + bool valid = false; + co_await tryBuildLocally(valid); + if (valid) + co_return Return{}; + } + { + bool valid = false; + co_await tryHookLoop(valid); + if (valid) + co_return Return{}; + } + } else { + // Default preference is a remote build: they tend to be faster and preserve local + // resources for other tasks. Fall back to local if no remote is available. + { + bool valid = false; + co_await tryHookLoop(valid); + if (valid) + co_return Return{}; + } + { + bool valid = false; + co_await tryBuildLocally(valid); + if (valid) + co_return Return{}; + } + } - /* Since we got an EOF on the logger pipe, the builder is presumed - to have terminated. In fact, the builder could also have - simply have closed its end of the pipe, so just to be sure, - kill it. */ - int status = -#ifndef _WIN32 // TODO enable build hook on Windows - hook->pid.kill(); + std::string storePath = worker.store.printStorePath(drvPath); + auto * rejection = std::get_if(&localBuildResult); + assert(rejection); + co_return doneFailure(reject(*rejection, storePath)); +} + +Goal::Co DerivationBuildingGoal::buildWithHook( + StorePathSet inputPaths, + std::map initialOutputs, + DerivationOptions drvOptions, + PathLocks outputLocks) +{ +#ifdef _WIN32 // TODO enable build hook on Windows + unreachable(); #else - 0; -#endif + std::unique_ptr hook = std::move(worker.hook); - debug("build hook for '%s' finished", worker.store.printStorePath(drvPath)); + /* Set up callback so childTerminated is called if the hook is + destroyed (e.g., during failure cascades). */ + hook->onKillChild = [this]() { worker.childTerminated(this, JobCategory::Build); }; - buildResult.timesBuilt++; - buildResult.stopTime = time(0); + try { + hook->machineName = readLine(hook->fromHook.readSide.get()); + } catch (Error & e) { + e.addTrace({}, "while reading the machine name from the build hook"); + throw; + } - /* So the child is gone now. */ - worker.childTerminated(this); + CommonProto::WriteConn conn{hook->sink}; - /* Close the read side of the logger pipe. */ -#ifndef _WIN32 // TODO enable build hook on Windows - hook->builderOut.readSide.close(); - hook->fromHook.readSide.close(); -#endif + /* Tell the hook all the inputs that have to be copied to the + remote system. */ + CommonProto::write(worker.store, conn, inputPaths); - /* Close the log file. */ - closeLogFile(); + /* Tell the hooks the missing outputs that have to be copied back + from the remote system. */ + { + StringSet missingOutputs; + for (auto & [outputName, status] : initialOutputs) { + // XXX: Does this include known CA outputs? + if (buildMode != bmCheck && status.known && status.known->isValid()) + continue; + missingOutputs.insert(outputName); + } + CommonProto::write(worker.store, conn, missingOutputs); + } - /* Check the exit status. */ - if (!statusOk(status)) { - auto e = fixupBuilderFailureErrorMessage({BuildResult::Failure::MiscFailure, status, ""}); + hook->sink = FdSink(); + hook->toHook.writeSide.close(); - outputLocks.unlock(); + /* Create the log file and pipe. */ + std::unique_ptr logFile = std::make_unique(worker.store, drvPath, settings.getLogFileSettings()); + + std::set fds; + fds.insert(hook->fromHook.readSide.get()); + fds.insert(hook->builderOut.readSide.get()); + worker.childStarted(shared_from_this(), fds, false, false); - /* TODO (once again) support fine-grained error codes, see issue #12641. */ + buildResult.startTime = time(nullptr); // inexact - co_return doneFailure(std::move(e)); + auto msg = + fmt(buildMode == bmRepair ? "repairing outputs of '%s'" + : buildMode == bmCheck ? "checking outputs of '%s'" + : "building '%s'", + worker.store.printStorePath(drvPath)); + msg += fmt(" on '%s'", hook->machineName); + + std::unique_ptr buildLog = std::make_unique( + worker.settings.logLines, + make_ref( + *logger, + lvlInfo, + actBuild, + msg, + Logger::Fields{worker.store.printStorePath(drvPath), hook->machineName, 1, 1})); + mcRunningBuilds = std::make_unique>(worker.runningBuilds); + worker.updateProgress(); + + std::string currentHookLine; + uint64_t logSize = 0; + + while (true) { + auto event = co_await WaitForChildEvent{}; + if (auto * output = std::get_if(&event)) { + auto & fd = output->fd; + auto & data = output->data; + if (fd == hook->builderOut.readSide.get()) { + logSize += data.size(); + if (worker.settings.maxLogSize && logSize > worker.settings.maxLogSize) { + hook.reset(); + co_return doneFailureLogTooLong(*buildLog); + } + (*buildLog)(data); + if (logFile->sink) + (*logFile->sink)(data); + } else if (fd == hook->fromHook.readSide.get()) { + for (auto c : data) + if (c == '\n') { + auto json = parseJSONMessage(currentHookLine, "the derivation builder"); + if (json) { + auto s = handleJSONLogMessage( + *json, worker.act, hook->activities, "the derivation builder", true); + // ensure that logs from a builder using `ssh-ng://` as protocol + // are also available to `nix log`. + if (s && logFile->sink) { + const auto type = (*json)["type"]; + const auto fields = (*json)["fields"]; + if (type == resBuildLogLine) { + (*logFile->sink)((fields.size() > 0 ? fields[0].get() : "") + "\n"); + } else if (type == resSetPhase && !fields.is_null()) { + const auto phase = fields[0]; + if (!phase.is_null()) { + // nixpkgs' stdenv produces lines in the log to signal + // phase changes. + // We want to get the same lines in case of remote builds. + // The format is: + // @nix { "action": "setPhase", "phase": "$curPhase" } + const auto logLine = + nlohmann::json::object({{"action", "setPhase"}, {"phase", phase}}); + (*logFile->sink)( + "@nix " + + logLine.dump(-1, ' ', false, nlohmann::json::error_handler_t::replace) + + "\n"); + } + } + } + } + currentHookLine.clear(); + } else + currentHookLine += c; + } + } else if (std::get_if(&event)) { + buildLog->flush(); + break; + } else if (auto * timeout = std::get_if(&event)) { + hook.reset(); + co_return doneFailure(std::move(*timeout)); } + } - /* Compute the FS closure of the outputs and register them as - being valid. */ - auto builtOutputs = - /* When using a build hook, the build hook can register the output - as valid (by doing `nix-store --import'). If so we don't have - to do anything here. + trace("hook build done"); - We can only early return when the outputs are known a priori. For - floating content-addressing derivations this isn't the case. + /* Since we got an EOF on the logger pipe, the builder is presumed + to have terminated. In fact, the builder could also have + simply have closed its end of the pipe, so just to be sure, + kill it. */ + int status = hook->pid.kill(); - Aborts if any output is not valid or corrupt, and otherwise - returns a 'SingleDrvOutputs' structure containing all outputs. - */ - [&] { - auto [allValid, validOutputs] = checkPathValidity(initialOutputs); - if (!allValid) - throw Error("some outputs are unexpectedly invalid"); - return validOutputs; - }(); + debug("build hook for '%s' finished", worker.store.printStorePath(drvPath)); - StorePathSet outputPaths; - for (auto & [_, output] : builtOutputs) - outputPaths.insert(output.outPath); - runPostBuildHook(worker.store, *logger, drvPath, outputPaths); + buildResult.timesBuilt++; + buildResult.stopTime = time(nullptr); + + /* So the child is gone now. */ + worker.childTerminated(this); + + /* Close the read side of the logger pipe. */ + hook->builderOut.readSide.close(); + hook->fromHook.readSide.close(); + + /* Close the log file. */ + logFile.reset(); + + /* Check the exit status. */ + if (!statusOk(status)) { + auto e = fixupBuilderFailureErrorMessage({BuildResult::Failure::MiscFailure, status, ""}, *buildLog); - /* It is now safe to delete the lock files, since all future - lockers will see that the output paths are valid; they will - not create new lock files with the same names as the old - (unlinked) lock files. */ - outputLocks.setDeletion(true); outputLocks.unlock(); - co_return doneSuccess(BuildResult::Success::Built, std::move(builtOutputs), provenance); + /* TODO (once again) support fine-grained error codes, see issue #12641. */ + + co_return doneFailure(std::move(e)); } - co_await yield(); + /* Compute the FS closure of the outputs and register them as + being valid. */ + auto builtOutputs = + /* When using a build hook, the build hook can register the output + as valid (by doing `nix-store --import'). If so we don't have + to do anything here. - if (!dynamic_cast(&worker.store)) { - throw Error( - R"( - Unable to build with a primary store that isn't a local store; - either pass a different '--store' or enable remote builds. + We can only early return when the outputs are known a priori. For + floating content-addressing derivations this isn't the case. - For more information check 'man nix.conf' and search for '/machines'. - )"); - } + Aborts if any output is not valid or corrupt, and otherwise + returns a 'SingleDrvOutputs' structure containing all outputs. + */ + [&] { + auto [allValid, validOutputs] = checkPathValidity(initialOutputs); + if (!allValid) + throw Error("some outputs are unexpectedly invalid"); + return validOutputs; + }(); + + StorePathSet outputPaths; + for (auto & [_, output] : builtOutputs) + outputPaths.insert(output.outPath); + runPostBuildHook(worker.settings, worker.store, *logger, drvPath, outputPaths); + + /* It is now safe to delete the lock files, since all future + lockers will see that the output paths are valid; they will + not create new lock files with the same names as the old + (unlinked) lock files. */ + outputLocks.setDeletion(true); + outputLocks.unlock(); + + co_return doneSuccess(BuildResult::Success::Built, std::move(builtOutputs)); +#endif +} + +Goal::Co DerivationBuildingGoal::buildLocally( + LocalBuildCapability localBuildCap, + StorePathSet inputPaths, + std::map initialOutputs, + DerivationOptions drvOptions, + PathLocks outputLocks) +{ + co_await yield(); #ifdef _WIN32 // TODO enable `DerivationBuilder` on Windows throw UnimplementedError("building derivations is not yet implemented on Windows"); #else - assert(!hook); + auto msg = + fmt(buildMode == bmRepair ? "repairing outputs of '%s'" + : buildMode == bmCheck ? "checking outputs of '%s'" + : "building '%s'", + worker.store.printStorePath(drvPath)); + auto act = make_ref( + *logger, lvlInfo, actBuild, msg, Logger::Fields{worker.store.printStorePath(drvPath), "", 1, 1}); + std::unique_ptr buildLog; + std::unique_ptr logFile; + + auto openLogFile = [&]() { + logFile = std::make_unique(worker.store, drvPath, settings.getLogFileSettings()); + }; + + auto closeLogFile = [&]() { logFile.reset(); }; + auto started = [&]() { + buildLog = std::make_unique(worker.settings.logLines, act); + mcRunningBuilds = std::make_unique>(worker.runningBuilds); + worker.updateProgress(); + }; + + std::unique_ptr actLock; + DerivationBuilderUnique builder; Descriptor builderOut; + /* Get the provenance of the derivation, if available. */ + std::shared_ptr provenance; + if (auto info = worker.evalStore.maybeQueryPathInfo(drvPath)) + provenance = info->provenance; + // Will continue here while waiting for a build user below while (true) { unsigned int curBuilds = worker.getNrLocalBuilds(); - if (curBuilds >= settings.maxBuildJobs) { + if (curBuilds >= worker.settings.maxBuildJobs) { outputLocks.unlock(); co_await waitForBuildSlot(); - co_return tryToBuild(); + co_return tryToBuild(std::move(inputPaths)); } if (!builder) { @@ -567,9 +836,14 @@ Goal::Co DerivationBuildingGoal::tryToBuild() struct DerivationBuildingGoalCallbacks : DerivationBuilderCallbacks { DerivationBuildingGoal & goal; + fun openLogFileFn; + fun closeLogFileFn; - DerivationBuildingGoalCallbacks(DerivationBuildingGoal & goal) + DerivationBuildingGoalCallbacks( + DerivationBuildingGoal & goal, fun openLogFileFn, fun closeLogFileFn) : goal{goal} + , openLogFileFn{std::move(openLogFileFn)} + , closeLogFileFn{std::move(closeLogFileFn)} { } @@ -577,35 +851,34 @@ Goal::Co DerivationBuildingGoal::tryToBuild() void childTerminated() override { - goal.worker.childTerminated(&goal); + goal.worker.childTerminated(&goal, JobCategory::Build); } - Path openLogFile() override + void openLogFile() override { - return goal.openLogFile(); + openLogFileFn(); } void closeLogFile() override { - goal.closeLogFile(); + closeLogFileFn(); } }; - auto * localStoreP = dynamic_cast(&worker.store); - assert(localStoreP); - - decltype(DerivationBuilderParams::defaultPathsInChroot) defaultPathsInChroot = settings.sandboxPaths.get(); + decltype(DerivationBuilderParams::defaultPathsInChroot) defaultPathsInChroot = + localBuildCap.localStore.config->getLocalSettings().sandboxPaths.get(); DesugaredEnv desugaredEnv; /* Add the closure of store paths to the chroot. */ StorePathSet closure; for (auto & i : defaultPathsInChroot) try { - if (worker.store.isInStore(i.second.source)) - worker.store.computeFSClosure(worker.store.toStorePath(i.second.source).first, closure); + if (worker.store.isInStore(i.second.source.string())) + worker.store.computeFSClosure( + worker.store.toStorePath(i.second.source.string()).first, closure); } catch (InvalidPath & e) { } catch (Error & e) { - e.addTrace({}, "while processing sandbox path '%s'", i.second.source); + e.addTrace({}, "while processing sandbox path %s", PathFmt(i.second.source)); throw; } for (auto & i : closure) { @@ -617,7 +890,6 @@ Goal::Co DerivationBuildingGoal::tryToBuild() desugaredEnv = DesugaredEnv::create(worker.store, *drv, drvOptions, inputPaths); } catch (BuildError & e) { outputLocks.unlock(); - worker.permanentFailure = true; co_return doneFailure(std::move(e)); } @@ -638,15 +910,16 @@ Goal::Co DerivationBuildingGoal::tryToBuild() /* If we have to wait and retry (see below), then `builder` will already be created, so we don't need to create it again. */ - builder = - externalBuilder - ? makeExternalDerivationBuilder( - *localStoreP, - std::make_unique(*this), - std::move(params), - *externalBuilder) - : makeDerivationBuilder( - *localStoreP, std::make_unique(*this), std::move(params)); + builder = localBuildCap.externalBuilder + ? makeExternalDerivationBuilder( + localBuildCap.localStore, + std::make_unique(*this, openLogFile, closeLogFile), + std::move(params), + *localBuildCap.externalBuilder) + : makeDerivationBuilder( + localBuildCap.localStore, + std::make_unique(*this, openLogFile, closeLogFile), + std::move(params)); } if (auto builderOutOpt = builder->startBuild()) { @@ -670,7 +943,30 @@ Goal::Co DerivationBuildingGoal::tryToBuild() worker.childStarted(shared_from_this(), {builderOut}, true, true); started(); - co_await Suspend{}; + + uint64_t logSize = 0; + + while (true) { + auto event = co_await WaitForChildEvent{}; + if (auto * output = std::get_if(&event)) { + if (output->fd == builder->builderOut.get()) { + logSize += output->data.size(); + if (worker.settings.maxLogSize && logSize > worker.settings.maxLogSize) { + builder->killChild(); + co_return doneFailureLogTooLong(*buildLog); + } + (*buildLog)(output->data); + if (logFile->sink) + (*logFile->sink)(output->data); + } + } else if (std::get_if(&event)) { + buildLog->flush(); + break; + } else if (auto * timeout = std::get_if(&event)) { + builder->killChild(); + co_return doneFailure(std::move(*timeout)); + } + } trace("build done"); @@ -680,30 +976,10 @@ Goal::Co DerivationBuildingGoal::tryToBuild() } catch (BuilderFailureError & e) { builder.reset(); outputLocks.unlock(); - co_return doneFailure(fixupBuilderFailureErrorMessage(std::move(e))); + co_return doneFailure(fixupBuilderFailureErrorMessage(std::move(e), *buildLog)); } catch (BuildError & e) { builder.reset(); outputLocks.unlock(); -// Allow selecting a subset of enum values -# pragma GCC diagnostic push -# pragma GCC diagnostic ignored "-Wswitch-enum" - switch (e.status) { - case BuildResult::Failure::HashMismatch: - worker.hashMismatch = true; - /* See header, the protocols don't know about `HashMismatch` - yet, so change it to `OutputRejected`, which they expect - for this case (hash mismatch is a type of output - rejection). */ - e.status = BuildResult::Failure::OutputRejected; - break; - case BuildResult::Failure::NotDeterministic: - worker.checkMismatch = true; - break; - default: - /* Other statuses need no adjusting */ - break; - } -# pragma GCC diagnostic pop co_return doneFailure(std::move(e)); } { @@ -722,7 +998,7 @@ Goal::Co DerivationBuildingGoal::tryToBuild() worker.markContentsGood(output.outPath); outputPaths.insert(output.outPath); } - runPostBuildHook(worker.store, *logger, drvPath, outputPaths); + runPostBuildHook(worker.settings, worker.store, *logger, drvPath, outputPaths); /* It is now safe to delete the lock files, since all future lockers will see that the output paths are valid; they will @@ -736,9 +1012,13 @@ Goal::Co DerivationBuildingGoal::tryToBuild() } static void runPostBuildHook( - const StoreDirConfig & store, Logger & logger, const StorePath & drvPath, const StorePathSet & outputPaths) + const WorkerSettings & workerSettings, + const StoreDirConfig & store, + Logger & logger, + const StorePath & drvPath, + const StorePathSet & outputPaths) { - auto hook = settings.postBuildHook; + auto hook = workerSettings.postBuildHook; if (hook == "") return; @@ -746,14 +1026,15 @@ static void runPostBuildHook( logger, lvlTalkative, actPostBuildHook, - fmt("running post-build-hook '%s'", settings.postBuildHook), + fmt("running post-build-hook '%s'", workerSettings.postBuildHook), Logger::Fields{store.printStorePath(drvPath)}); PushActivity pact(act.id); - StringMap hookEnvironment = getEnv(); + OsStringMap hookEnvironment = getEnvOs(); - hookEnvironment.emplace("DRV_PATH", store.printStorePath(drvPath)); - hookEnvironment.emplace("OUT_PATHS", chomp(concatStringsSep(" ", store.printStorePathSet(outputPaths)))); - hookEnvironment.emplace("NIX_CONFIG", globalConfig.toKeyValue()); + hookEnvironment.emplace(OS_STR("DRV_PATH"), string_to_os_string(store.printStorePath(drvPath))); + hookEnvironment.emplace( + OS_STR("OUT_PATHS"), string_to_os_string(chomp(concatStringsSep(" ", store.printStorePathSet(outputPaths))))); + hookEnvironment.emplace(OS_STR("NIX_CONFIG"), string_to_os_string(globalConfig.toKeyValue())); struct LogSink : Sink { @@ -794,14 +1075,14 @@ static void runPostBuildHook( LogSink sink(act); runProgram2({ - .program = settings.postBuildHook, + .program = workerSettings.postBuildHook.get(), .environment = hookEnvironment, .standardOut = &sink, .mergeStderrToStdout = true, }); } -BuildError DerivationBuildingGoal::fixupBuilderFailureErrorMessage(BuilderFailureError e) +BuildError DerivationBuildingGoal::fixupBuilderFailureErrorMessage(BuilderFailureError e, BuildLog & buildLog) { auto msg = fmt("Cannot build '%s'.\n" @@ -811,6 +1092,7 @@ BuildError DerivationBuildingGoal::fixupBuilderFailureErrorMessage(BuilderFailur msg += showKnownOutputs(worker.store, *drv); + auto & logTail = buildLog.getTail(); if (!logger->isVerbose() && !logTail.empty()) { msg += fmt("\nLast %d log lines:\n", logTail.size()); for (auto & line : logTail) { @@ -833,25 +1115,25 @@ BuildError DerivationBuildingGoal::fixupBuilderFailureErrorMessage(BuilderFailur return BuildError{e.status, msg}; } -HookReply DerivationBuildingGoal::tryBuildHook( - const std::map & initialOutputs, const DerivationOptions & drvOptions) +HookReply DerivationBuildingGoal::tryBuildHook(const DerivationOptions & drvOptions) { #ifdef _WIN32 // TODO enable build hook on Windows return rpDecline; #else /* This should use `worker.evalStore`, but per #13179 the build hook doesn't work with eval store anyways. */ - if (settings.buildHook.get().empty() || !worker.tryBuildHook || !worker.store.isValidPath(drvPath)) + if (worker.settings.buildHook.get().empty() || !worker.tryBuildHook || !worker.store.isValidPath(drvPath)) return rpDecline; if (!worker.hook) - worker.hook = std::make_unique(); + worker.hook = std::make_unique(worker.settings.buildHook); try { /* Send the request to the hook. */ - worker.hook->sink << "try" << (worker.getNrLocalBuilds() < settings.maxBuildJobs ? 1 : 0) << drv->platform - << worker.store.printStorePath(drvPath) << drvOptions.getRequiredSystemFeatures(*drv); + worker.hook->sink << "try" << (worker.getNrLocalBuilds() < worker.settings.maxBuildJobs ? 1 : 0) + << drv->platform << worker.store.printStorePath(drvPath) + << drvOptions.getRequiredSystemFeatures(*drv); worker.hook->sink.flush(); /* Read the first line of input, which should be a word indicating @@ -890,8 +1172,8 @@ HookReply DerivationBuildingGoal::tryBuildHook( else if (reply != "accept") throw Error("bad hook reply '%s'", reply); - } catch (SysError & e) { - if (e.errNo == EPIPE) { + } catch (SystemError & e) { + if (e.is(std::errc::broken_pipe)) { printError("build hook died unexpectedly: %s", chomp(drainFD(worker.hook->fromHook.readSide.get()))); worker.hook = 0; return rpDecline; @@ -899,202 +1181,65 @@ HookReply DerivationBuildingGoal::tryBuildHook( throw; } - hook = std::move(worker.hook); - - try { - hook->machineName = readLine(hook->fromHook.readSide.get()); - } catch (Error & e) { - e.addTrace({}, "while reading the machine name from the build hook"); - throw; - } - - CommonProto::WriteConn conn{hook->sink}; - - /* Tell the hook all the inputs that have to be copied to the - remote system. */ - CommonProto::write(worker.store, conn, inputPaths); - - /* Tell the hooks the missing outputs that have to be copied back - from the remote system. */ - { - StringSet missingOutputs; - for (auto & [outputName, status] : initialOutputs) { - // XXX: Does this include known CA outputs? - if (buildMode != bmCheck && status.known && status.known->isValid()) - continue; - missingOutputs.insert(outputName); - } - CommonProto::write(worker.store, conn, missingOutputs); - } - - hook->sink = FdSink(); - hook->toHook.writeSide.close(); - - /* Create the log file and pipe. */ - [[maybe_unused]] Path logFile = openLogFile(); - - std::set fds; - fds.insert(hook->fromHook.readSide.get()); - fds.insert(hook->builderOut.readSide.get()); - worker.childStarted(shared_from_this(), fds, false, false); - return rpAccept; #endif } -Path DerivationBuildingGoal::openLogFile() +LogFile::LogFile(Store & store, const StorePath & drvPath, const LogFileSettings & logSettings) { - logSize = 0; - - if (!settings.keepLog) - return ""; + if (!logSettings.keepLog) + return; - auto baseName = std::string(baseNameOf(worker.store.printStorePath(drvPath))); + auto baseName = std::string(baseNameOf(store.printStorePath(drvPath))); - /* Create a log file. */ - Path logDir; - if (auto localStore = dynamic_cast(&worker.store)) - logDir = localStore->config->logDir; + std::filesystem::path logDir; + if (auto localStore = dynamic_cast(&store)) + logDir = localStore->config->logDir.get(); else - logDir = settings.nixLogDir; - Path dir = fmt("%s/%s/%s/", logDir, LocalFSStore::drvsLogDir, baseName.substr(0, 2)); + logDir = logSettings.nixLogDir; + auto dir = logDir / LocalFSStore::drvsLogDir / baseName.substr(0, 2); createDirs(dir); - Path logFileName = fmt("%s/%s%s", dir, baseName.substr(2), settings.compressLog ? ".bz2" : ""); + auto logFileName = dir / (baseName.substr(2) + (logSettings.compressLog ? ".bz2" : "")); - fdLogFile = toDescriptor(open( - logFileName.c_str(), - O_CREAT | O_WRONLY | O_TRUNC -#ifndef _WIN32 - | O_CLOEXEC -#endif - , - 0666)); - if (!fdLogFile) - throw SysError("creating log file '%1%'", logFileName); + fd = openNewFileForWrite( + logFileName, + 0666, + { + .truncateExisting = true, + .followSymlinksOnTruncate = true, /* FIXME: Probably shouldn't follow symlinks. */ + }); + if (!fd) + throw SysError("creating log file %1%", PathFmt(logFileName)); - logFileSink = std::make_shared(fdLogFile.get()); + fileSink = std::make_shared(fd.get()); - if (settings.compressLog) - logSink = std::shared_ptr(makeCompressionSink("bzip2", *logFileSink)); + if (logSettings.compressLog) + sink = std::shared_ptr(makeCompressionSink(CompressionAlgo::bzip2, *fileSink)); else - logSink = logFileSink; - - return logFileName; -} - -void DerivationBuildingGoal::closeLogFile() -{ - auto logSink2 = std::dynamic_pointer_cast(logSink); - if (logSink2) - logSink2->finish(); - if (logFileSink) - logFileSink->flush(); - logSink = logFileSink = 0; - fdLogFile.close(); + sink = fileSink; } -bool DerivationBuildingGoal::isReadDesc(Descriptor fd) +LogFile::~LogFile() { -#ifdef _WIN32 // TODO enable build hook on Windows - return false; -#else - return (hook && fd == hook->builderOut.readSide.get()) || (builder && fd == builder->builderOut.get()); -#endif -} - -void DerivationBuildingGoal::handleChildOutput(Descriptor fd, std::string_view data) -{ - // local & `ssh://`-builds are dealt with here. - auto isWrittenToLog = isReadDesc(fd); - if (isWrittenToLog) { - logSize += data.size(); - if (settings.maxLogSize && logSize > settings.maxLogSize) { - killChild(); - // We're not inside a coroutine, hence we can't use co_return here. - // Thus we ignore the return value. - [[maybe_unused]] Done _ = doneFailure(BuildError( - BuildResult::Failure::LogLimitExceeded, - "%s killed after writing more than %d bytes of log output", - getName(), - settings.maxLogSize)); - return; - } - - for (auto c : data) - if (c == '\r') - currentLogLinePos = 0; - else if (c == '\n') - flushLine(); - else { - if (currentLogLinePos >= currentLogLine.size()) - currentLogLine.resize(currentLogLinePos + 1); - currentLogLine[currentLogLinePos++] = c; - } - - if (logSink) - (*logSink)(data); - } - -#ifndef _WIN32 // TODO enable build hook on Windows - if (hook && fd == hook->fromHook.readSide.get()) { - for (auto c : data) - if (c == '\n') { - auto json = parseJSONMessage(currentHookLine, "the derivation builder"); - if (json) { - auto s = handleJSONLogMessage(*json, worker.act, hook->activities, "the derivation builder", true); - // ensure that logs from a builder using `ssh-ng://` as protocol - // are also available to `nix log`. - if (s && !isWrittenToLog && logSink) { - const auto type = (*json)["type"]; - const auto fields = (*json)["fields"]; - if (type == resBuildLogLine) { - (*logSink)((fields.size() > 0 ? fields[0].get() : "") + "\n"); - } else if (type == resSetPhase && !fields.is_null()) { - const auto phase = fields[0]; - if (!phase.is_null()) { - // nixpkgs' stdenv produces lines in the log to signal - // phase changes. - // We want to get the same lines in case of remote builds. - // The format is: - // @nix { "action": "setPhase", "phase": "$curPhase" } - const auto logLine = nlohmann::json::object({{"action", "setPhase"}, {"phase", phase}}); - (*logSink)( - "@nix " + logLine.dump(-1, ' ', false, nlohmann::json::error_handler_t::replace) - + "\n"); - } - } - } - } - currentHookLine.clear(); - } else - currentHookLine += c; + try { + auto sink2 = std::dynamic_pointer_cast(sink); + if (sink2) + sink2->finish(); + if (fileSink) + fileSink->flush(); + } catch (...) { + ignoreExceptionInDestructor(); } -#endif -} - -void DerivationBuildingGoal::handleEOF(Descriptor fd) -{ - if (!currentLogLine.empty()) - flushLine(); - worker.wakeUp(shared_from_this()); } -void DerivationBuildingGoal::flushLine() +Goal::Done DerivationBuildingGoal::doneFailureLogTooLong(BuildLog & buildLog) { - if (handleJSONLogMessage(currentLogLine, *act, builderActivities, "the derivation builder", false)) - ; - - else { - logTail.push_back(currentLogLine); - if (logTail.size() > settings.logLines) - logTail.pop_front(); - - act->result(resBuildLogLine, currentLogLine); - } - - currentLogLine = ""; - currentLogLinePos = 0; + return doneFailure(BuildError( + BuildResult::Failure::LogLimitExceeded, + "%s killed after writing more than %d bytes of log output", + getName(), + worker.settings.maxLogSize)); } std::map> DerivationBuildingGoal::queryPartialDerivationOutputMap() @@ -1198,7 +1343,7 @@ Goal::Done DerivationBuildingGoal::doneSuccess( }); logger->result( - act ? act->id : getCurActivity(), + getCurActivity(), resBuildResult, nlohmann::json(KeyedBuildResult( buildResult, @@ -1211,31 +1356,19 @@ Goal::Done DerivationBuildingGoal::doneFailure(BuildError ex) { mcRunningBuilds.reset(); - if (ex.status == BuildResult::Failure::TimedOut) - worker.timedOut = true; - if (ex.status == BuildResult::Failure::PermanentFailure) - worker.permanentFailure = true; + worker.exitStatusFlags.updateFromStatus(ex.status); if (ex.status != BuildResult::Failure::DependencyFailed) worker.failedBuilds++; worker.updateProgress(); - auto res = Goal::doneFailure( - ecFailed, - BuildResult::Failure{ - .status = ex.status, - .errorMsg = fmt("%s", Uncolored(ex.info().msg)), - }, - std::move(ex)); - logger->result( - act ? act->id : getCurActivity(), + getCurActivity(), resBuildResult, nlohmann::json(KeyedBuildResult( - buildResult, - DerivedPath::Built{.drvPath = makeConstantStorePathRef(drvPath), .outputs = OutputsSpec::All{}}))); + {ex}, DerivedPath::Built{.drvPath = makeConstantStorePathRef(drvPath), .outputs = OutputsSpec::All{}}))); - return res; + return Goal::doneFailure(ecFailed, std::move(ex)); } } // namespace nix diff --git a/src/libstore/build/derivation-check.cc b/src/libstore/build/derivation-check.cc index 677546e878bb..d132b24649ec 100644 --- a/src/libstore/build/derivation-check.cc +++ b/src/libstore/build/derivation-check.cc @@ -15,9 +15,9 @@ void checkOutputs( const std::map & outputs, Activity & act) { - std::map outputsByPath; + std::map outputsByPath; for (auto & output : outputs) - outputsByPath.emplace(store.printStorePath(output.second.path), output.second); + outputsByPath.emplace(output.second.path, output.second); for (auto & pair : outputs) { // We can't use auto destructuring here because @@ -77,7 +77,7 @@ void checkOutputs( if (!pathsDone.insert(path).second) continue; - auto i = outputsByPath.find(store.printStorePath(path)); + auto i = outputsByPath.find(path); if (i != outputsByPath.end()) { closureSize += i->second.narSize; for (auto & ref : i->second.references) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 97188d30c314..2877314979fa 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -1,4 +1,5 @@ #include "nix/store/build/derivation-goal.hh" +#include "nix/store/build/drv-output-substitution-goal.hh" #include "nix/store/build/derivation-building-goal.hh" #include "nix/store/build/derivation-resolution-goal.hh" #ifndef _WIN32 // TODO enable build hook on Windows @@ -99,10 +100,25 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation) /* We are first going to try to create the invalid output paths through substitutes. If that doesn't work, we'll build them. */ - if (settings.useSubstitutes && drvOptions.substitutesAllowed()) { - if (!checkResult) - waitees.insert(upcast_goal(worker.makeDrvOutputSubstitutionGoal(DrvOutput{outputHash, wantedOutput}))); - else { + if (worker.settings.useSubstitutes && drvOptions.substitutesAllowed(worker.settings)) { + if (!checkResult) { + DrvOutput id{outputHash, wantedOutput}; + auto g = worker.makeDrvOutputSubstitutionGoal(id); + waitees.insert(g); + co_await await(std::move(waitees)); + + if (nrFailed == 0) { + waitees.insert(upcast_goal(worker.makePathSubstitutionGoal(g->outputInfo->outPath))); + co_await await(std::move(waitees)); + + trace("output path substituted"); + + if (nrFailed == 0) + worker.store.registerDrvOutput({*g->outputInfo, id}); + else + debug("The output path of the derivation output '%s' could not be substituted", id.to_string()); + } + } else { auto * cap = getDerivationCA(*drv); waitees.insert(upcast_goal(worker.makePathSubstitutionGoal( checkResult->first.outPath, @@ -117,7 +133,7 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation) assert(!drv->type().isImpure()); - if (nrFailed > 0 && nrFailed > nrNoSubstituters && !settings.tryFallback) { + if (nrFailed > 0 && nrFailed > nrNoSubstituters && !worker.settings.tryFallback) { co_return doneFailure(BuildError( BuildResult::Failure::TransientFailure, "some substitutes for the outputs of derivation '%s' failed (usually happens due to networking issues); try '--fallback' to build derivation from source ", @@ -138,7 +154,9 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation) } if (buildMode == bmCheck && !allValid) throw Error( - "some outputs of '%s' are not valid, so checking is not possible", + "some outputs of '%s' are not valid, so checking is not possible\n" + "Hint: --rebuild and --check error if the derivation was not previously built and cannot be substituted.\n" + " Remove it to perform a fresh build, or use --repair to rewrite missing or corrupted builds in the store.", worker.store.printStorePath(drvPath)); } @@ -210,11 +228,6 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation) .outputName = wantedOutput, }}; newRealisation.signatures.clear(); - if (!drv->type().isFixed()) { - auto & drvStore = worker.evalStore.isValidPath(drvPath) ? worker.evalStore : worker.store; - newRealisation.dependentRealisations = - drvOutputReferences(worker.store, *drv, realisation.outPath, &drvStore); - } worker.store.signRealisation(newRealisation); worker.store.registerDrvOutput(newRealisation); } @@ -239,7 +252,7 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation) auto g = worker.makeDerivationBuildingGoal(drvPath, *drv, buildMode, storeDerivation); /* We will finish with it ourselves, as if we were the derivational goal. */ - g->preserveException = true; + g->preserveFailure = true; { Goals waitees; @@ -299,7 +312,7 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation) } } - co_return amDone(g->exitCode, g->ex); + co_return amDone(g->exitCode); } Goal::Co DerivationGoal::repairClosure() @@ -488,31 +501,19 @@ Goal::Done DerivationGoal::doneFailure(BuildError ex) { mcExpectedBuilds.reset(); - if (ex.status == BuildResult::Failure::TimedOut) - worker.timedOut = true; - if (ex.status == BuildResult::Failure::PermanentFailure) - worker.permanentFailure = true; + worker.exitStatusFlags.updateFromStatus(ex.status); if (ex.status != BuildResult::Failure::DependencyFailed) worker.failedBuilds++; worker.updateProgress(); - auto res = Goal::doneFailure( - ecFailed, - BuildResult::Failure{ - .status = ex.status, - .errorMsg = fmt("%s", Uncolored(ex.info().msg)), - }, - std::move(ex)); - logger->result( getCurActivity(), resBuildResult, nlohmann::json(KeyedBuildResult( - buildResult, - DerivedPath::Built{.drvPath = makeConstantStorePathRef(drvPath), .outputs = OutputsSpec::All{}}))); + {ex}, DerivedPath::Built{.drvPath = makeConstantStorePathRef(drvPath), .outputs = OutputsSpec::All{}}))); - return res; + return Goal::doneFailure(ecFailed, std::move(ex)); } } // namespace nix diff --git a/src/libstore/build/derivation-resolution-goal.cc b/src/libstore/build/derivation-resolution-goal.cc index 6cb9702f4f6e..81c698e18563 100644 --- a/src/libstore/build/derivation-resolution-goal.cc +++ b/src/libstore/build/derivation-resolution-goal.cc @@ -90,7 +90,12 @@ Goal::Co DerivationResolutionGoal::resolveDerivation() nrFailed, nrFailed == 1 ? "dependency" : "dependencies"); msg += showKnownOutputs(worker.store, *drv); - co_return amDone(ecFailed, {BuildError(BuildResult::Failure::DependencyFailed, msg)}); + co_return doneFailure( + ecFailed, + BuildResult::Failure{{ + .status = BuildResult::Failure::DependencyFailed, + .msg = HintFmt(msg), + }}); } /* Gather information necessary for computing the closure and/or @@ -102,32 +107,7 @@ Goal::Co DerivationResolutionGoal::resolveDerivation() { auto & fullDrv = *drv; - auto drvType = fullDrv.type(); - bool resolveDrv = - std::visit( - overloaded{ - [&](const DerivationType::InputAddressed & ia) { - /* must resolve if deferred. */ - return ia.deferred; - }, - [&](const DerivationType::ContentAddressed & ca) { - return !fullDrv.inputDrvs.map.empty() - && (ca.fixed - /* Can optionally resolve if fixed, which is good - for avoiding unnecessary rebuilds. */ - ? experimentalFeatureSettings.isEnabled(Xp::CaDerivations) - /* Must resolve if floating and there are any inputs - drvs. */ - : true); - }, - [&](const DerivationType::Impure &) { return true; }}, - drvType.raw) - /* no inputs are outputs of dynamic derivations */ - || std::ranges::any_of(fullDrv.inputDrvs.map.begin(), fullDrv.inputDrvs.map.end(), [](auto & pair) { - return !pair.second.childMap.empty(); - }); - - if (resolveDrv && !fullDrv.inputDrvs.map.empty()) { + if (fullDrv.shouldResolve()) { experimentalFeatureSettings.require(Xp::CaDerivations); /* We are be able to resolve this derivation based on the @@ -164,7 +144,7 @@ Goal::Co DerivationResolutionGoal::resolveDerivation() } assert(attempt); - auto pathResolved = writeDerivation(worker.store, *attempt, NoRepair, /*readOnly =*/true); + auto pathResolved = computeStorePath(worker.store, Derivation{*attempt}); auto msg = fmt("resolved derivation: '%s' -> '%s'", @@ -185,7 +165,7 @@ Goal::Co DerivationResolutionGoal::resolveDerivation() } } - co_return amDone(ecSuccess, std::nullopt); + co_return amDone(ecSuccess); } } // namespace nix diff --git a/src/libstore/build/derivation-trampoline-goal.cc b/src/libstore/build/derivation-trampoline-goal.cc index cfa0c538f952..58a43c043a2a 100644 --- a/src/libstore/build/derivation-trampoline-goal.cc +++ b/src/libstore/build/derivation-trampoline-goal.cc @@ -100,11 +100,10 @@ Goal::Co DerivationTrampolineGoal::init() if (nrFailed != 0) { co_return doneFailure( ecFailed, - BuildResult::Failure{ + BuildResult::Failure{{ .status = BuildResult::Failure::DependencyFailed, - .errorMsg = fmt("failed to obtain derivation of '%s'", drvReq->to_string(worker.store)), - }, - Error("failed to obtain derivation of '%s'", drvReq->to_string(worker.store))); + .msg = HintFmt("failed to obtain derivation of '%s'", drvReq->to_string(worker.store)), + }}); } StorePath drvPath = resolveDerivedPath(worker.store, *drvReq); @@ -152,7 +151,7 @@ Goal::Co DerivationTrampolineGoal::haveDerivation(StorePath drvPath, Derivation for (auto & output : resolvedWantedOutputs) { auto g = upcast_goal(worker.makeDerivationGoal(drvPath, drv, output, buildMode, false)); - g->preserveException = true; + g->preserveFailure = true; /* We will finish with it ourselves, as if we were the derivational goal. */ concreteDrvGoals.insert(std::move(g)); } @@ -170,7 +169,7 @@ Goal::Co DerivationTrampolineGoal::haveDerivation(StorePath drvPath, Derivation for (auto && [x, y] : successP2->builtOutputs) successP->builtOutputs.insert_or_assign(x, y); - co_return amDone(g->exitCode, g->ex); + co_return amDone(g->exitCode); } } // namespace nix diff --git a/src/libstore/build/drv-output-substitution-goal.cc b/src/libstore/build/drv-output-substitution-goal.cc index 8d0a307bedaa..71f0ae1e7db4 100644 --- a/src/libstore/build/drv-output-substitution-goal.cc +++ b/src/libstore/build/drv-output-substitution-goal.cc @@ -3,8 +3,6 @@ #include "nix/store/build/worker.hh" #include "nix/store/build/substitution-goal.hh" #include "nix/util/callback.hh" -#include "nix/store/store-open.hh" -#include "nix/store/globals.hh" namespace nix { @@ -21,11 +19,11 @@ Goal::Co DrvOutputSubstitutionGoal::init() trace("init"); /* If the derivation already exists, we’re done */ - if (worker.store.queryRealisation(id)) { + if ((outputInfo = worker.store.queryRealisation(id))) { co_return amDone(ecSuccess); } - auto subs = settings.useSubstitutes ? getDefaultSubstituters() : std::list>(); + auto subs = worker.getSubstituters(); bool substituterFailed = false; @@ -66,16 +64,19 @@ Goal::Co DrvOutputSubstitutionGoal::init() true, false); - co_await Suspend{}; + while (true) { + auto event = co_await WaitForChildEvent{}; + if (std::get_if(&event)) { + // Doesn't process child output + } else if (std::get_if(&event)) { + break; + } else if (std::get_if(&event)) { + unreachable(); + } + } worker.childTerminated(this); - /* - * The realisation corresponding to the given output id. - * Will be filled once we can get it. - */ - std::shared_ptr outputInfo; - try { outputInfo = promise->get_future().get(); } catch (std::exception & e) { @@ -86,45 +87,6 @@ Goal::Co DrvOutputSubstitutionGoal::init() if (!outputInfo) continue; - bool failed = false; - - Goals waitees; - - for (const auto & [depId, depPath] : outputInfo->dependentRealisations) { - if (depId != id) { - if (auto localOutputInfo = worker.store.queryRealisation(depId); - localOutputInfo && localOutputInfo->outPath != depPath) { - warn( - "substituter '%s' has an incompatible realisation for '%s', ignoring.\n" - "Local: %s\n" - "Remote: %s", - sub->config.getHumanReadableURI(), - depId.to_string(), - worker.store.printStorePath(localOutputInfo->outPath), - worker.store.printStorePath(depPath)); - failed = true; - break; - } - waitees.insert(worker.makeDrvOutputSubstitutionGoal(depId)); - } - } - - if (failed) - continue; - - waitees.insert(worker.makePathSubstitutionGoal(outputInfo->outPath)); - - co_await await(std::move(waitees)); - - trace("output path substituted"); - - if (nrFailed > 0) { - debug("The output path of the derivation output '%s' could not be substituted", id.to_string()); - co_return amDone(nrNoSubstituters > 0 ? ecNoSubstituters : ecFailed); - } - - worker.store.registerDrvOutput({*outputInfo, id}); - trace("finished"); co_return amDone(ecSuccess); } @@ -149,9 +111,4 @@ std::string DrvOutputSubstitutionGoal::key() return "a$" + std::string(id.to_string()); } -void DrvOutputSubstitutionGoal::handleEOF(Descriptor fd) -{ - worker.wakeUp(shared_from_this()); -} - } // namespace nix diff --git a/src/libstore/build/entry-points.cc b/src/libstore/build/entry-points.cc index 4bbd4c8f0591..50d84969a373 100644 --- a/src/libstore/build/entry-points.cc +++ b/src/libstore/build/entry-points.cc @@ -18,13 +18,13 @@ void Store::buildPaths(const std::vector & reqs, BuildMode buildMod worker.run(goals); StringSet failed; - std::optional ex; + BuildResult::Failure * failure = nullptr; for (auto & i : goals) { - if (i->ex) { - if (ex) - logError(i->ex->info()); + if (auto * f = i->buildResult.tryGetFailure()) { + if (failure) + logError(f->info()); else - ex = std::move(i->ex); + failure = f; } if (i->exitCode != Goal::ecSuccess) { if (auto i2 = dynamic_cast(i.get())) @@ -34,13 +34,14 @@ void Store::buildPaths(const std::vector & reqs, BuildMode buildMod } } - if (failed.size() == 1 && ex) { - ex->withExitStatus(worker.failingExitStatus()); - throw std::move(*ex); + if (failed.size() == 1 && failure) { + failure->withExitStatus(worker.exitStatusFlags.failingExitStatus()); + throw *failure; } else if (!failed.empty()) { - if (ex) - logError(ex->info()); - throw Error(worker.failingExitStatus(), "build of %s failed", concatStringsSep(", ", quoteStrings(failed))); + auto exitStatus = worker.exitStatusFlags.failingExitStatus(); + if (failure) + logError(failure->info()); + throw Error(exitStatus, "build of %s failed", concatStringsSep(", ", quoteStrings(failed))); } } @@ -63,12 +64,13 @@ std::vector Store::buildPathsWithResults( std::vector results; results.reserve(state.size()); - for (auto & [req, goalPtr] : state) + for (auto & [req, goalPtr] : state) { results.emplace_back( KeyedBuildResult{ goalPtr->buildResult, /* .path = */ req, }); + } return results; } @@ -82,10 +84,11 @@ BuildResult Store::buildDerivation(const StorePath & drvPath, const BasicDerivat worker.run(Goals{goal}); return goal->buildResult; } catch (Error & e) { - return BuildResult{.inner{BuildResult::Failure{ - .status = BuildResult::Failure::MiscFailure, - .errorMsg = e.msg(), - }}}; + return BuildResult{ + .inner = BuildResult::Failure{{ + .status = BuildResult::Failure::MiscFailure, + .msg = e.msg(), + }}}; }; } @@ -102,12 +105,9 @@ void Store::ensurePath(const StorePath & path) worker.run(goals); if (goal->exitCode != Goal::ecSuccess) { - if (goal->ex) { - goal->ex->withExitStatus(worker.failingExitStatus()); - throw std::move(*goal->ex); - } else - throw Error( - worker.failingExitStatus(), "path '%s' does not exist and cannot be created", printStorePath(path)); + auto exitStatus = worker.exitStatusFlags.failingExitStatus(); + goal->buildResult.tryThrowBuildError(exitStatus); + throw Error(exitStatus, "path '%s' does not exist and cannot be created", printStorePath(path)); } } @@ -134,7 +134,7 @@ void Store::repairPath(const StorePath & path) bmRepair)); worker.run(goals); } else - throw Error(worker.failingExitStatus(), "cannot repair path '%s'", printStorePath(path)); + throw Error(worker.exitStatusFlags.failingExitStatus(), "cannot repair path '%s'", printStorePath(path)); } } diff --git a/src/libstore/build/goal.cc b/src/libstore/build/goal.cc index 1c5c6bfe48c8..ef3501b00916 100644 --- a/src/libstore/build/goal.cc +++ b/src/libstore/build/goal.cc @@ -1,11 +1,61 @@ #include "nix/store/build/goal.hh" #include "nix/store/build/worker.hh" -#include "nix/store/globals.hh" +#include "nix/store/worker-settings.hh" namespace nix { +TimedOut::TimedOut(time_t maxDuration) + : CloneableError(BuildResult::Failure::TimedOut, "timed out after %1% seconds", maxDuration) + , maxDuration(maxDuration) +{ +} + using Co = nix::Goal::Co; using promise_type = nix::Goal::promise_type; +using ChildEvents = decltype(promise_type::childEvents); + +void ChildEvents::pushChildEvent(ChildOutput event) +{ + if (childTimeout) + return; // Already timed out, ignore + childOutputs.push(std::move(event)); +} + +void ChildEvents::pushChildEvent(ChildEOF event) +{ + if (childTimeout) + return; // Already timed out, ignore + assert(!childEOF); + childEOF = std::move(event); +} + +void ChildEvents::pushChildEvent(TimedOut event) +{ + // Timeout is immediate - flush pending events + childOutputs = {}; + childEOF.reset(); + childTimeout = std::move(event); +} + +bool ChildEvents::hasChildEvent() const +{ + return !childOutputs.empty() || childEOF || childTimeout; +} + +Goal::ChildEvent ChildEvents::popChildEvent() +{ + if (!childOutputs.empty()) { + auto event = std::move(childOutputs.front()); + childOutputs.pop(); + return event; + } + if (childEOF) + return *std::exchange(childEOF, std::nullopt); + if (childTimeout) + return *std::exchange(childTimeout, std::nullopt); + unreachable(); +} + using handle_type = nix::Goal::handle_type; using Suspend = nix::Goal::Suspend; @@ -139,14 +189,14 @@ Goal::Done Goal::doneSuccess(BuildResult::Success success) return amDone(ecSuccess); } -Goal::Done Goal::doneFailure(ExitCode result, BuildResult::Failure failure, std::optional ex) +Goal::Done Goal::doneFailure(ExitCode result, BuildResult::Failure failure) { assert(result == ecFailed || result == ecNoSubstituters); buildResult.inner = std::move(failure); - return amDone(result, std::move(ex)); + return amDone(result); } -Goal::Done Goal::amDone(ExitCode result, std::optional ex) +Goal::Done Goal::amDone(ExitCode result) { trace("done"); assert(top_co); @@ -154,11 +204,15 @@ Goal::Done Goal::amDone(ExitCode result, std::optional ex) assert(result == ecSuccess || result == ecFailed || result == ecNoSubstituters); exitCode = result; - if (ex) { - if (!preserveException && !waiters.empty()) - logError(ex->info()); - else - this->ex = std::move(*ex); + // Log the failure if we have one and shouldn't preserve it. + // Only log for actual failures (ecFailed), not for ecNoSubstituters + // which indicates "couldn't substitute, will try building" - that's + // expected behavior, not an error. + if (result == ecFailed) { + if (auto * failure = buildResult.tryGetFailure()) { + if (!preserveFailure && !waiters.empty()) + logError(failure->info()); + } } for (auto & i : waiters) { @@ -178,11 +232,11 @@ Goal::Done Goal::amDone(ExitCode result, std::optional ex) if (goal->waitees.empty()) { worker.wakeUp(goal); - } else if (result == ecFailed && !settings.keepGoing) { + } else if (result == ecFailed && !worker.settings.keepGoing) { /* If we failed and keepGoing is not set, we remove all remaining waitees. */ for (auto & g : goal->waitees) { - g->waiters.extract(goal); + g->waiters.erase(goal); } goal->waitees.clear(); @@ -219,6 +273,27 @@ void Goal::work() assert(top_co || exitCode != ecBusy); } +void Goal::handleChildOutput(Descriptor fd, std::string_view data) +{ + assert(top_co); + top_co->handle.promise().childEvents.pushChildEvent(ChildOutput{fd, std::string{data}}); + worker.wakeUp(shared_from_this()); +} + +void Goal::handleEOF(Descriptor fd) +{ + assert(top_co); + top_co->handle.promise().childEvents.pushChildEvent(ChildEOF{fd}); + worker.wakeUp(shared_from_this()); +} + +void Goal::timedOut(TimedOut && ex) +{ + assert(top_co); + top_co->handle.promise().childEvents.pushChildEvent(std::move(ex)); + worker.wakeUp(shared_from_this()); +} + Goal::Co Goal::yield() { worker.wakeUp(shared_from_this()); diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc index 48fbe2c98c52..d1279d5271a2 100644 --- a/src/libstore/build/substitution-goal.cc +++ b/src/libstore/build/substitution-goal.cc @@ -1,5 +1,4 @@ #include "nix/store/build/worker.hh" -#include "nix/store/store-open.hh" #include "nix/store/build/substitution-goal.hh" #include "nix/store/nar-info.hh" #include "nix/util/finally.hh" @@ -29,40 +28,12 @@ PathSubstitutionGoal::~PathSubstitutionGoal() cleanup(); } -Goal::Done -PathSubstitutionGoal::doneSuccess(BuildResult::Success::Status status, std::shared_ptr provenance) +Goal::Done PathSubstitutionGoal::doneFailure(ExitCode result, BuildResult::Failure failure) { - auto res = Goal::doneSuccess( - BuildResult::Success{ - .status = status, - .provenance = provenance, - }); - - logger->result( - getCurActivity(), - resBuildResult, - nlohmann::json(KeyedBuildResult(buildResult, DerivedPath::Opaque{storePath}))); - - return res; -} - -Goal::Done PathSubstitutionGoal::doneFailure(ExitCode result, BuildResult::Failure::Status status, std::string errorMsg) -{ - debug(errorMsg); - - auto res = Goal::doneFailure( - result, - BuildResult::Failure{ - .status = status, - .errorMsg = std::move(errorMsg), - }); - logger->result( - getCurActivity(), - resBuildResult, - nlohmann::json(KeyedBuildResult(buildResult, DerivedPath::Opaque{storePath}))); + getCurActivity(), resBuildResult, nlohmann::json(KeyedBuildResult({failure}, DerivedPath::Opaque{storePath}))); - return res; + return Goal::doneFailure(result, std::move(failure)); } Goal::Co PathSubstitutionGoal::init() @@ -73,14 +44,14 @@ Goal::Co PathSubstitutionGoal::init() /* If the path already exists we're done. */ if (!repair && worker.store.isValidPath(storePath)) { - co_return doneSuccess(BuildResult::Success::AlreadyValid, nullptr); + co_return doneSuccess(BuildResult::Success{.status = BuildResult::Success::AlreadyValid}); } - if (settings.readOnlyMode) + if (worker.store.config.getReadOnly()) throw Error( "cannot substitute path '%s' - no write access to the Nix store", worker.store.printStorePath(storePath)); - auto subs = settings.useSubstitutes ? getDefaultSubstituters() : std::list>(); + auto subs = worker.getSubstituters(); bool substituterFailed = false; std::optional lastStoresException = std::nullopt; @@ -184,7 +155,7 @@ Goal::Co PathSubstitutionGoal::init() worker.updateProgress(); } if (lastStoresException.has_value()) { - if (!settings.tryFallback) { + if (!worker.settings.tryFallback) { throw *lastStoresException; } else logError(lastStoresException->info()); @@ -195,9 +166,12 @@ Goal::Co PathSubstitutionGoal::init() build. */ co_return doneFailure( substituterFailed ? ecFailed : ecNoSubstituters, - BuildResult::Failure::NoSubstituters, - fmt("path '%s' is required, but there is no substituter that can build it", - worker.store.printStorePath(storePath))); + BuildResult::Failure{{ + .status = BuildResult::Failure::NoSubstituters, + .msg = HintFmt( + "path '%s' is required, but there is no substituter that can build it", + worker.store.printStorePath(storePath)), + }}); } Goal::Co PathSubstitutionGoal::tryToRun( @@ -208,8 +182,11 @@ Goal::Co PathSubstitutionGoal::tryToRun( if (nrFailed > 0) { co_return doneFailure( nrNoSubstituters > 0 ? ecNoSubstituters : ecFailed, - BuildResult::Failure::DependencyFailed, - fmt("some references of path '%s' could not be realised", worker.store.printStorePath(storePath))); + BuildResult::Failure{{ + .status = BuildResult::Failure::DependencyFailed, + .msg = HintFmt( + "some references of path '%s' could not be realised", worker.store.printStorePath(storePath)), + }}); } for (auto & i : info->references) @@ -230,7 +207,7 @@ Goal::Co PathSubstitutionGoal::tryToRun( /* Make sure that we are allowed to start a substitution. Note that even if maxSubstitutionJobs == 0, we still allow a substituter to run. This prevents infinite waiting. */ - while (worker.getNrSubstitutions() >= std::max(1U, (unsigned int) settings.maxSubstitutionJobs)) { + while (worker.getNrSubstitutions() >= std::max(1U, (unsigned int) worker.settings.maxSubstitutionJobs)) { co_await waitForBuildSlot(); } @@ -277,7 +254,16 @@ Goal::Co PathSubstitutionGoal::tryToRun( true, false); - co_await Suspend{}; + while (true) { + auto event = co_await WaitForChildEvent{}; + if (std::get_if(&event)) { + // Substitution doesn't process child output + } else if (std::get_if(&event)) { + break; + } else if (std::get_if(&event)) { + unreachable(); // Substitution doesn't use timeouts + } + } trace("substitute finished"); @@ -330,12 +316,12 @@ Goal::Co PathSubstitutionGoal::tryToRun( worker.updateProgress(); - co_return doneSuccess(BuildResult::Success::Substituted, provenance); -} + auto success = BuildResult::Success{.status = BuildResult::Success::Substituted, .provenance = provenance}; -void PathSubstitutionGoal::handleEOF(Descriptor fd) -{ - worker.wakeUp(shared_from_this()); + logger->result( + getCurActivity(), resBuildResult, nlohmann::json(KeyedBuildResult({success}, DerivedPath::Opaque{storePath}))); + + co_return doneSuccess(std::move(success)); } void PathSubstitutionGoal::cleanup() @@ -344,7 +330,7 @@ void PathSubstitutionGoal::cleanup() if (thr.joinable()) { // FIXME: signal worker thread to quit. thr.join(); - worker.childTerminated(this); + worker.childTerminated(this, JobCategory::Substitution); } outPipe.close(); diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 3663a2c919f1..ffb1027b42c5 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -1,5 +1,6 @@ #include "nix/store/local-store.hh" #include "nix/store/machines.hh" +#include "nix/store/store-open.hh" #include "nix/store/build/worker.hh" #include "nix/store/build/substitution-goal.hh" #include "nix/store/build/drv-output-substitution-goal.hh" @@ -19,16 +20,23 @@ Worker::Worker(Store & store, Store & evalStore) : act(*logger, actRealise) , actDerivations(*logger, actBuilds) , actSubstitutions(*logger, actCopyPaths) +#ifdef _WIN32 + , ioport{CreateIoCompletionPort(INVALID_HANDLE_VALUE, NULL, 0, 0)} +#endif , store(store) , evalStore(evalStore) + , settings(nix::settings.getWorkerSettings()) + , getSubstituters{[] { + return nix::settings.getWorkerSettings().useSubstitutes ? getDefaultSubstituters() : std::list>{}; + }} { +#ifdef _WIN32 + if (!ioport) + throw windows::WinError("CreateIoCompletionPort"); +#endif nrLocalBuilds = 0; nrSubstitutions = 0; lastWokenUp = steady_time_point::min(); - permanentFailure = false; - timedOut = false; - hashMismatch = false; - checkMismatch = false; } Worker::~Worker() @@ -160,7 +168,9 @@ template static bool removeGoal(std::shared_ptr goal, typename DerivedPathMap>>::ChildNode & node) { - return removeGoal(goal, node.value) || removeGoal(goal, node.childMap); + bool valueKeep = removeGoal(goal, node.value); + bool childMapKeep = removeGoal(goal, node.childMap); + return valueKeep || childMapKeep; } void Worker::removeGoal(GoalPtr goal) @@ -243,13 +253,18 @@ void Worker::childStarted( } void Worker::childTerminated(Goal * goal, bool wakeSleepers) +{ + childTerminated(goal, goal->jobCategory(), wakeSleepers); +} + +void Worker::childTerminated(Goal * goal, JobCategory jobCategory, bool wakeSleepers) { auto i = std::find_if(children.begin(), children.end(), [&](const Child & child) { return child.goal2 == goal; }); if (i == children.end()) return; if (i->inBuildSlot) { - switch (goal->jobCategory()) { + switch (jobCategory) { case JobCategory::Substitution: assert(nrSubstitutions > 0); nrSubstitutions--; @@ -359,7 +374,7 @@ void Worker::run(const Goals & _topGoals) if (!children.empty() || !waitingForAWhile.empty()) waitForInput(); else if (awake.empty() && 0U == settings.maxBuildJobs) { - if (getMachines().empty()) + if (Machine::parseConfig({nix::settings.thisSystem}, nix::settings.getWorkerSettings().builders).empty()) throw Error( "Unable to start any build; either increase '--max-jobs' or enable remote builds.\n" "\n" @@ -399,8 +414,12 @@ void Worker::waitForInput() is a build timeout, then wait for input until the first deadline for any child. */ auto nearest = steady_time_point::max(); // nearest deadline - if (settings.minFree.get() != 0) - // Periodicallty wake up to see if we need to run the garbage collector. + + auto localStore = dynamic_cast(&store); + if (localStore && localStore->config->getLocalSettings().getGCSettings().minFree.get() != 0) + // If we have a local store (and thus are capable of automatically collecting garbage) and configured to do so, + // periodically wake up to see if we need to run the garbage collector. (See the `autoGC` call site above in + // this file, also gated on having a local store. when we wake up, we intended to reach that call site.) nearest = before + std::chrono::seconds(10); for (auto & i : children) { if (!i.respectTimeouts) @@ -479,14 +498,13 @@ void Worker::waitForInput() if (goal->exitCode == Goal::ecBusy && 0 != settings.maxSilentTime && j->respectTimeouts && after - j->lastOutput >= std::chrono::seconds(settings.maxSilentTime)) { - goal->timedOut( - Error("%1% timed out after %2% seconds of silence", goal->getName(), settings.maxSilentTime)); + goal->timedOut(TimedOut(settings.maxSilentTime)); } else if ( goal->exitCode == Goal::ecBusy && 0 != settings.buildTimeout && j->respectTimeouts && after - j->timeStarted >= std::chrono::seconds(settings.buildTimeout)) { - goal->timedOut(Error("%1% timed out after %2% seconds", goal->getName(), settings.buildTimeout)); + goal->timedOut(TimedOut(settings.buildTimeout)); } } @@ -501,26 +519,6 @@ void Worker::waitForInput() } } -unsigned int Worker::failingExitStatus() -{ - // See API docs in header for explanation - unsigned int mask = 0; - bool buildFailure = permanentFailure || timedOut || hashMismatch; - if (buildFailure) - mask |= 0x04; // 100 - if (timedOut) - mask |= 0x01; // 101 - if (hashMismatch) - mask |= 0x02; // 102 - if (checkMismatch) { - mask |= 0x08; // 104 - } - - if (mask) - mask |= 0x60; - return mask ? mask : 1; -} - bool Worker::pathContentsGood(const StorePath & path) { auto i = pathContentsGoodCache.find(path); diff --git a/src/libstore/builtins/buildenv.cc b/src/libstore/builtins/buildenv.cc index 4db37d43a793..3dd66be2ccae 100644 --- a/src/libstore/builtins/buildenv.cc +++ b/src/libstore/builtins/buildenv.cc @@ -20,22 +20,23 @@ namespace { struct State { - std::map priorities; + std::map priorities; unsigned long symlinks = 0; }; } // namespace /* For each activated package, create symlinks */ -static void createLinks(State & state, const Path & srcDir, const Path & dstDir, int priority) +static void +createLinks(State & state, const std::filesystem::path & srcDir, const std::filesystem::path & dstDir, int priority) { DirectoryIterator srcFiles; try { srcFiles = DirectoryIterator{srcDir}; - } catch (SysError & e) { - if (e.errNo == ENOTDIR) { - warn("not including '%s' in the user environment because it's not a directory", srcDir); + } catch (SystemError & e) { + if (e.is(std::errc::not_a_directory)) { + warn("not including %s in the user environment because it's not a directory", PathFmt(srcDir)); return; } throw; @@ -50,17 +51,12 @@ static void createLinks(State & state, const Path & srcDir, const Path & dstDir, auto srcFile = (std::filesystem::path{srcDir} / name).string(); auto dstFile = (std::filesystem::path{dstDir} / name).string(); - struct stat srcSt; - try { - if (stat(srcFile.c_str(), &srcSt) == -1) - throw SysError("getting status of '%1%'", srcFile); - } catch (SysError & e) { - if (e.errNo == ENOENT || e.errNo == ENOTDIR) { - warn("skipping dangling symlink '%s'", dstFile); - continue; - } - throw; + auto srcStOpt = maybeStat(srcFile.c_str()); + if (!srcStOpt) { + warn("skipping dangling symlink '%s'", dstFile); + continue; } + auto & srcSt = *srcStOpt; /* The files below are special-cased to that they don't show * up in user profiles, either because they are useless, or @@ -83,9 +79,8 @@ static void createLinks(State & state, const Path & srcDir, const Path & dstDir, } else if (S_ISLNK(dstSt.st_mode)) { auto target = canonPath(dstFile, true); if (!S_ISDIR(lstat(target).st_mode)) - throw Error("collision between '%1%' and non-directory '%2%'", srcFile, target); - if (unlink(dstFile.c_str()) == -1) - throw SysError("unlinking '%1%'", dstFile); + throw Error("collision between %1% and non-directory %2%", PathFmt(srcFile), PathFmt(target)); + unlink(dstFile); if (mkdir( dstFile.c_str() #ifndef _WIN32 // TODO abstract mkdir perms for Windows @@ -112,8 +107,7 @@ static void createLinks(State & state, const Path & srcDir, const Path & dstDir, throw BuildEnvFileConflictError(readLink(dstFile), srcFile, priority); if (prevPriority < priority) continue; - if (unlink(dstFile.c_str()) == -1) - throw SysError("unlinking '%1%'", dstFile); + unlink(dstFile); } else if (S_ISDIR(dstSt.st_mode)) throw Error("collision between non-directory '%1%' and directory '%2%'", srcFile, dstFile); } @@ -125,24 +119,24 @@ static void createLinks(State & state, const Path & srcDir, const Path & dstDir, } } -void buildProfile(const Path & out, Packages && pkgs) +void buildProfile(const std::filesystem::path & out, Packages && pkgs) { State state; - PathSet done, postponed; + std::set done, postponed; - auto addPkg = [&](const Path & pkgDir, int priority) { + auto addPkg = [&](const std::filesystem::path & pkgDir, int priority) { if (!done.insert(pkgDir).second) return; createLinks(state, pkgDir, out, priority); try { for (const auto & p : tokenizeString>( - readFile(pkgDir + "/nix-support/propagated-user-env-packages"), " \n")) + readFile(pkgDir / "nix-support" / "propagated-user-env-packages"), " \n")) if (!done.count(p)) postponed.insert(p); - } catch (SysError & e) { - if (e.errNo != ENOENT && e.errNo != ENOTDIR) + } catch (SystemError & e) { + if (!e.is(std::errc::no_such_file_or_directory) && !e.is(std::errc::not_a_directory)) throw; } }; @@ -165,7 +159,7 @@ void buildProfile(const Path & out, Packages && pkgs) */ auto priorityCounter = 1000; while (!postponed.empty()) { - PathSet pkgDirs; + std::set pkgDirs; postponed.swap(pkgDirs); for (const auto & pkgDir : pkgDirs) addPkg(pkgDir, priorityCounter++); diff --git a/src/libstore/builtins/fetchurl.cc b/src/libstore/builtins/fetchurl.cc index 126fb922eba8..1ff66eb89fa4 100644 --- a/src/libstore/builtins/fetchurl.cc +++ b/src/libstore/builtins/fetchurl.cc @@ -4,6 +4,7 @@ #include "nix/store/globals.hh" #include "nix/util/archive.hh" #include "nix/util/compression.hh" +#include "nix/util/file-system.hh" namespace nix { @@ -13,12 +14,12 @@ static void builtinFetchurl(const BuiltinBuilderContext & ctx) this to be stored in a file. It would be nice if we could just pass a pointer to the data. */ if (ctx.netrcData != "") { - settings.netrcFile = "netrc"; - writeFile(settings.netrcFile, ctx.netrcData, 0600); + fileTransferSettings.netrcFile = "netrc"; + writeFile(fileTransferSettings.netrcFile.get(), ctx.netrcData, 0600); } - settings.caFile = "ca-certificates.crt"; - writeFile(settings.caFile, ctx.caFileData, 0600); + fileTransferSettings.caFile = "ca-certificates.crt"; + writeFile(*fileTransferSettings.caFile.get(), ctx.caFileData, 0600); auto out = get(ctx.drv.outputs, "out"); if (!out) @@ -65,15 +66,14 @@ static void builtinFetchurl(const BuiltinBuilderContext & ctx) auto executable = ctx.drv.env.find("executable"); if (executable != ctx.drv.env.end() && executable->second == "1") { - if (chmod(storePath.c_str(), 0755) == -1) - throw SysError("making '%1%' executable", storePath); + chmod(storePath, 0755); } }; /* Try the hashed mirrors first. */ auto dof = std::get_if(&out->raw); if (dof && dof->ca.method.getFileIngestionMethod() == FileIngestionMethod::Flat) - for (auto hashedMirror : settings.hashedMirrors.get()) + for (auto hashedMirror : ctx.hashedMirrors) try { if (!hasSuffix(hashedMirror, "/")) hashedMirror += '/'; diff --git a/src/libstore/builtins/unpack-channel.cc b/src/libstore/builtins/unpack-channel.cc index 317cbe9ef1fb..a3cc16bc71f3 100644 --- a/src/libstore/builtins/unpack-channel.cc +++ b/src/libstore/builtins/unpack-channel.cc @@ -27,6 +27,8 @@ static void builtinUnpackChannel(const BuiltinBuilderContext & ctx) size_t fileCount; std::string fileName; auto entries = DirectoryIterator{out}; + if (entries == DirectoryIterator{}) + throw Error("channel tarball '%s' is empty", src); fileName = entries->path().string(); fileCount = std::distance(entries.begin(), entries.end()); @@ -36,8 +38,8 @@ static void builtinUnpackChannel(const BuiltinBuilderContext & ctx) auto target = out / channelName; try { std::filesystem::rename(fileName, target); - } catch (std::filesystem::filesystem_error &) { - throw SysError("failed to rename %1% to %2%", fileName, target.string()); + } catch (std::filesystem::filesystem_error & e) { + throw SystemError(e.code(), "failed to rename %1% to %2%", fileName, target.string()); } } diff --git a/src/libstore/common-protocol.cc b/src/libstore/common-protocol.cc index 3db3c419fb21..6e3a1921dc04 100644 --- a/src/libstore/common-protocol.cc +++ b/src/libstore/common-protocol.cc @@ -6,6 +6,7 @@ #include "nix/store/common-protocol-impl.hh" #include "nix/util/archive.hh" #include "nix/store/derivations.hh" +#include "nix/util/signature/local-keys.hh" #include @@ -101,4 +102,71 @@ void CommonProto::Serialise>::write( conn.to << (caOpt ? renderContentAddress(*caOpt) : ""); } +Signature CommonProto::Serialise::read(const StoreDirConfig & store, CommonProto::ReadConn conn) +{ + return Signature::parse(readString(conn.from)); +} + +void CommonProto::Serialise::write( + const StoreDirConfig & store, CommonProto::WriteConn conn, const Signature & sig) +{ + conn.to << sig.to_string(); +} + +/** + * Mapping from protocol wire values to BuildResultStatus. + * + * The array index is the wire value. + * Note: HashMismatch is not in the protocol; it gets converted + * to OutputRejected before serialization. + */ +constexpr static BuildResultStatus buildResultStatusTable[] = { + BuildResultSuccessStatus::Built, // 0 + BuildResultSuccessStatus::Substituted, // 1 + BuildResultSuccessStatus::AlreadyValid, // 2 + BuildResultFailureStatus::PermanentFailure, // 3 + BuildResultFailureStatus::InputRejected, // 4 + BuildResultFailureStatus::OutputRejected, // 5 + BuildResultFailureStatus::TransientFailure, // 6 + BuildResultFailureStatus::CachedFailure, // 7 + BuildResultFailureStatus::TimedOut, // 8 + BuildResultFailureStatus::MiscFailure, // 9 + BuildResultFailureStatus::DependencyFailed, // 10 + BuildResultFailureStatus::LogLimitExceeded, // 11 + BuildResultFailureStatus::NotDeterministic, // 12 + BuildResultSuccessStatus::ResolvesToAlreadyValid, // 13 + BuildResultFailureStatus::NoSubstituters, // 14 + BuildResultFailureStatus::HashMismatch, // 15 + BuildResultFailureStatus::Cancelled, // 16 +}; + +BuildResultStatus +CommonProto::Serialise::read(const StoreDirConfig & store, CommonProto::ReadConn conn) +{ + auto rawStatus = readNum(conn.from); + + if (rawStatus >= std::size(buildResultStatusTable)) + throw Error("Invalid BuildResult status %d from remote", rawStatus); + + return buildResultStatusTable[rawStatus]; +} + +void CommonProto::Serialise::write( + const StoreDirConfig & store, CommonProto::WriteConn conn, const BuildResultStatus & status) +{ + /* See definition, the protocols don't know about `HashMismatch` + yet, so change it to `OutputRejected`, which they expect + for this case (hash mismatch is a type of output + rejection). */ + if (status == BuildResultStatus{BuildResultFailureStatus::HashMismatch}) { + return write(store, conn, BuildResultFailureStatus::OutputRejected); + } + for (auto && [wire, val] : enumerate(buildResultStatusTable)) + if (val == status) { + conn.to << uint8_t(wire); + return; + } + unreachable(); +} + } // namespace nix diff --git a/src/libstore/common-ssh-store-config.cc b/src/libstore/common-ssh-store-config.cc index 12f187b4c9e4..fd61b12efeee 100644 --- a/src/libstore/common-ssh-store-config.cc +++ b/src/libstore/common-ssh-store-config.cc @@ -1,17 +1,9 @@ -#include - #include "nix/store/common-ssh-store-config.hh" #include "nix/store/ssh.hh" namespace nix { -CommonSSHStoreConfig::CommonSSHStoreConfig(std::string_view scheme, std::string_view authority, const Params & params) - : CommonSSHStoreConfig(scheme, ParsedURL::Authority::parse(authority), params) -{ -} - -CommonSSHStoreConfig::CommonSSHStoreConfig( - std::string_view scheme, const ParsedURL::Authority & authority, const Params & params) +CommonSSHStoreConfig::CommonSSHStoreConfig(const ParsedURL::Authority & authority, const Params & params) : StoreConfig(params) , authority(authority) { diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index dbf2922b5f00..4332c692b2a1 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -6,9 +6,11 @@ #include "nix/store/build-result.hh" #include "nix/store/store-api.hh" #include "nix/store/store-cast.hh" +#include "nix/store/filetransfer.hh" #include "nix/store/gc-store.hh" #include "nix/store/log-store.hh" #include "nix/store/indirect-root-store.hh" +#include "nix/store/remote-store.hh" #include "nix/store/path-with-outputs.hh" #include "nix/util/finally.hh" #include "nix/util/archive.hh" @@ -134,7 +136,7 @@ struct TunnelLogger : public Logger if (!ex) to << STDERR_LAST; else { - if (GET_PROTOCOL_MINOR(clientVersion) >= 26) { + if (clientVersion >= WorkerProto::Version{.number = {1, 26}}) { to << STDERR_ERROR << *ex; } else { to << STDERR_ERROR << ex->what() << ex->info().status; @@ -150,7 +152,7 @@ struct TunnelLogger : public Logger const Fields & fields, ActivityId parent) override { - if (GET_PROTOCOL_MINOR(clientVersion) < 20) { + if (clientVersion.number < WorkerProto::Version::Number{1, 20}) { if (!s.empty()) log(lvl, s + "..."); return; @@ -163,7 +165,7 @@ struct TunnelLogger : public Logger void stopActivity(ActivityId act) override { - if (GET_PROTOCOL_MINOR(clientVersion) < 20) + if (clientVersion.number < WorkerProto::Version::Number{1, 20}) return; StringSink buf; buf << STDERR_STOP_ACTIVITY << act; @@ -172,7 +174,7 @@ struct TunnelLogger : public Logger void result(ActivityId act, ResultType type, const Fields & fields) override { - if (GET_PROTOCOL_MINOR(clientVersion) < 20) + if (clientVersion.number < WorkerProto::Version::Number{1, 20}) return; StringSink buf; buf << STDERR_RESULT << act << type << fields; @@ -234,38 +236,49 @@ struct ClientSettings void apply(TrustedFlag trusted) { settings.keepFailed = keepFailed; - settings.keepGoing = keepGoing; - settings.tryFallback = tryFallback; + settings.getWorkerSettings().keepGoing = keepGoing; + settings.getWorkerSettings().tryFallback = tryFallback; nix::verbosity = verbosity; - settings.maxBuildJobs.assign(maxBuildJobs); - settings.maxSilentTime = maxSilentTime; + settings.getWorkerSettings().maxBuildJobs.assign(maxBuildJobs); + settings.getWorkerSettings().maxSilentTime = maxSilentTime; settings.verboseBuild = verboseBuild; - settings.buildCores = buildCores; - settings.useSubstitutes = useSubstitutes; + settings.getLocalSettings().buildCores = buildCores; + settings.getWorkerSettings().useSubstitutes = useSubstitutes; for (auto & i : overrides) { auto & name(i.first); auto & value(i.second); - auto setSubstituters = [&](Setting & res) { + auto setSubstituters = [&](Setting> & res) { if (name != res.name && res.aliases.count(name) == 0) return false; - StringSet trusted = settings.trustedSubstituters; - for (auto & s : settings.substituters.get()) - trusted.insert(s); - Strings subs; + std::set trusted = settings.trustedSubstituters; + for (auto & ref : settings.getWorkerSettings().substituters.get()) + trusted.insert(ref); + std::vector subs; auto ss = tokenizeString(value); - for (auto & s : ss) - if (trusted.count(s)) - subs.push_back(s); - else if (!hasSuffix(s, "/") && trusted.count(s + "/")) - subs.push_back(s + "/"); + for (auto & s : ss) { + auto ref = StoreReference::parse(s); + auto tryTrust = [&] { + if (trusted.count(ref)) + return true; + if (auto * specified = std::get_if(&ref.variant); + specified && !hasSuffix(specified->authority, "/")) { + specified->authority += "/"; + if (trusted.count(ref)) + return true; + } + return false; + }; + if (tryTrust()) + subs.push_back(std::move(ref)); else warn( "ignoring untrusted substituter '%s', you are not a trusted user.\n" "Run `man nix.conf` for more information on the `substituters` configuration option.", s); - res = subs; + } + res = std::move(subs); return true; }; @@ -283,18 +296,20 @@ struct ClientSettings "Ignoring the client-specified plugin-files.\n" "The client specifying plugins to the daemon never made sense, and was removed in Nix >=2.14."); } else if ( - trusted || name == settings.buildTimeout.name || name == settings.maxSilentTime.name - || name == settings.pollInterval.name || name == "connect-timeout" - || (name == "builders" && value == "")) + trusted || name == settings.getWorkerSettings().buildTimeout.name + || name == settings.getWorkerSettings().maxSilentTime.name + || name == settings.getWorkerSettings().pollInterval.name || name == "connect-timeout" + || (name == "builders" && value == "")) { settings.set(name, value); - else if (setSubstituters(settings.substituters)) + fileTransferSettings.set(name, value); + } else if (setSubstituters(settings.getWorkerSettings().substituters)) ; else warn( "ignoring the client-specified setting '%s', because it is a restricted setting and you are not a trusted user", name); } catch (UsageError & e) { - warn(e.what()); + logWarning(e.info()); } } } @@ -326,7 +341,7 @@ static void performOp( auto paths = WorkerProto::Serialise::read(*store, rconn); SubstituteFlag substitute = NoSubstitute; - if (GET_PROTOCOL_MINOR(conn.protoVersion) >= 27) { + if (conn.protoVersion >= WorkerProto::Version{.number = {1, 27}}) { substitute = readInt(conn.from) ? Substitute : NoSubstitute; } @@ -340,17 +355,6 @@ static void performOp( break; } - case WorkerProto::Op::HasSubstitutes: { - auto path = WorkerProto::Serialise::read(*store, rconn); - logger->startWork(); - StorePathSet paths; // FIXME - paths.insert(path); - auto res = store->querySubstitutablePaths(paths); - logger->stopWork(); - conn.to << (res.count(path) != 0); - break; - } - case WorkerProto::Op::QuerySubstitutablePaths: { auto paths = WorkerProto::Serialise::read(*store, rconn); logger->startWork(); @@ -360,26 +364,13 @@ static void performOp( break; } - case WorkerProto::Op::QueryPathHash: { - auto path = WorkerProto::Serialise::read(*store, rconn); - logger->startWork(); - auto hash = store->queryPathInfo(path)->narHash; - logger->stopWork(); - conn.to << hash.to_string(HashFormat::Base16, false); - break; - } - - case WorkerProto::Op::QueryReferences: case WorkerProto::Op::QueryReferrers: case WorkerProto::Op::QueryValidDerivers: case WorkerProto::Op::QueryDerivationOutputs: { auto path = WorkerProto::Serialise::read(*store, rconn); logger->startWork(); StorePathSet paths; - if (op == WorkerProto::Op::QueryReferences) - for (auto & i : store->queryPathInfo(path)->references) - paths.insert(i); - else if (op == WorkerProto::Op::QueryReferrers) + if (op == WorkerProto::Op::QueryReferrers) store->queryReferrers(path, paths); else if (op == WorkerProto::Op::QueryValidDerivers) paths = store->queryValidDerivers(path); @@ -427,14 +418,14 @@ static void performOp( } case WorkerProto::Op::AddToStore: { - if (GET_PROTOCOL_MINOR(conn.protoVersion) >= 25) { + if (conn.protoVersion >= WorkerProto::Version{.number = {1, 25}}) { auto name = readString(conn.from); auto camStr = readString(conn.from); auto refs = WorkerProto::Serialise::read(*store, rconn); bool repairBool; conn.from >> repairBool; auto repair = RepairFlag{repairBool}; - auto provenance = conn.features.contains(WorkerProto::featureProvenance) + auto provenance = conn.protoVersion.features.contains(WorkerProto::featureProvenance) ? Provenance::from_json_str_optional(readString(conn.from)) : nullptr; @@ -524,7 +515,20 @@ static void performOp( logger->startWork(); { FramedSource source(conn.from); - store->addMultipleToStore(source, RepairFlag{repair}, dontCheckSigs ? NoCheckSigs : CheckSigs); + auto expected = readNum(source); + for (uint64_t i = 0; i < expected; ++i) { + auto info = WorkerProto::Serialise::read( + *store, + WorkerProto::ReadConn{ + .from = source, + .version = conn.protoVersion.features.contains(WorkerProto::featureVersionedAddToStoreMultiple) + ? conn.protoVersion + : WorkerProto::Version{.number = {.major = 1, .minor = 16}}, + }); + info.ultimate = false; + EnsureRead wrapper{source, info.narSize}; + store->addToStore(info, wrapper, RepairFlag{repair}, dontCheckSigs ? NoCheckSigs : CheckSigs); + } } logger->stopWork(); break; @@ -663,7 +667,7 @@ static void performOp( Derivation drv2; static_cast(drv2) = drv; - drvPath = writeDerivation(*store, Derivation{drv2}); + drvPath = store->writeDerivation(Derivation{drv2}); } auto res = store->buildDerivation(drvPath, drv, buildMode); @@ -696,17 +700,17 @@ static void performOp( "you are not privileged to create perm roots\n\n" "hint: you can just do this client-side without special privileges, and probably want to do that instead."); auto storePath = WorkerProto::Serialise::read(*store, rconn); - Path gcRoot = absPath(readString(conn.from)); + std::filesystem::path gcRoot = absPath(readString(conn.from)); logger->startWork(); auto & localFSStore = require(*store); localFSStore.addPermRoot(storePath, gcRoot); logger->stopWork(); - conn.to << gcRoot; + conn.to << gcRoot.string(); break; } case WorkerProto::Op::AddIndirectRoot: { - Path path = absPath(readString(conn.from)); + std::filesystem::path path = absPath(readString(conn.from)); logger->startWork(); auto & indirectRootStore = require(*store); @@ -827,7 +831,7 @@ static void performOp( case WorkerProto::Op::QuerySubstitutablePathInfos: { SubstitutablePathInfos infos; StorePathCAMap pathsMap = {}; - if (GET_PROTOCOL_MINOR(conn.protoVersion) < 22) { + if (conn.protoVersion.number < WorkerProto::Version::Number{1, 22}) { auto paths = WorkerProto::Serialise::read(*store, rconn); for (auto & path : paths) pathsMap.emplace(path, std::nullopt); @@ -893,7 +897,7 @@ static void performOp( case WorkerProto::Op::AddSignatures: { auto path = WorkerProto::Serialise::read(*store, rconn); - StringSet sigs = readStrings(conn.from); + auto sigs = WorkerProto::Serialise>::read(*store, rconn); logger->startWork(); store->addSignatures(path, sigs); logger->stopWork(); @@ -918,9 +922,9 @@ static void performOp( info.deriver = std::move(deriver); info.references = WorkerProto::Serialise::read(*store, rconn); conn.from >> info.registrationTime >> info.narSize >> info.ultimate; - info.sigs = readStrings(conn.from); + info.sigs = WorkerProto::Serialise>::read(*store, rconn); info.ca = ContentAddress::parseOpt(readString(conn.from)); - info.provenance = conn.features.contains(WorkerProto::featureProvenance) + info.provenance = conn.protoVersion.features.contains(WorkerProto::featureProvenance) ? Provenance::from_json_str_optional(readString(conn.from)) : nullptr; conn.from >> repair >> dontCheckSigs; @@ -929,7 +933,7 @@ static void performOp( if (!trusted) info.ultimate = false; - if (GET_PROTOCOL_MINOR(conn.protoVersion) >= 23) { + if (conn.protoVersion >= WorkerProto::Version{.number = {1, 23}}) { logger->startWork(); { FramedSource source(conn.from); @@ -941,7 +945,7 @@ static void performOp( else { std::unique_ptr source; StringSink saved; - if (GET_PROTOCOL_MINOR(conn.protoVersion) >= 21) + if (conn.protoVersion >= WorkerProto::Version{.number = {1, 21}}) source = std::make_unique(conn.from, conn.to); else { TeeSource tee{conn.from, saved}; @@ -975,7 +979,7 @@ static void performOp( case WorkerProto::Op::RegisterDrvOutput: { logger->startWork(); - if (GET_PROTOCOL_MINOR(conn.protoVersion) < 31) { + if (conn.protoVersion.number < WorkerProto::Version::Number{1, 31}) { auto outputId = WorkerProto::Serialise::read(*store, rconn); auto outputPath = StorePath(readString(conn.from)); store->registerDrvOutput(Realisation{{.outPath = outputPath}, outputId}); @@ -992,7 +996,7 @@ static void performOp( auto outputId = WorkerProto::Serialise::read(*store, rconn); auto info = store->queryRealisation(outputId); logger->stopWork(); - if (GET_PROTOCOL_MINOR(conn.protoVersion) < 31) { + if (conn.protoVersion.number < WorkerProto::Version::Number{1, 31}) { std::set outPaths; if (info) outPaths.insert(info->outPath); @@ -1023,10 +1027,6 @@ static void performOp( break; } - case WorkerProto::Op::QueryFailedPaths: - case WorkerProto::Op::ClearFailedPaths: - throw Error("Removed operation %1%", op); - case WorkerProto::Op::QueryActiveBuilds: { logger->startWork(); auto & activeBuildsStore = require(*store); @@ -1047,26 +1047,32 @@ void processConnection(ref store, FdSource && from, FdSink && to, Trusted auto monitor = !recursive ? std::make_unique(from.fd) : nullptr; (void) monitor; // suppress warning ReceiveInterrupts receiveInterrupts; + + /* When interrupted (e.g., SSH client disconnects), shutdown any downstream + store connections to break circular waits. This fixes deadlocks where the + daemon is waiting for a response from a downstream store while the downstream + is waiting for more data from this daemon. */ + auto shutdownStoreOnInterrupt = createInterruptCallback([&store]() { + if (auto remoteStore = dynamic_cast(&*store)) { + remoteStore->shutdownConnections(); + } + }); #endif /* Exchange the greeting. */ - auto myFeatures = WorkerProto::allFeatures; + WorkerProto::BasicServerConnection conn; + auto version = WorkerProto::latest; if (!experimentalFeatureSettings.isEnabled(Xp::Provenance)) - myFeatures.erase(std::string(WorkerProto::featureProvenance)); + version.features.erase(std::string(WorkerProto::featureProvenance)); + conn.protoVersion = WorkerProto::BasicServerConnection::handshake(to, from, version); - auto [protoVersion, features] = - WorkerProto::BasicServerConnection::handshake(to, from, PROTOCOL_VERSION, myFeatures); - - if (protoVersion < MINIMUM_PROTOCOL_VERSION) + if (conn.protoVersion.number < WorkerProto::minimum.number) throw Error("the Nix client version is too old"); - WorkerProto::BasicServerConnection conn; conn.to = std::move(to); conn.from = std::move(from); - conn.protoVersion = protoVersion; - conn.features = features; - auto tunnelLogger_ = std::make_unique(conn.to, protoVersion); + auto tunnelLogger_ = std::make_unique(conn.to, conn.protoVersion); auto tunnelLogger = tunnelLogger_.get(); std::unique_ptr prevLogger_; auto prevLogger = logger.get(); diff --git a/src/libstore/derivation-options.cc b/src/libstore/derivation-options.cc index 5403db288ac6..5208440c7d94 100644 --- a/src/libstore/derivation-options.cc +++ b/src/libstore/derivation-options.cc @@ -6,7 +6,6 @@ #include "nix/store/store-api.hh" #include "nix/util/types.hh" #include "nix/util/util.hh" -#include "nix/store/globals.hh" #include "nix/util/variant-wrapper.hh" #include @@ -21,76 +20,53 @@ static std::optional getStringAttr(const StringMap & env, const StructuredAttrs * parsed, const std::string & name) { if (parsed) { - auto i = parsed->structuredAttrs.find(name); - if (i == parsed->structuredAttrs.end()) - return {}; - else { - if (!i->second.is_string()) - throw Error("attribute '%s' of must be a string", name); - return i->second.get(); - } + if (auto * i = get(parsed->structuredAttrs, name)) + try { + return getString(*i); + } catch (Error & e) { + e.addTrace({}, "while parsing attribute \"%s\"", name); + throw; + } } else { - auto i = env.find(name); - if (i == env.end()) - return {}; - else - return i->second; + if (auto * i = get(env, name)) + return *i; } + return {}; } static bool getBoolAttr(const StringMap & env, const StructuredAttrs * parsed, const std::string & name, bool def) { if (parsed) { - auto i = parsed->structuredAttrs.find(name); - if (i == parsed->structuredAttrs.end()) - return def; - else { - if (!i->second.is_boolean()) - throw Error("attribute '%s' must be a Boolean", name); - return i->second.get(); - } + if (auto * i = get(parsed->structuredAttrs, name)) + try { + return getBoolean(*i); + } catch (Error & e) { + e.addTrace({}, "while parsing attribute \"%s\"", name); + throw; + } } else { - auto i = env.find(name); - if (i == env.end()) - return def; - else - return i->second == "1"; + if (auto * i = get(env, name)) + return *i == "1"; } + return def; } -static std::optional -getStringsAttr(const StringMap & env, const StructuredAttrs * parsed, const std::string & name) +static std::optional +getStringSetAttr(const StringMap & env, const StructuredAttrs * parsed, const std::string & name) { if (parsed) { - auto i = parsed->structuredAttrs.find(name); - if (i == parsed->structuredAttrs.end()) - return {}; - else { - if (!i->second.is_array()) - throw Error("attribute '%s' must be a list of strings", name); - auto & a = getArray(i->second); - Strings res; - for (auto j = a.begin(); j != a.end(); ++j) { - if (!j->is_string()) - throw Error("attribute '%s' must be a list of strings", name); - res.push_back(j->get()); + if (auto * i = get(parsed->structuredAttrs, name)) + try { + return getStringSet(*i); + } catch (Error & e) { + e.addTrace({}, "while parsing attribute \"%s\"", name); + throw; } - return res; - } } else { - auto i = env.find(name); - if (i == env.end()) - return {}; - else - return tokenizeString(i->second); + if (auto * i = get(env, name)) + return tokenizeString(*i); } -} - -static std::optional -getStringSetAttr(const StringMap & env, const StructuredAttrs * parsed, const std::string & name) -{ - auto ss = getStringsAttr(env, parsed, name); - return ss ? (std::optional{StringSet{ss->begin(), ss->end()}}) : (std::optional{}); + return {}; } template @@ -233,26 +209,21 @@ DerivationOptions derivationOptionsFromStructuredAttrs( std::map> res; if (auto * outputChecks = get(structuredAttrs, "outputChecks")) { for (auto & [outputName, output_] : getObject(*outputChecks)) { - OutputChecks checks; - auto & output = getObject(output_); - if (auto maxSize = get(output, "maxSize")) - checks.maxSize = maxSize->get(); - - if (auto maxClosureSize = get(output, "maxClosureSize")) - checks.maxClosureSize = maxClosureSize->get(); - auto get_ = [&](const std::string & name) -> std::optional>> { - if (auto i = get(output, name)) { - std::set> res; - for (auto j = i->begin(); j != i->end(); ++j) { - if (!j->is_string()) - throw Error("attribute '%s' must be a list of strings", name); - res.insert(parseRef(j->get())); + if (auto * i = get(output, name)) { + try { + std::set> res; + for (auto & s : getStringList(*i)) + res.insert(parseRef(s)); + return res; + } catch (Error & e) { + e.addTrace( + {}, "while parsing attribute 'outputChecks.\"%s\".%s'", outputName, name); + throw; } - return res; } return {}; }; @@ -260,18 +231,8 @@ DerivationOptions derivationOptionsFromStructuredAttrs( res.insert_or_assign( outputName, OutputChecks{ - .maxSize = [&]() -> std::optional { - if (auto maxSize = get(output, "maxSize")) - return maxSize->get(); - else - return std::nullopt; - }(), - .maxClosureSize = [&]() -> std::optional { - if (auto maxClosureSize = get(output, "maxClosureSize")) - return maxClosureSize->get(); - else - return std::nullopt; - }(), + .maxSize = ptrToOwned(get(output, "maxSize")), + .maxClosureSize = ptrToOwned(get(output, "maxClosureSize")), .allowedReferences = get_("allowedReferences"), .disallowedReferences = get_("disallowedReferences").value_or(std::set>{}), @@ -307,13 +268,13 @@ DerivationOptions derivationOptionsFromStructuredAttrs( std::map res; if (parsed) { - auto & structuredAttrs = parsed->structuredAttrs; - - if (auto * udr = get(structuredAttrs, "unsafeDiscardReferences")) { - for (auto & [outputName, output] : getObject(*udr)) { - if (!output.is_boolean()) - throw Error("attribute 'unsafeDiscardReferences.\"%s\"' must be a Boolean", outputName); - res.insert_or_assign(outputName, output.get()); + if (auto * udr = get(parsed->structuredAttrs, "unsafeDiscardReferences")) { + try { + for (auto & [outputName, output] : getObject(*udr)) + res.insert_or_assign(outputName, getBoolean(output)); + } catch (Error & e) { + e.addTrace({}, "while parsing attribute 'unsafeDiscardReferences'"); + throw; } } } @@ -340,9 +301,13 @@ DerivationOptions derivationOptionsFromStructuredAttrs( std::map> ret; if (parsed) { - auto * e = optionalValueAt(parsed->structuredAttrs, "exportReferencesGraph"); - if (!e || !e->is_object()) + auto * e = get(parsed->structuredAttrs, "exportReferencesGraph"); + if (!e) + return ret; + if (!e->is_object()) { + warn("'exportReferencesGraph' in structured attrs is not a JSON object, ignoring"); return ret; + } for (auto & [key, storePathsJson] : getObject(*e)) { StringSet ss; flatten(storePathsJson, ss); @@ -394,32 +359,9 @@ StringSet DerivationOptions::getRequiredSystemFeatures(const BasicDerivat } template -bool DerivationOptions::canBuildLocally(Store & localStore, const BasicDerivation & drv) const -{ - if (drv.platform != settings.thisSystem.get() && drv.platform != "wasm32-wasip1" - && !settings.extraPlatforms.get().count(drv.platform) && !drv.isBuiltin()) - return false; - - if (settings.maxBuildJobs.get() == 0 && !drv.isBuiltin()) - return false; - - for (auto & feature : getRequiredSystemFeatures(drv)) - if (!localStore.config.systemFeatures.get().count(feature)) - return false; - - return true; -} - -template -bool DerivationOptions::willBuildLocally(Store & localStore, const BasicDerivation & drv) const -{ - return preferLocalBuild && canBuildLocally(localStore, drv); -} - -template -bool DerivationOptions::substitutesAllowed() const +bool DerivationOptions::substitutesAllowed(const WorkerSettings & workerSettings) const { - return settings.alwaysAllowSubstitutes ? true : allowSubstitutes; + return workerSettings.alwaysAllowSubstitutes ? true : allowSubstitutes; } template @@ -430,7 +372,7 @@ bool DerivationOptions::useUidRange(const BasicDerivation & drv) const std::optional> tryResolve( const DerivationOptions & drvOptions, - std::function(ref drvPath, const std::string & outputName)> + fun(ref drvPath, const std::string & outputName)> queryResolutionChain) { auto tryResolvePath = [&](const SingleDerivedPath & input) -> std::optional { @@ -591,8 +533,8 @@ DerivationOptions adl_serializer OutputChecksVariant { auto outputChecks = getObject(valueAt(json, "outputChecks")); - auto forAllOutputsOpt = optionalValueAt(outputChecks, "forAllOutputs"); - auto perOutputOpt = optionalValueAt(outputChecks, "perOutput"); + auto forAllOutputsOpt = get(outputChecks, "forAllOutputs"); + auto perOutputOpt = get(outputChecks, "perOutput"); if (forAllOutputsOpt && !perOutputOpt) { return static_cast>(*forAllOutputsOpt); diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 5994e7cb43eb..9e861389c3a1 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -107,7 +107,7 @@ bool BasicDerivation::isBuiltin() const return builder.substr(0, 8) == "builtin:"; } -static auto infoForDerivation(Store & store, const Derivation & drv) +static auto infoForDerivation(const StoreDirConfig & store, const Derivation & drv) { auto references = drv.inputSrcs; for (auto & i : drv.inputDrvs.map) @@ -127,18 +127,10 @@ static auto infoForDerivation(Store & store, const Derivation & drv) }; } -StorePath writeDerivation( - Store & store, - const Derivation & drv, - RepairFlag repair, - bool readOnly, - std::shared_ptr provenance) +StorePath computeStorePath(const StoreDirConfig & store, const Derivation & drv) { - if (readOnly || settings.readOnlyMode) { - auto [_x, _y, _z, path] = infoForDerivation(store, drv); - return path; - } else - return store.writeDerivation(drv, repair, provenance); + auto [_suffix, _contents, _references, path] = infoForDerivation(store, drv); + return path; } StorePath @@ -170,24 +162,17 @@ Store::writeDerivation(const Derivation & drv, RepairFlag repair, std::shared_pt return path; } -StorePath writeDerivation( - Store & store, +StorePath Store::writeDerivation( AsyncPathWriter & asyncPathWriter, const Derivation & drv, RepairFlag repair, - bool readOnly, std::shared_ptr provenance) { auto references = drv.inputSrcs; for (auto & i : drv.inputDrvs.map) references.insert(i.first); return asyncPathWriter.addPath( - drv.unparse(store, false), - std::string(drv.name) + drvExtension, - references, - repair, - readOnly || settings.readOnlyMode, - provenance); + drv.unparse(*this, false), std::string(drv.name) + drvExtension, references, repair, provenance); } namespace { @@ -557,7 +542,6 @@ Derivation parseDerivation( */ static void printString(std::string & res, std::string_view s) { - res.reserve(res.size() + s.size() * 2 + 2); res += '"'; static constexpr auto chunkSize = 1024; std::array buffer; @@ -1154,6 +1138,39 @@ static void rewriteDerivation(Store & store, BasicDerivation & drv, const String } } +bool Derivation::shouldResolve() const +{ + /* No input drvs means nothing to resolve. */ + if (inputDrvs.map.empty()) + return false; + + auto drvType = type(); + + bool typeNeedsResolve = std::visit( + overloaded{ + [&](const DerivationType::InputAddressed & ia) { + /* Must resolve if deferred. */ + return ia.deferred; + }, + [&](const DerivationType::ContentAddressed & ca) { + return ca.fixed + /* Can optionally resolve if fixed, which is good + for avoiding unnecessary rebuilds. */ + ? experimentalFeatureSettings.isEnabled(Xp::CaDerivations) + /* Must resolve if floating. */ + : true; + }, + [&](const DerivationType::Impure &) { return true; }, + }, + drvType.raw); + + /* Also need to resolve if any inputs are outputs of dynamic derivations. */ + bool hasDynamicInputs = std::ranges::any_of( + inputDrvs.map.begin(), inputDrvs.map.end(), [](auto & pair) { return !pair.second.childMap.empty(); }); + + return typeNeedsResolve || hasDynamicInputs; +} + std::optional Derivation::tryResolve(Store & store, Store * evalStore) const { return tryResolve( @@ -1173,7 +1190,7 @@ static bool tryResolveInput( const DownstreamPlaceholder * placeholderOpt, ref drvPath, const DerivedPathMap::ChildNode & inputNode, - std::function(ref drvPath, const std::string & outputName)> + fun(ref drvPath, const std::string & outputName)> queryResolutionChain) { auto getPlaceholder = [&](const std::string & outputName) { @@ -1213,7 +1230,7 @@ static bool tryResolveInput( std::optional Derivation::tryResolve( Store & store, - std::function(ref drvPath, const std::string & outputName)> + fun(ref drvPath, const std::string & outputName)> queryResolutionChain) const { BasicDerivation resolved{*this}; diff --git a/src/libstore/derived-path-map.cc b/src/libstore/derived-path-map.cc index bcbdc85bd631..3d799ab1c583 100644 --- a/src/libstore/derived-path-map.cc +++ b/src/libstore/derived-path-map.cc @@ -1,4 +1,5 @@ #include "nix/store/derived-path-map.hh" +#include "nix/util/fun.hh" #include "nix/util/util.hh" namespace nix { @@ -6,8 +7,7 @@ namespace nix { template typename DerivedPathMap::ChildNode & DerivedPathMap::ensureSlot(const SingleDerivedPath & k) { - std::function initIter; - initIter = [&](const auto & k) -> auto & { + fun initIter = [&](const auto & k) -> auto & { return std::visit( overloaded{ [&](const SingleDerivedPath::Opaque & bo) -> auto & { @@ -27,8 +27,7 @@ typename DerivedPathMap::ChildNode & DerivedPathMap::ensureSlot(const Sing template typename DerivedPathMap::ChildNode * DerivedPathMap::findSlot(const SingleDerivedPath & k) { - std::function initIter; - initIter = [&](const auto & k) { + fun initIter = [&](const auto & k) { return std::visit( overloaded{ [&](const SingleDerivedPath::Opaque & bo) { diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index 32e95646a740..dde799ba528c 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -80,13 +80,7 @@ class WholeStoreViewAccessor : public SourceAccessor subdirs.emplace(baseName, std::move(accessor)); } - std::string readFile(const CanonPath & path) override - { - return callWithAccessorForPath( - path, [](SourceAccessor & accessor, const CanonPath & path) { return accessor.readFile(path); }); - } - - void readFile(const CanonPath & path, Sink & sink, std::function sizeCallback) override + void readFile(const CanonPath & path, Sink & sink, fun sizeCallback) override { return callWithAccessorForPath(path, [&](SourceAccessor & accessor, const CanonPath & path) { return accessor.readFile(path, sink, sizeCallback); @@ -125,6 +119,11 @@ ref DummyStoreConfig::openStore() const return openDummyStore(); } +bool DummyStoreConfig::getReadOnly() const +{ + return readOnly.get() || StoreConfig::getReadOnly(); +} + struct DummyStoreImpl : DummyStore { using Config = DummyStoreConfig; @@ -307,12 +306,13 @@ struct DummyStoreImpl : DummyStore StorePath writeDerivation(const Derivation & drv, RepairFlag repair, std::shared_ptr provenance) override { - auto drvPath = ::nix::writeDerivation(*this, drv, repair, /*readonly=*/true, provenance); + auto drvPath = nix::computeStorePath(*this, drv); if (!derivations.contains(drvPath) || repair) { if (config->readOnly) unsupported("writeDerivation"); derivations.insert({drvPath, drv}); + // FIXME: record provenance } return drvPath; @@ -418,7 +418,7 @@ ref adl_serializer>::from_json(const j { auto & obj = getObject(json); auto cfg = make_ref(DummyStore::Config::Params{}); - const_cast(cfg->storeDir_).set(getString(valueAt(obj, "store"))); + cfg->storeDir_.set(getString(valueAt(obj, "store"))); cfg->readOnly = true; return cfg; } diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc index 1c6283befa60..4dd1598442c4 100644 --- a/src/libstore/export-import.cc +++ b/src/libstore/export-import.cc @@ -11,6 +11,14 @@ namespace nix { static const uint32_t exportMagicV1 = 0x4558494e; static const uint64_t exportMagicV2 = 0x324f4952414e; // = 'NARIO2' +static WorkerProto::Version exportProtoVersion{ + .number = + { + .major = 1, + .minor = 16, + }, +}; + void exportPaths(Store & store, const StorePathSet & paths, Sink & sink, unsigned int version) { auto sorted = store.topoSortPaths(paths); @@ -57,7 +65,9 @@ void exportPaths(Store & store, const StorePathSet & paths, Sink & sink, unsigne auto info = store.queryPathInfo(path); // FIXME: move to CommonProto? WorkerProto::Serialise::write( - store, WorkerProto::WriteConn{.to = sink, .version = 16, .shortStorePaths = true}, *info); + store, + WorkerProto::WriteConn{.to = sink, .version = exportProtoVersion, .shortStorePaths = true}, + *info); dumpNar(*info); } @@ -144,7 +154,7 @@ StorePaths importPaths(Store & store, Source & source, CheckSigsFlag checkSigs) throw Error("input doesn't look like a nario"); auto info = WorkerProto::Serialise::read( - store, WorkerProto::ReadConn{.from = source, .version = 16, .shortStorePaths = true}); + store, WorkerProto::ReadConn{.from = source, .version = exportProtoVersion, .shortStorePaths = true}); Activity act( *logger, lvlTalkative, actUnknown, fmt("importing path '%s'", store.printStorePath(info.path))); diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 42cc2bc0e78d..5da4bc4b5535 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -2,7 +2,6 @@ #include "nix/store/globals.hh" #include "nix/util/config-global.hh" #include "nix/store/store-api.hh" -#include "nix/util/compression.hh" #include "nix/util/finally.hh" #include "nix/util/callback.hh" #include "nix/util/signals.hh" @@ -35,13 +34,49 @@ namespace nix { const unsigned int RETRY_TIME_MS_DEFAULT = 250; const unsigned int RETRY_TIME_MS_TOO_MANY_REQUESTS = 60000; +std::filesystem::path FileTransferSettings::getDefaultSSLCertFile() +{ + for (auto & fn : + {"/etc/ssl/certs/ca-certificates.crt", "/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt"}) + if (pathAccessible(fn)) + return fn; + return ""; +} + +FileTransferSettings::FileTransferSettings() +{ + auto sslOverride = getEnv("NIX_SSL_CERT_FILE").value_or(getEnv("SSL_CERT_FILE").value_or("")); + if (sslOverride != "") + caFile = sslOverride; +} + FileTransferSettings fileTransferSettings; static GlobalConfig::Register rFileTransferSettings(&fileTransferSettings); +namespace { + +using curlSList = std::unique_ptr<::curl_slist, decltype([](::curl_slist * list) { ::curl_slist_free_all(list); })>; +using curlMulti = std::unique_ptr<::CURLM, decltype([](::CURLM * multi) { ::curl_multi_cleanup(multi); })>; + +struct curlMultiError final : CloneableError +{ + ::CURLMcode code; + + curlMultiError(::CURLMcode code) + : CloneableError{"unexpected curl multi error: %s", ::curl_multi_strerror(code)} + { + assert(code != CURLM_OK); + } +}; + +} // namespace + struct curlFileTransfer : public FileTransfer { - CURLM * curlm = 0; + const FileTransferSettings & settings; + + curlMulti curlm; std::random_device rd; std::mt19937 mt19937; @@ -52,14 +87,10 @@ struct curlFileTransfer : public FileTransfer FileTransferRequest request; FileTransferResult result; std::unique_ptr _act; - bool done = false; // whether either the success or failure function has been called Callback callback; CURL * req = 0; // buffer to accompany the `req` above char errbuf[CURL_ERROR_SIZE]; - bool active = false; // whether the handle has been added to the multi object - bool paused = false; // whether the request has been paused previously - bool enqueued = false; // whether the request has been added to the incoming queue std::string statusMsg; unsigned int attempt = 0; @@ -68,11 +99,37 @@ struct curlFileTransfer : public FileTransfer has been reached. */ std::chrono::steady_clock::time_point embargo; - struct curl_slist * requestHeaders = 0; + curlSList requestHeaders; + + /** + * Whether either the success or failure function has been called. + */ + bool done:1 = false; + + /** + * Whether the handle has been added to the multi object. + */ + bool active:1 = false; + + /** + * Whether the request has been paused previously. + */ + bool paused:1 = false; + + /** + * Whether the request has been added the incoming queue. + */ + bool enqueued:1 = false; - std::string encoding; + /** + * Whether we can use range downloads for retries. + */ + bool acceptRanges:1 = false; - bool acceptRanges = false; + /** + * Whether the response has a non-trivial (not "identity") Content-Encoding. + */ + bool hasContentEncoding:1 = false; curl_off_t writtenToSink = 0; @@ -91,6 +148,15 @@ struct curlFileTransfer : public FileTransfer return httpStatus; } + void appendHeaders(const std::string & header) + { + curlSList tmpSList = curlSList(::curl_slist_append(requestHeaders.get(), requireCString(header))); + if (!tmpSList) + throw std::bad_alloc(); + requestHeaders.release(); + requestHeaders = std::move(tmpSList); + } + TransferItem( curlFileTransfer & fileTransfer, const FileTransferRequest & request, @@ -124,22 +190,12 @@ struct curlFileTransfer : public FileTransfer { result.urls.push_back(request.uri.to_string()); - /* Don't set Accept-Encoding for S3 requests that use AWS SigV4 signing. - curl's SigV4 implementation signs all headers including Accept-Encoding, - but some S3-compatible services (like GCS) modify this header in transit, - causing signature verification to fail. - See https://github.com/NixOS/nix/issues/15019 */ -#if NIX_WITH_AWS_AUTH - if (!request.awsSigV4Provider) -#endif - requestHeaders = - curl_slist_append(requestHeaders, "Accept-Encoding: zstd, br, gzip, deflate, bzip2, xz"); if (!request.expectedETag.empty()) - requestHeaders = curl_slist_append(requestHeaders, ("If-None-Match: " + request.expectedETag).c_str()); + appendHeaders("If-None-Match: " + request.expectedETag); if (!request.mimeType.empty()) - requestHeaders = curl_slist_append(requestHeaders, ("Content-Type: " + request.mimeType).c_str()); + appendHeaders("Content-Type: " + request.mimeType); for (auto it = request.headers.begin(); it != request.headers.end(); ++it) { - requestHeaders = curl_slist_append(requestHeaders, fmt("%s: %s", it->first, it->second).c_str()); + appendHeaders(fmt("%s: %s", it->first, it->second)); } } @@ -147,11 +203,9 @@ struct curlFileTransfer : public FileTransfer { if (req) { if (active) - curl_multi_remove_handle(fileTransfer.curlm, req); + curl_multi_remove_handle(fileTransfer.curlm.get(), req); curl_easy_cleanup(req); } - if (requestHeaders) - curl_slist_free_all(requestHeaders); try { if (!done && enqueued) fail(FileTransferError( @@ -167,6 +221,8 @@ struct curlFileTransfer : public FileTransfer done = true; try { std::rethrow_exception(ex); + } catch (FileTransferError & e) { + /* Already descriptive enough. */ } catch (nix::Error & e) { /* Add more context to the error message. */ e.addTrace({}, "during %s of '%s'", Uncolored(request.noun()), request.uri.to_string()); @@ -183,7 +239,6 @@ struct curlFileTransfer : public FileTransfer } LambdaSink finalSink; - std::shared_ptr decompressionSink; std::optional errorSink; std::exception_ptr callbackException; @@ -193,18 +248,15 @@ struct curlFileTransfer : public FileTransfer size_t realSize = size * nmemb; result.bodySize += realSize; - if (!decompressionSink) { - decompressionSink = makeDecompressionSink(encoding, finalSink); - if (!successfulStatuses.count(getHTTPStatus())) { - // In this case we want to construct a TeeSink, to keep - // the response around (which we figure won't be big - // like an actual download should be) to improve error - // messages. - errorSink = StringSink{}; - } + if (!errorSink && !successfulStatuses.count(getHTTPStatus())) { + // In this case we want to construct a TeeSink, to keep + // the response around (which we figure won't be big + // like an actual download should be) to improve error + // messages. + errorSink = StringSink{}; } - (*decompressionSink)({(char *) contents, realSize}); + finalSink({static_cast(contents), realSize}); if (paused) { /* The callback has signaled that the transfer needs to be paused. Already consumed data won't be returned twice unlike @@ -246,7 +298,7 @@ struct curlFileTransfer : public FileTransfer result.bodySize = 0; statusMsg = trim(match.str(1)); acceptRanges = false; - encoding = ""; + hasContentEncoding = false; appendCurrentUrl(); } else { @@ -269,8 +321,10 @@ struct curlFileTransfer : public FileTransfer } } - else if (name == "content-encoding") - encoding = trim(line.substr(i + 1)); + else if (name == "content-encoding") { + auto value = toLower(trim(line.substr(i + 1))); + hasContentEncoding = !value.empty() && value != "identity"; + } else if (name == "accept-ranges" && toLower(trim(line.substr(i + 1))) == "bytes") acceptRanges = true; @@ -288,14 +342,10 @@ struct curlFileTransfer : public FileTransfer } return realSize; } catch (...) { -#if LIBCURL_VERSION_NUM >= 0x075700 /* https://curl.se/libcurl/c/CURLOPT_HEADERFUNCTION.html: You can also abort the transfer by returning CURL_WRITEFUNC_ERROR. */ callbackException = std::current_exception(); return CURL_WRITEFUNC_ERROR; -#else - return realSize; -#endif } static size_t headerCallbackWrapper(void * contents, size_t size, size_t nmemb, void * userp) @@ -370,7 +420,7 @@ struct curlFileTransfer : public FileTransfer return ((TransferItem *) userp)->readCallback(buffer, size, nitems); } -#if !defined(_WIN32) && LIBCURL_VERSION_NUM >= 0x071000 +#if !defined(_WIN32) static int cloexec_callback(void *, curl_socket_t curlfd, curlsocktype purpose) { unix::closeOnExec(curlfd); @@ -433,6 +483,16 @@ struct curlFileTransfer : public FileTransfer } curl_easy_setopt(req, CURLOPT_URL, request.uri.to_string().c_str()); + + /* Enable transparent decompression for downloads. + Skip for uploads (Accept-Encoding is meaningless when sending data) + and when resuming from an offset (byte ranges don't work with + compressed content). */ + if (writtenToSink == 0 && !request.data) + /* Empty string means to enable all supported (that libcurl has + been linked to support) encodings. */ + curl_easy_setopt(req, CURLOPT_ACCEPT_ENCODING, ""); + curl_easy_setopt(req, CURLOPT_FOLLOWLOCATION, 1L); curl_easy_setopt(req, CURLOPT_MAXREDIRS, 10); curl_easy_setopt(req, CURLOPT_NOSIGNAL, 1); @@ -440,17 +500,14 @@ struct curlFileTransfer : public FileTransfer req, CURLOPT_USERAGENT, ("curl/" LIBCURL_VERSION " Nix/" + nixVersion + " DeterminateNix/" + determinateNixVersion - + (fileTransferSettings.userAgentSuffix != "" ? " " + fileTransferSettings.userAgentSuffix.get() : "")) + + (fileTransfer.settings.userAgentSuffix != "" ? " " + fileTransfer.settings.userAgentSuffix.get() + : "")) .c_str()); -#if LIBCURL_VERSION_NUM >= 0x072b00 curl_easy_setopt(req, CURLOPT_PIPEWAIT, 1); -#endif -#if LIBCURL_VERSION_NUM >= 0x072f00 - if (fileTransferSettings.enableHttp2) + if (fileTransfer.settings.enableHttp2) curl_easy_setopt(req, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2TLS); else curl_easy_setopt(req, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1); -#endif curl_easy_setopt(req, CURLOPT_WRITEFUNCTION, TransferItem::writeCallbackWrapper); curl_easy_setopt(req, CURLOPT_WRITEDATA, this); curl_easy_setopt(req, CURLOPT_HEADERFUNCTION, TransferItem::headerCallbackWrapper); @@ -460,10 +517,11 @@ struct curlFileTransfer : public FileTransfer curl_easy_setopt(req, CURLOPT_XFERINFODATA, this); curl_easy_setopt(req, CURLOPT_NOPROGRESS, 0); - curl_easy_setopt(req, CURLOPT_HTTPHEADER, requestHeaders); + curl_easy_setopt(req, CURLOPT_HTTPHEADER, requestHeaders.get()); - if (settings.downloadSpeed.get() > 0) - curl_easy_setopt(req, CURLOPT_MAX_RECV_SPEED_LARGE, (curl_off_t) (settings.downloadSpeed.get() * 1024)); + if (fileTransfer.settings.downloadSpeed.get() > 0) + curl_easy_setopt( + req, CURLOPT_MAX_RECV_SPEED_LARGE, (curl_off_t) (fileTransfer.settings.downloadSpeed.get() * 1024)); if (request.method == HttpMethod::Head) curl_easy_setopt(req, CURLOPT_NOBODY, 1); @@ -492,26 +550,40 @@ struct curlFileTransfer : public FileTransfer curl_easy_setopt(req, CURLOPT_SEEKDATA, this); } - if (settings.caFile != "") - curl_easy_setopt(req, CURLOPT_CAINFO, settings.caFile.get().c_str()); + if (auto & caFile = fileTransfer.settings.caFile.get()) + curl_easy_setopt(req, CURLOPT_CAINFO, caFile->c_str()); -#if !defined(_WIN32) && LIBCURL_VERSION_NUM >= 0x071000 +#if !defined(_WIN32) curl_easy_setopt(req, CURLOPT_SOCKOPTFUNCTION, cloexec_callback); #endif - - curl_easy_setopt(req, CURLOPT_CONNECTTIMEOUT, fileTransferSettings.connectTimeout.get()); + curl_easy_setopt(req, CURLOPT_CONNECTTIMEOUT, fileTransfer.settings.connectTimeout.get()); curl_easy_setopt(req, CURLOPT_LOW_SPEED_LIMIT, 1L); - curl_easy_setopt(req, CURLOPT_LOW_SPEED_TIME, fileTransferSettings.stalledDownloadTimeout.get()); + curl_easy_setopt(req, CURLOPT_LOW_SPEED_TIME, fileTransfer.settings.stalledDownloadTimeout.get()); /* If no file exist in the specified path, curl continues to work anyway as if netrc support was disabled. */ - curl_easy_setopt(req, CURLOPT_NETRC_FILE, settings.netrcFile.get().c_str()); + curl_easy_setopt(req, CURLOPT_NETRC_FILE, fileTransfer.settings.netrcFile.get().c_str()); curl_easy_setopt(req, CURLOPT_NETRC, CURL_NETRC_OPTIONAL); if (writtenToSink) curl_easy_setopt(req, CURLOPT_RESUME_FROM_LARGE, writtenToSink); + /* Note that the underlying strings get copied by libcurl, so the path -> string conversion is ok: + > The application does not have to keep the string around after setting this option. + https://curl.se/libcurl/c/CURLOPT_SSLKEY.html + https://curl.se/libcurl/c/CURLOPT_SSLCERT.html */ + + if (request.tlsCert) { + curl_easy_setopt(req, CURLOPT_SSLCERTTYPE, "PEM"); + curl_easy_setopt(req, CURLOPT_SSLCERT, request.tlsCert->string().c_str()); + } + + if (request.tlsKey) { + curl_easy_setopt(req, CURLOPT_SSLKEYTYPE, "PEM"); + curl_easy_setopt(req, CURLOPT_SSLKEY, request.tlsKey->string().c_str()); + } + curl_easy_setopt(req, CURLOPT_ERRORBUFFER, errbuf); errbuf[0] = 0; @@ -552,7 +624,7 @@ struct curlFileTransfer : public FileTransfer debug( "finished %s of '%s'; curl status = %d, HTTP status = %d, body = %d bytes, duration = %.2f s", - request.noun(), + Uncolored(request.noun()), request.uri, code, httpStatus, @@ -561,14 +633,6 @@ struct curlFileTransfer : public FileTransfer appendCurrentUrl(); - if (decompressionSink) { - try { - decompressionSink->finish(); - } catch (...) { - callbackException = std::current_exception(); - } - } - if (code == CURLE_WRITE_ERROR && result.etag == request.expectedETag) { code = CURLE_OK; httpStatus = 304; @@ -586,7 +650,9 @@ struct curlFileTransfer : public FileTransfer if (httpStatus == 304 && result.etag == "") result.etag = request.expectedETag; - act().progress(result.bodySize, result.bodySize); + curl_off_t dlSize = 0; + curl_easy_getinfo(req, CURLINFO_SIZE_DOWNLOAD_T, &dlSize); + act().progress(dlSize, dlSize); done = true; callback(std::move(result)); } @@ -635,6 +701,7 @@ struct curlFileTransfer : public FileTransfer case CURLE_TOO_MANY_REDIRECTS: case CURLE_WRITE_ERROR: case CURLE_UNSUPPORTED_PROTOCOL: + case CURLE_BAD_CONTENT_ENCODING: err = Misc; break; default: // Shut up warnings @@ -652,14 +719,14 @@ struct curlFileTransfer : public FileTransfer Interrupted, std::move(response), "%s of '%s' was interrupted", - request.noun(), + Uncolored(request.noun()), request.uri) : httpStatus != 0 ? FileTransferError( err, std::move(response), "unable to %s '%s': HTTP error %d%s", - request.verb(), + Uncolored(request.verb()), request.uri, httpStatus, code == CURLE_OK ? "" : fmt(" (curl error: %s)", curl_easy_strerror(code))) @@ -667,7 +734,7 @@ struct curlFileTransfer : public FileTransfer err, std::move(response), "unable to %s '%s': %s (%d) %s", - request.verb(), + Uncolored(request.verb()), request.uri, curl_easy_strerror(code), code, @@ -677,16 +744,29 @@ struct curlFileTransfer : public FileTransfer download after a while. If we're writing to a sink, we can only retry if the server supports ranged requests. */ - if (err == Transient && attempt < request.tries - && (!this->request.dataCallback || writtenToSink == 0 || (acceptRanges && encoding.empty()))) { + if (err == Transient && attempt < fileTransfer.settings.tries + && (!this->request.dataCallback || writtenToSink == 0 || (acceptRanges && !hasContentEncoding))) { int ms = retryTimeMs * std::pow( 2.0f, attempt - 1 + std::uniform_real_distribution<>(0.0, 0.5)(fileTransfer.mt19937)); - if (writtenToSink) - warn("%s; retrying from offset %d in %d ms", exc.what(), writtenToSink, ms); - else - warn("%s; retrying in %d ms", exc.what(), ms); - decompressionSink.reset(); + + if (writtenToSink) { + warn( + "%s; retrying from offset %d in %d ms (attempt %d/%d)", + exc.message(), + writtenToSink, + ms, + attempt, + fileTransfer.settings.tries); + } else { + warn( + "%s; retrying in %d ms (attempt %d/%d)", + exc.message(), + ms, + attempt, + fileTransfer.settings.tries); + } + errorSink.reset(); embargo = std::chrono::steady_clock::now() + std::chrono::milliseconds(ms); try { @@ -735,63 +815,52 @@ struct curlFileTransfer : public FileTransfer Sync state_; -#ifndef _WIN32 // TODO need graceful async exit support on Windows? - /* We can't use a std::condition_variable to wake up the curl - thread, because it only monitors file descriptors. So use a - pipe instead. */ - Pipe wakeupPipe; -#endif - std::thread workerThread; - const size_t maxQueueSize = - (fileTransferSettings.httpConnections.get() ? fileTransferSettings.httpConnections.get() - : std::max(1U, std::thread::hardware_concurrency())) - * 5; - - curlFileTransfer() - : mt19937(rd()) + const size_t maxQueueSize; + + curlFileTransfer(const FileTransferSettings & settings) + : settings(settings) + , mt19937(rd()) + , maxQueueSize([&]() -> std::size_t { + if (settings.httpConnections.get()) + return settings.httpConnections.get() * 5; + /* Zero means unlimited. See https://curl.se/libcurl/c/CURLMOPT_MAX_TOTAL_CONNECTIONS.html. */ + return std::numeric_limits::max(); + }()) { static std::once_flag globalInit; std::call_once(globalInit, curl_global_init, CURL_GLOBAL_ALL); - curlm = curl_multi_init(); - -#if LIBCURL_VERSION_NUM >= 0x072b00 // Multiplex requires >= 7.43.0 - curl_multi_setopt(curlm, CURLMOPT_PIPELINING, CURLPIPE_MULTIPLEX); -#endif -#if LIBCURL_VERSION_NUM >= 0x071e00 // Max connections requires >= 7.30.0 - curl_multi_setopt(curlm, CURLMOPT_MAX_TOTAL_CONNECTIONS, fileTransferSettings.httpConnections.get()); -#endif + curlm = curlMulti(curl_multi_init()); -#ifndef _WIN32 // TODO need graceful async exit support on Windows? - wakeupPipe.create(); - fcntl(wakeupPipe.readSide.get(), F_SETFL, O_NONBLOCK); -#endif + curl_multi_setopt(curlm.get(), CURLMOPT_PIPELINING, CURLPIPE_MULTIPLEX); + curl_multi_setopt(curlm.get(), CURLMOPT_MAX_TOTAL_CONNECTIONS, settings.httpConnections.get()); workerThread = std::thread([&]() { workerThreadEntry(); }); } ~curlFileTransfer() { - stopWorkerThread(); - + try { + stopWorkerThread(); + } catch (...) { + ignoreExceptionInDestructor(); + } workerThread.join(); - - if (curlm) - curl_multi_cleanup(curlm); } void stopWorkerThread() { /* Signal the worker thread to exit. */ - { - auto state(state_.lock()); - state->quit(); - } -#ifndef _WIN32 // TODO need graceful async exit support on Windows? - writeFull(wakeupPipe.writeSide.get(), " ", false); -#endif + state_.lock()->quit(); + wakeupMulti(); + } + + void wakeupMulti() + { + if (auto ec = ::curl_multi_wakeup(curlm.get())) + throw curlMultiError(ec); } void workerThreadMain() @@ -826,32 +895,25 @@ struct curlFileTransfer : public FileTransfer /* Let curl do its thing. */ int running; - CURLMcode mc = curl_multi_perform(curlm, &running); + CURLMcode mc = curl_multi_perform(curlm.get(), &running); if (mc != CURLM_OK) throw nix::Error("unexpected error from curl_multi_perform(): %s", curl_multi_strerror(mc)); /* Set the promises of any finished requests. */ CURLMsg * msg; int left; - while ((msg = curl_multi_info_read(curlm, &left))) { + while ((msg = curl_multi_info_read(curlm.get(), &left))) { if (msg->msg == CURLMSG_DONE) { auto i = items.find(msg->easy_handle); assert(i != items.end()); i->second->finish(msg->data.result); - curl_multi_remove_handle(curlm, i->second->req); + curl_multi_remove_handle(curlm.get(), i->second->req); i->second->active = false; items.erase(i); } } /* Wait for activity, including wakeup events. */ - int numfds = 0; - struct curl_waitfd extraFDs[1]; -#ifndef _WIN32 // TODO need graceful async exit support on Windows? - extraFDs[0].fd = wakeupPipe.readSide.get(); - extraFDs[0].events = CURL_WAIT_POLLIN; - extraFDs[0].revents = 0; -#endif long maxSleepTimeMs = items.empty() ? 10000 : 100; auto sleepTimeMs = nextWakeup != std::chrono::steady_clock::time_point() ? std::max( @@ -860,23 +922,14 @@ struct curlFileTransfer : public FileTransfer nextWakeup - std::chrono::steady_clock::now()) .count()) : maxSleepTimeMs; - vomit("download thread waiting for %d ms", sleepTimeMs); - mc = curl_multi_wait(curlm, extraFDs, 1, sleepTimeMs, &numfds); + + int numfds = 0; + mc = curl_multi_poll(curlm.get(), nullptr, 0, sleepTimeMs, &numfds); if (mc != CURLM_OK) - throw nix::Error("unexpected error from curl_multi_wait(): %s", curl_multi_strerror(mc)); + throw curlMultiError(mc); nextWakeup = std::chrono::steady_clock::time_point(); - /* Add new curl requests from the incoming requests queue, - except for requests that are embargoed (waiting for a - retry timeout to expire). */ - if (extraFDs[0].revents & CURL_WAIT_POLLIN) { - char buf[1024]; - auto res = read(extraFDs[0].fd, buf, sizeof(buf)); - if (res == -1 && errno != EINTR) - throw SysError("reading curl wakeup socket"); - } - std::vector> incoming; auto now = std::chrono::steady_clock::now(); @@ -904,9 +957,9 @@ struct curlFileTransfer : public FileTransfer } for (auto & item : incoming) { - debug("starting %s of %s", item->request.noun(), item->request.uri); + debug("starting %s of '%s'", Uncolored(item->request.noun()), item->request.uri); item->init(); - curl_multi_add_handle(curlm, item->req); + curl_multi_add_handle(curlm.get(), item->req); item->active = true; items[item->req] = item; } @@ -956,12 +1009,10 @@ struct curlFileTransfer : public FileTransfer if (state->isQuitting()) throw nix::Error("cannot enqueue download request because the download thread is shutting down"); state->incoming.push(item); - item->enqueued = true; + item->enqueued = true; /* Now any exceptions should be reported via the callback. */ } -#ifndef _WIN32 // TODO need graceful async exit support on Windows? - writeFull(wakeupPipe.writeSide.get(), " "); -#endif + wakeupMulti(); return ItemHandle(static_cast(*item)); } @@ -981,9 +1032,7 @@ struct curlFileTransfer : public FileTransfer { auto state(state_.lock()); state->unpause.push_back(std::move(item)); -#ifndef _WIN32 // TODO need graceful async exit support on Windows? - writeFull(wakeupPipe.writeSide.get(), " "); -#endif + wakeupMulti(); } void unpauseTransfer(ItemHandle handle) override @@ -992,6 +1041,11 @@ struct curlFileTransfer : public FileTransfer } }; +ref makeCurlFileTransfer(const FileTransferSettings & settings = fileTransferSettings) +{ + return make_ref(settings); +} + static Sync> _fileTransfer; ref getFileTransfer() @@ -999,16 +1053,11 @@ ref getFileTransfer() auto fileTransfer(_fileTransfer.lock()); if (!*fileTransfer || (*fileTransfer)->state_.lock()->isQuitting()) - *fileTransfer = std::make_shared(); + *fileTransfer = makeCurlFileTransfer().get_ptr(); return ref(*fileTransfer); } -ref makeFileTransfer() -{ - return make_ref(); -} - std::shared_ptr resetFileTransfer() { auto fileTransfer(_fileTransfer.lock()); @@ -1017,6 +1066,11 @@ std::shared_ptr resetFileTransfer() return prev; } +ref makeFileTransfer(const FileTransferSettings & settings) +{ + return makeCurlFileTransfer(settings); +} + void FileTransferRequest::setupForS3() { auto parsedS3 = ParsedS3URL::parse(uri.parsed()); @@ -1198,7 +1252,7 @@ void FileTransfer::download( template FileTransferError::FileTransferError( FileTransfer::Error error, std::optional response, const Args &... args) - : Error(args...) + : CloneableError(args...) , error(error) , response(response) { diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 37f148cbc431..f41944a92ce0 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -1,20 +1,19 @@ #include "nix/store/derivations.hh" #include "nix/store/globals.hh" +#include "nix/store/local-gc.hh" #include "nix/store/local-store.hh" #include "nix/store/path.hh" +#include "nix/util/configuration.hh" #include "nix/util/finally.hh" #include "nix/util/unix-domain-socket.hh" #include "nix/util/signals.hh" +#include "nix/util/serialise.hh" #include "nix/util/util.hh" +#include "nix/util/file-system.hh" #include "nix/store/posix-fs-canonicalise.hh" #include "store-config-private.hh" -#if !defined(__linux__) -// For shelling out to lsof -# include "nix/util/processes.hh" -#endif - #include #include #include @@ -36,13 +35,13 @@ namespace nix { -static std::string gcSocketPath = "/gc-socket/socket"; +static std::string gcSocketPath = "gc-socket/socket"; static std::string gcRootsDir = "gcroots"; -void LocalStore::addIndirectRoot(const Path & path) +void LocalStore::addIndirectRoot(const std::filesystem::path & path) { - std::string hash = hashString(HashAlgorithm::SHA1, path).to_string(HashFormat::Nix32, false); - Path realRoot = canonPath(fmt("%1%/%2%/auto/%3%", config->stateDir, gcRootsDir, hash)); + std::string hash = hashString(HashAlgorithm::SHA1, path.string()).to_string(HashFormat::Nix32, false); + auto realRoot = canonPath(config->stateDir.get() / gcRootsDir / "auto" / hash); makeSymlink(realRoot, path); } @@ -58,19 +57,16 @@ void LocalStore::createTempRootsFile() if (pathExists(fnTempRoots)) /* It *must* be stale, since there can be no two processes with the same pid. */ - unlink(fnTempRoots.c_str()); + tryUnlink(fnTempRoots); *fdTempRoots = openLockFile(fnTempRoots, true); - debug("acquiring write lock on '%s'", fnTempRoots); + debug("acquiring write lock on %s", PathFmt(fnTempRoots)); lockFile(fdTempRoots->get(), ltWrite, true); /* Check whether the garbage collector didn't get in our way. */ - struct stat st; - if (fstat(fromDescriptorReadOnly(fdTempRoots->get()), &st) == -1) - throw SysError("statting '%1%'", fnTempRoots); - if (st.st_size == 0) + if (getFileSize(fdTempRoots->get()) == 0) break; /* The garbage collector deleted this file before we could get @@ -109,15 +105,15 @@ void LocalStore::addTempRoot(const StorePath & path) auto fdRootsSocket(_fdRootsSocket.lock()); if (!*fdRootsSocket) { - auto socketPath = config->stateDir.get() + gcSocketPath; - debug("connecting to '%s'", socketPath); + auto socketPath = config->stateDir.get() / gcSocketPath; + debug("connecting to '%s'", PathFmt(socketPath)); *fdRootsSocket = createUnixDomainSocket(); try { nix::connect(toSocket(fdRootsSocket->get()), socketPath); - } catch (SysError & e) { + } catch (SystemError & e) { /* The garbage collector may have exited or not created the socket yet, so we need to restart. */ - if (e.errNo == ECONNREFUSED || e.errNo == ENOENT) { + if (e.is(std::errc::connection_refused) || e.is(std::errc::no_such_file_or_directory)) { debug("GC socket connection refused: %s", e.msg()); fdRootsSocket->close(); std::this_thread::sleep_for(std::chrono::milliseconds(100)); @@ -134,10 +130,10 @@ void LocalStore::addTempRoot(const StorePath & path) readFull(fdRootsSocket->get(), &c, 1); assert(c == '1'); debug("got ack for GC root '%s'", printStorePath(path)); - } catch (SysError & e) { + } catch (SystemError & e) { /* The garbage collector may have exited, so we need to restart. */ - if (e.errNo == EPIPE || e.errNo == ECONNRESET) { + if (e.is(std::errc::broken_pipe) || e.is(std::errc::connection_reset)) { debug("GC socket disconnected"); fdRootsSocket->close(); goto restart; @@ -170,13 +166,13 @@ void LocalStore::findTempRoots(Roots & tempRoots, bool censor) // those to keep the directory alive. continue; } - Path path = i.path().string(); + auto path = i.path(); pid_t pid = std::stoi(name); - debug("reading temporary root file '%1%'", path); + debug("reading temporary root file %1%", PathFmt(path)); AutoCloseFD fd(toDescriptor(open( - path.c_str(), + path.string().c_str(), #ifndef _WIN32 O_CLOEXEC | #endif @@ -186,15 +182,15 @@ void LocalStore::findTempRoots(Roots & tempRoots, bool censor) /* It's okay if the file has disappeared. */ if (errno == ENOENT) continue; - throw SysError("opening temporary roots file '%1%'", path); + throw SysError("opening temporary roots file %1%", PathFmt(path)); } /* Try to acquire a write lock without blocking. This can only succeed if the owning process has died. In that case we don't care about its temporary roots. */ if (lockFile(fd.get(), ltWrite, false)) { - printInfo("removing stale temporary roots file '%1%'", path); - unlink(path.c_str()); + printInfo("removing stale temporary roots file %1%", PathFmt(path)); + tryUnlink(path); writeFull(fd.get(), "d"); continue; } @@ -206,7 +202,7 @@ void LocalStore::findTempRoots(Roots & tempRoots, bool censor) std::string::size_type pos = 0, end; while ((end = contents.find((char) 0, pos)) != std::string::npos) { - Path root(contents, pos, end - pos); + auto root = std::string_view(contents).substr(pos, end - pos); debug("got temporary root '%s'", root); tempRoots[parseStorePath(root)].emplace(censor ? censored : fmt("{nix-process:%d}", pid)); pos = end + 1; @@ -214,15 +210,15 @@ void LocalStore::findTempRoots(Roots & tempRoots, bool censor) } } -void LocalStore::findRoots(const Path & path, std::filesystem::file_type type, Roots & roots) +void LocalStore::findRoots(const std::filesystem::path & path, std::filesystem::file_type type, Roots & roots) { - auto foundRoot = [&](const Path & path, const Path & target) { + auto foundRoot = [&](const std::filesystem::path & path, const std::filesystem::path & target) { try { - auto storePath = toStorePath(target).first; + auto storePath = toStorePath(target.string()).first; if (isValidPath(storePath)) - roots[std::move(storePath)].emplace(path); + roots[std::move(storePath)].emplace(path.string()); else - printInfo("skipping invalid root from '%1%' to '%2%'", path, target); + printInfo("skipping invalid root from %1% to %2%", PathFmt(path), PathFmt(target)); } catch (BadStorePath &) { } }; @@ -235,37 +231,38 @@ void LocalStore::findRoots(const Path & path, std::filesystem::file_type type, R if (type == std::filesystem::file_type::directory) { for (auto & i : DirectoryIterator{path}) { checkInterrupt(); - findRoots(i.path().string(), i.symlink_status().type(), roots); + findRoots(i.path(), i.symlink_status().type(), roots); } } else if (type == std::filesystem::file_type::symlink) { - Path target = readLink(path); - if (isInStore(target)) + auto target = readLink(path); + if (isInStore(target.string())) foundRoot(path, target); /* Handle indirect roots. */ else { - target = absPath(target, dirOf(path)); + auto parentPath = path.parent_path(); + target = absPath(target, &parentPath); if (!pathExists(target)) { - if (isInDir(path, std::filesystem::path{config->stateDir.get()} / gcRootsDir / "auto")) { - printInfo("removing stale link from '%1%' to '%2%'", path, target); - unlink(path.c_str()); + if (isInDir(path, config->stateDir.get() / gcRootsDir / "auto")) { + printInfo("removing stale link from %1% to %2%", PathFmt(path), PathFmt(target)); + tryUnlink(path); } } else { if (!std::filesystem::is_symlink(target)) return; - Path target2 = readLink(target); - if (isInStore(target2)) + auto target2 = readLink(target); + if (isInStore(target2.string())) foundRoot(target, target2); } } } else if (type == std::filesystem::file_type::regular) { - auto storePath = maybeParseStorePath(storeDir + "/" + std::string(baseNameOf(path))); + auto storePath = maybeParseStorePath(storeDir + "/" + std::string(baseNameOf(path.string()))); if (storePath && isValidPath(*storePath)) - roots[std::move(*storePath)].emplace(path); + roots[std::move(*storePath)].emplace(path.string()); } } @@ -274,15 +271,16 @@ void LocalStore::findRoots(const Path & path, std::filesystem::file_type type, R /* We only ignore permanent failures. */ if (e.code() == std::errc::permission_denied || e.code() == std::errc::no_such_file_or_directory || e.code() == std::errc::not_a_directory) - printInfo("cannot read potential root '%1%'", path); + printInfo("cannot read potential root %1%", PathFmt(path)); else - throw; + throw SystemError(e.code(), "finding GC roots in %1%", PathFmt(path)); } - catch (SysError & e) { + catch (SystemError & e) { /* We only ignore permanent failures. */ - if (e.errNo == EACCES || e.errNo == ENOENT || e.errNo == ENOTDIR) - printInfo("cannot read potential root '%1%'", path); + if (e.is(std::errc::permission_denied) || e.is(std::errc::no_such_file_or_directory) + || e.is(std::errc::not_a_directory)) + printInfo("cannot read potential root %1%", PathFmt(path)); else throw; } @@ -291,8 +289,8 @@ void LocalStore::findRoots(const Path & path, std::filesystem::file_type type, R void LocalStore::findRootsNoTemp(Roots & roots, bool censor) { /* Process direct roots in {gcroots,profiles}. */ - findRoots(config->stateDir + "/" + gcRootsDir, std::filesystem::file_type::unknown, roots); - findRoots(config->stateDir + "/profiles", std::filesystem::file_type::unknown, roots); + findRoots(config->stateDir.get() / gcRootsDir, std::filesystem::file_type::unknown, roots); + findRoots(config->stateDir.get() / "profiles", std::filesystem::file_type::unknown, roots); /* Add additional roots returned by different platforms-specific heuristics. This is typically used to add running programs to @@ -310,151 +308,42 @@ Roots LocalStore::findRoots(bool censor) return roots; } -/** - * Key is a mere string because cannot has path with macOS's libc++ - */ -typedef boost::unordered_flat_map< - std::string, - boost::unordered_flat_set>, - StringViewHash, - std::equal_to<>> - UncheckedRoots; - -static void readProcLink(const std::filesystem::path & file, UncheckedRoots & roots) +static Roots requestRuntimeRoots(const LocalStoreConfig & config, const std::filesystem::path & socketPath) { - std::filesystem::path buf; - try { - buf = std::filesystem::read_symlink(file); - } catch (std::filesystem::filesystem_error & e) { - if (e.code() == std::errc::no_such_file_or_directory || e.code() == std::errc::permission_denied - || e.code() == std::errc::no_such_process) - return; - throw; - } - if (buf.is_absolute()) - roots[buf.string()].emplace(file.string()); -} + Roots roots; -static std::string quoteRegexChars(const std::string & raw) -{ - static auto specialRegex = boost::regex(R"([.^$\\*+?()\[\]{}|])"); - return boost::regex_replace(raw, specialRegex, R"(\$&)"); -} + auto socket = connect(socketPath); + auto socketSource = FdSource(socket.get()); -#ifdef __linux__ -static void readFileRoots(const std::filesystem::path & path, UncheckedRoots & roots) -{ - try { - roots[readFile(path)].emplace(path.string()); - } catch (SysError & e) { - if (e.errNo != ENOENT && e.errNo != EACCES) - throw; - } + while (1) { + auto line = socketSource.readLine(true, '\0'); + if (line == "") + break; + roots[config.parseStorePath(line)].insert(censored); + }; + + return roots; } -#endif void LocalStore::findRuntimeRoots(Roots & roots, bool censor) { - UncheckedRoots unchecked; - - auto procDir = AutoCloseDir{opendir("/proc")}; - if (procDir) { - struct dirent * ent; - static const auto digitsRegex = boost::regex(R"(^\d+$)"); - static const auto mapRegex = boost::regex(R"(^\s*\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+(/\S+)\s*$)"); - auto storePathRegex = boost::regex(quoteRegexChars(storeDir) + R"(/[0-9a-z]+[0-9a-zA-Z\+\-\._\?=]*)"); - while (errno = 0, ent = readdir(procDir.get())) { - checkInterrupt(); - if (boost::regex_match(ent->d_name, digitsRegex)) { - try { - readProcLink(fmt("/proc/%s/exe", ent->d_name), unchecked); - readProcLink(fmt("/proc/%s/cwd", ent->d_name), unchecked); - - auto fdStr = fmt("/proc/%s/fd", ent->d_name); - auto fdDir = AutoCloseDir(opendir(fdStr.c_str())); - if (!fdDir) { - if (errno == ENOENT || errno == EACCES) - continue; - throw SysError("opening %1%", fdStr); - } - struct dirent * fd_ent; - while (errno = 0, fd_ent = readdir(fdDir.get())) { - if (fd_ent->d_name[0] != '.') - readProcLink(fmt("%s/%s", fdStr, fd_ent->d_name), unchecked); - } - if (errno) { - if (errno == ESRCH) - continue; - throw SysError("iterating /proc/%1%/fd", ent->d_name); - } - fdDir.reset(); - - std::filesystem::path mapFile = fmt("/proc/%s/maps", ent->d_name); - auto mapLines = tokenizeString>(readFile(mapFile.string()), "\n"); - for (const auto & line : mapLines) { - auto match = boost::smatch{}; - if (boost::regex_match(line, match, mapRegex)) - unchecked[match[1]].emplace(mapFile.string()); - } + Roots unchecked; - auto envFile = fmt("/proc/%s/environ", ent->d_name); - auto envString = readFile(envFile); - auto env_end = boost::sregex_iterator{}; - for (auto i = boost::sregex_iterator{envString.begin(), envString.end(), storePathRegex}; - i != env_end; - ++i) - unchecked[i->str()].emplace(envFile); - } catch (SystemError & e) { - if (errno == ENOENT || errno == EACCES || errno == ESRCH) - continue; - throw; - } - } - } - if (errno) - throw SysError("iterating /proc"); - } - -#if !defined(__linux__) - // lsof is really slow on OS X. This actually causes the gc-concurrent.sh test to fail. - // See: https://github.com/NixOS/nix/issues/3011 - // Because of this we disable lsof when running the tests. - if (getEnv("_NIX_TEST_NO_LSOF") != "1") { - try { - boost::regex lsofRegex(R"(^n(/.*)$)"); - auto lsofLines = - tokenizeString>(runProgram(LSOF, true, {"-n", "-w", "-F", "n"}), "\n"); - for (const auto & line : lsofLines) { - boost::smatch match; - if (boost::regex_match(line, match, lsofRegex)) - unchecked[match[1].str()].emplace("{lsof}"); - } - } catch (ExecError & e) { - /* lsof not installed, lsof failed */ - } + if (config->useRootsDaemon) { + experimentalFeatureSettings.require(Xp::LocalOverlayStore); + unchecked = requestRuntimeRoots(*config, config->getRootsSocketPath()); + } else { + unchecked = findRuntimeRootsUnchecked(*config); } -#endif - -#ifdef __linux__ - readFileRoots("/proc/sys/kernel/modprobe", unchecked); - readFileRoots("/proc/sys/kernel/fbsplash", unchecked); - readFileRoots("/proc/sys/kernel/poweroff_cmd", unchecked); -#endif - for (auto & [target, links] : unchecked) { - if (!isInStore(target)) + for (auto & [path, links] : unchecked) { + if (!isValidPath(path)) continue; - try { - auto path = toStorePath(target).first; - if (!isValidPath(path)) - continue; - debug("got additional root '%1%'", printStorePath(path)); - if (censor) - roots[path].insert(censored); - else - roots[path].insert(links.begin(), links.end()); - } catch (BadStorePath &) { - } + debug("got additional root '%1%'", printStorePath(path)); + if (censor) + roots[path].insert(censored); + else + roots[path].insert(links.begin(), links.end()); } } @@ -463,9 +352,11 @@ struct GCLimitReached void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) { + const auto & gcSettings = config->getLocalSettings().getGCSettings(); + bool shouldDelete = options.action == GCOptions::gcDeleteDead || options.action == GCOptions::gcDeleteSpecific; - bool gcKeepOutputs = settings.gcKeepOutputs; - bool gcKeepDerivations = settings.gcKeepDerivations; + bool keepOutputs = gcSettings.keepOutputs; + bool keepDerivations = gcSettings.keepDerivations; Roots roots; boost::unordered_flat_set> dead, alive; @@ -490,8 +381,8 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) (the garbage collector will recurse into deleting the outputs or derivers, respectively). So disable them. */ if (options.action == GCOptions::gcDeleteSpecific && options.ignoreLiveness) { - gcKeepOutputs = false; - gcKeepDerivations = false; + keepOutputs = false; + keepDerivations = false; } if (shouldDelete) @@ -509,8 +400,8 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) readFile(*p); /* Start the server for receiving new roots. */ - auto socketPath = config->stateDir.get() + gcSocketPath; - createDirs(dirOf(socketPath)); + auto socketPath = config->stateDir.get() / gcSocketPath; + createDirs(socketPath.parent_path()); auto fdServer = createUnixDomainSocket(socketPath, 0666); // TODO nonblocking socket on windows? @@ -518,7 +409,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) throw UnimplementedError("External GC client not implemented yet"); #else if (fcntl(fdServer.get(), F_SETFL, fcntl(fdServer.get(), F_GETFL) | O_NONBLOCK) == -1) - throw SysError("making socket '%1%' non-blocking", socketPath); + throw SysError("making socket %s non-blocking", PathFmt(socketPath)); Pipe shutdownPipe; shutdownPipe.create(); @@ -577,9 +468,10 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) if (fcntl(fdClient.get(), F_SETFL, fcntl(fdClient.get(), F_GETFL) & ~O_NONBLOCK) == -1) panic("Could not set non-blocking flag on client socket"); + FdSource source(fdClient.get()); while (true) { try { - auto path = readLine(fdClient.get()); + auto path = source.readLine(); auto storePath = maybeParseStorePath(path); if (storePath) { debug("got new GC root '%s'", path); @@ -644,17 +536,19 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) /* Helper function that deletes a path from the store and throws GCLimitReached if we've deleted enough garbage. */ - auto deleteFromStore = [&](std::string_view baseName) { - Path path = storeDir + "/" + std::string(baseName); - Path realPath = config->realStoreDir + "/" + std::string(baseName); + auto deleteFromStore = [&](std::string_view baseName, bool isKnownPath) { + assert(!std::filesystem::path(baseName).is_absolute()); + /* Using `std::string` since this is the logical store dir. Hopefully that is the right choice. */ + std::string path = storeDir + "/" + std::string(baseName); + auto realPath = config->realStoreDir.get() / std::string(baseName); /* There may be temp directories in the store that are still in use by another process. We need to be sure that we can acquire an exclusive lock before deleting them. */ if (baseName.find("tmp-", 0) == 0) { - AutoCloseFD tmpDirFd = openDirectory(realPath); + auto tmpDirFd = openDirectory(realPath); if (!tmpDirFd || !lockFile(tmpDirFd.get(), ltWrite, false)) { - debug("skipping locked tempdir '%s'", realPath); + debug("skipping locked tempdir %s", PathFmt(realPath)); return; } } @@ -664,7 +558,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) results.paths.insert(path); uint64_t bytesFreed; - deleteStorePath(realPath, bytesFreed); + deleteStorePath(realPath, bytesFreed, isKnownPath); results.bytesFreed += bytesFreed; @@ -723,8 +617,8 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) *path, closure, /* flipDirection */ false, - gcKeepOutputs, - gcKeepDerivations); + keepOutputs, + keepDerivations); for (auto & p : closure) alive.insert(p); } catch (InvalidPath &) { @@ -777,7 +671,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) /* If keep-derivations is set and this is a derivation, then visit the derivation outputs. */ - if (gcKeepDerivations && path->isDerivation()) { + if (keepDerivations && path->isDerivation()) { for (auto & [name, maybeOutPath] : queryPartialDerivationOutputMap(*path)) if (maybeOutPath && isValidPath(*maybeOutPath) && queryPathInfo(*maybeOutPath)->deriver == *path) @@ -785,7 +679,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) } /* If keep-outputs is set, then visit the derivers. */ - if (gcKeepOutputs) { + if (keepOutputs) { auto derivers = queryValidDerivers(*path); for (auto & i : derivers) enqueue(i); @@ -798,7 +692,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) if (shouldDelete) { try { invalidatePathChecked(path); - deleteFromStore(path.to_string()); + deleteFromStore(path.to_string(), true); referrersCache.erase(path); } catch (PathInUse & e) { // If we end up here, it's likely a new occurrence @@ -826,14 +720,14 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) printInfo("determining live/dead paths..."); try { - AutoCloseDir dir(opendir(config->realStoreDir.get().c_str())); + AutoCloseDir dir(opendir(config->realStoreDir.get().string().c_str())); if (!dir) - throw SysError("opening directory '%1%'", config->realStoreDir); + throw SysError("opening directory %1%", PathFmt(config->realStoreDir.get())); /* Read the store and delete all paths that are invalid or unreachable. We don't use readDirectory() here so that GCing can start faster. */ - auto linksName = baseNameOf(linksDir); + auto linksName = linksDir.filename(); struct dirent * dirent; while (errno = 0, dirent = readdir(dir.get())) { checkInterrupt(); @@ -844,7 +738,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) if (auto storePath = maybeParseStorePath(storeDir + "/" + name)) deleteReferrersClosure(*storePath); else - deleteFromStore(name); + deleteFromStore(name, false); } } catch (GCLimitReached & e) { } @@ -870,9 +764,9 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) if (options.action == GCOptions::gcDeleteDead || options.action == GCOptions::gcDeleteSpecific) { printInfo("deleting unused links..."); - AutoCloseDir dir(opendir(linksDir.c_str())); + AutoCloseDir dir(opendir(linksDir.string().c_str())); if (!dir) - throw SysError("opening directory '%1%'", linksDir); + throw SysError("opening directory %1%", PathFmt(linksDir)); int64_t actualSize = 0, unsharedSize = 0; @@ -882,7 +776,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) std::string name = dirent->d_name; if (name == "." || name == "..") continue; - Path path = linksDir + "/" + name; + auto path = linksDir / name; auto st = lstat(path); @@ -892,23 +786,22 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) continue; } - printMsg(lvlTalkative, "deleting unused link '%1%'", path); + printMsg(lvlTalkative, "deleting unused link %1%", PathFmt(path)); - if (unlink(path.c_str()) == -1) - throw SysError("deleting '%1%'", path); + unlink(path); /* Do not account for deleted file here. Rely on deletePath() accounting. */ } - struct stat st; - if (stat(linksDir.c_str(), &st) == -1) - throw SysError("statting '%1%'", linksDir); int64_t overhead = #ifdef _WIN32 0 #else - st.st_blocks * 512ULL + [&] { + auto st = stat(linksDir); + return st.st_blocks * 512ULL; + }() #endif ; @@ -922,6 +815,8 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) void LocalStore::autoGC(bool sync) { #if HAVE_STATVFS + const auto & gcSettings = config->getLocalSettings().getGCSettings(); + static auto fakeFreeSpaceFile = getEnv("_NIX_TEST_FREE_SPACE_FILE"); auto getAvail = [this]() -> uint64_t { @@ -930,7 +825,7 @@ void LocalStore::autoGC(bool sync) struct statvfs st; if (statvfs(config->realStoreDir.get().c_str(), &st)) - throw SysError("getting filesystem info about '%s'", config->realStoreDir); + throw SysError("getting filesystem info about '%s'", PathFmt(config->realStoreDir.get())); return (uint64_t) st.f_bavail * st.f_frsize; }; @@ -948,14 +843,14 @@ void LocalStore::autoGC(bool sync) auto now = std::chrono::steady_clock::now(); - if (now < state->lastGCCheck + std::chrono::seconds(settings.minFreeCheckInterval)) + if (now < state->lastGCCheck + std::chrono::seconds(gcSettings.minFreeCheckInterval)) return; auto avail = getAvail(); state->lastGCCheck = now; - if (avail >= settings.minFree || avail >= settings.maxFree) + if (avail >= gcSettings.minFree || avail >= gcSettings.maxFree) return; if (avail > state->availAfterGC * 0.97) @@ -966,7 +861,7 @@ void LocalStore::autoGC(bool sync) std::promise promise; future = state->gcFuture = promise.get_future().share(); - std::thread([promise{std::move(promise)}, this, avail, getAvail]() mutable { + std::thread([promise{std::move(promise)}, this, avail, getAvail, &gcSettings]() mutable { try { /* Wake up any threads waiting for the auto-GC to finish. */ @@ -978,7 +873,7 @@ void LocalStore::autoGC(bool sync) }); GCOptions options; - options.maxFreed = settings.maxFree - avail; + options.maxFreed = gcSettings.maxFree - avail; printInfo("running auto-GC to free %d bytes", options.maxFreed); diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 982a4f0b6928..cc793db89ef0 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -1,11 +1,16 @@ #include "nix/store/globals.hh" +#include "nix/store/profiles.hh" +#include "nix/util/config-impl.hh" #include "nix/util/config-global.hh" #include "nix/util/current-process.hh" +#include "nix/util/executable-path.hh" #include "nix/util/archive.hh" #include "nix/util/args.hh" #include "nix/util/abstract-setting-to-json.hh" #include "nix/util/compute-levels.hh" +#include "nix/util/executable-path.hh" #include "nix/util/signals.hh" +#include "nix/store/filetransfer.hh" #include #include @@ -37,6 +42,10 @@ # include #endif +#ifdef _WIN32 +# include "nix/util/windows-known-folders.hh" +#endif + #include "store-config-private.hh" namespace nix { @@ -46,38 +55,47 @@ namespace nix { Nix daemon by setting the mode/ownership of the directory appropriately. (This wouldn't work on the socket itself since it must be deleted and recreated on startup.) */ -#define DEFAULT_SOCKET_PATH "/daemon-socket/socket" +#define DEFAULT_SOCKET_PATH "daemon-socket/socket" + +/** + * Helper to resolve the NIX_CONF_DIR at runtime on Windows. + * On Windows, NIX_CONF_DIR is not defined at compile time, so we determine + * the path at runtime using the Windows known folders API (FOLDERID_ProgramData). + * This allows Nix to work correctly regardless of which drive Windows is installed on. + */ +static std::filesystem::path resolveNixConfDir() +{ +#ifdef _WIN32 +# ifdef NIX_CONF_DIR + // On Windows, NIX_CONF_DIR should not be defined at compile time +# error "NIX_CONF_DIR should not be defined on Windows" +# endif + return windows::known_folders::getProgramData() / "nix"; +#else + return NIX_CONF_DIR; +#endif +} + +LogFileSettings::LogFileSettings() + : nixLogDir(canonPath(getEnvNonEmpty("NIX_LOG_DIR").value_or(NIX_LOG_DIR))) +{ +} Settings settings; static GlobalConfig::Register rSettings(&settings); Settings::Settings() - : nixPrefix(NIX_PREFIX) - , nixStore( -#ifndef _WIN32 - // On Windows `/nix/store` is not a canonical path, but we dont' - // want to deal with that yet. - canonPath -#endif - (getEnvNonEmpty("NIX_STORE_DIR").value_or(getEnvNonEmpty("NIX_STORE").value_or(NIX_STORE_DIR)))) - , nixDataDir(canonPath(getEnvNonEmpty("NIX_DATA_DIR").value_or(NIX_DATA_DIR))) - , nixLogDir(canonPath(getEnvNonEmpty("NIX_LOG_DIR").value_or(NIX_LOG_DIR))) - , nixStateDir(canonPath(getEnvNonEmpty("NIX_STATE_DIR").value_or(NIX_STATE_DIR))) - , nixConfDir(canonPath(getEnvNonEmpty("NIX_CONF_DIR").value_or(NIX_CONF_DIR))) - , nixUserConfFiles(getUserConfigFiles()) - , nixDaemonSocketFile( - canonPath(getEnvNonEmpty("NIX_DAEMON_SOCKET_PATH").value_or(nixStateDir + DEFAULT_SOCKET_PATH))) + : nixStateDir(canonPath(getEnvNonEmpty("NIX_STATE_DIR").value_or(NIX_STATE_DIR))) + , nixDaemonSocketFile(canonPath(getEnvOsNonEmpty(OS_STR("NIX_DAEMON_SOCKET_PATH")) + .transform([](auto && s) { return std::filesystem::path(s); }) + .value_or(nixStateDir / DEFAULT_SOCKET_PATH))) { #ifndef _WIN32 buildUsersGroup = isRootUser() ? "nixbld" : ""; #endif allowSymlinkedStore = getEnv("NIX_IGNORE_SYMLINK_STORE") == "1"; - auto sslOverride = getEnv("NIX_SSL_CERT_FILE").value_or(getEnv("SSL_CERT_FILE").value_or("")); - if (sslOverride != "") - caFile = sslOverride; - /* Backwards compatibility. */ auto s = getEnv("NIX_REMOTE_SYSTEMS"); if (s) { @@ -104,29 +122,29 @@ Settings::Settings() }) { sandboxPaths.get().insert_or_assign(std::string{p}, ChrootPath{.source = std::string{p}}); } - allowedImpureHostPrefixes = tokenizeString("/System/Library /usr/lib /dev /bin/sh"); + allowedImpureHostPrefixes = std::set{"/System/Library", "/usr/lib", "/dev", "/bin/sh"}; #endif } void loadConfFile(AbstractConfig & config) { - auto applyConfigFile = [&](const Path & path) { + auto applyConfigFile = [&](const std::filesystem::path & path) { try { std::string contents = readFile(path); - config.applyConfig(contents, path); + config.applyConfig(contents, path.string()); } catch (SystemError &) { } }; - applyConfigFile((settings.nixConfDir / "nix.conf").string()); + applyConfigFile(nixConfFile()); /* We only want to send overrides to the daemon, i.e. stuff from ~/.nix/nix.conf or the command line. */ config.resetOverridden(); - auto files = settings.nixUserConfFiles; + auto files = nixUserConfFiles(); for (auto file = files.rbegin(); file != files.rend(); file++) { - applyConfigFile(*file); + applyConfigFile(file->string()); } auto nixConfEnv = getEnv("NIX_CONFIG"); @@ -135,20 +153,32 @@ void loadConfFile(AbstractConfig & config) } } -std::vector getUserConfigFiles() +const std::filesystem::path & nixConfDir() { - // Use the paths specified in NIX_USER_CONF_FILES if it has been defined - auto nixConfFiles = getEnv("NIX_USER_CONF_FILES"); - if (nixConfFiles.has_value()) { - return tokenizeString>(nixConfFiles.value(), ":"); - } + static const std::filesystem::path dir = + canonPath(getEnvOsNonEmpty(OS_STR("NIX_CONF_DIR")) + .transform([](auto && s) { return std::filesystem::path(s); }) + .value_or(resolveNixConfDir())); + return dir; +} - // Use the paths specified by the XDG spec - std::vector files; - auto dirs = getConfigDirs(); - for (auto & dir : dirs) { - files.insert(files.end(), (dir / "nix.conf").string()); - } +const std::vector & nixUserConfFiles() +{ + static const std::vector files = [] { + // Use the paths specified in NIX_USER_CONF_FILES if it has been defined + auto nixConfFiles = getEnvOs(OS_STR("NIX_USER_CONF_FILES")); + if (nixConfFiles.has_value()) { + return ExecutablePath::parse(*nixConfFiles).directories; + } + + // Use the paths specified by the XDG spec + std::vector files; + auto dirs = getConfigDirs(); + for (auto & dir : dirs) { + files.insert(files.end(), dir / "nix.conf"); + } + return files; + }(); return files; } @@ -251,16 +281,7 @@ bool Settings::isWSL1() #endif } -Path Settings::getDefaultSSLCertFile() -{ - for (auto & fn : - {"/etc/ssl/certs/ca-certificates.crt", "/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt"}) - if (pathAccessible(fn)) - return fn; - return ""; -} - -const ExternalBuilder * Settings::findExternalDerivationBuilderIfSupported(const Derivation & drv) +const ExternalBuilder * LocalSettings::findExternalDerivationBuilderIfSupported(const Derivation & drv) { if (auto it = std::ranges::find_if( externalBuilders.get(), [&](const auto & handler) { return handler.systems.contains(drv.platform); }); @@ -269,7 +290,7 @@ const ExternalBuilder * Settings::findExternalDerivationBuilderIfSupported(const return nullptr; } -std::optional Settings::getHostName() +std::optional WorkerSettings::getHostName() { if (hostName != "") return hostName; @@ -283,6 +304,14 @@ std::optional Settings::getHostName() return std::nullopt; } +ProfileDirsOptions Settings::getProfileDirsOptions() const +{ + return { + .nixStateDir = nixStateDir, + .useXDGBaseDirectories = useXDGBaseDirectories, + }; +} + std::string nixVersion = PACKAGE_VERSION; const std::string determinateNixVersion = DETERMINATE_NIX_VERSION; @@ -353,7 +382,37 @@ void BaseSetting::convertToArg(Args & args, const std::string & cat }); } -NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(ChrootPath, source, optional) +void to_json(nlohmann::json & j, const ChrootPath & cp) +{ + j = nlohmann::json{{"source", cp.source.string()}, {"optional", cp.optional}}; +} + +void from_json(const nlohmann::json & j, ChrootPath & cp) +{ + cp.source = j.at("source").get(); + cp.optional = j.at("optional").get(); +} + +static nlohmann::json pathsInChrootToJSON(const PathsInChroot & paths) +{ + auto j = nlohmann::json::object(); + for (auto & [target, chrootPath] : paths) { + nlohmann::json cp; + to_json(cp, chrootPath); + j[target.string()] = std::move(cp); + } + return j; +} + +template<> +std::map BaseSetting::toJSONObject() const +{ + auto obj = AbstractSetting::toJSONObject(); + obj.emplace("value", pathsInChrootToJSON(value)); + obj.emplace("defaultValue", pathsInChrootToJSON(defaultValue)); + obj.emplace("documentDefault", documentDefault); + return obj; +} template<> PathsInChroot BaseSetting::parse(const std::string & str) const @@ -386,7 +445,9 @@ std::string BaseSetting::to_string() const { std::vector accum; for (auto & [name, cp] : value) { - std::string s = name == cp.source ? name : name + "=" + cp.source; + auto nameStr = name.string(); + auto sourceStr = cp.source.string(); + std::string s = name == cp.source ? nameStr : nameStr + "=" + sourceStr; if (cp.optional) s += "?"; accum.push_back(std::move(s)); @@ -407,37 +468,39 @@ unsigned int MaxBuildJobsSetting::parse(const std::string & str) const } template<> -Settings::ExternalBuilders BaseSetting::parse(const std::string & str) const +LocalSettings::ExternalBuilders BaseSetting::parse(const std::string & str) const { try { - return nlohmann::json::parse(str).template get(); + return nlohmann::json::parse(str).template get(); } catch (std::exception & e) { throw UsageError("parsing setting '%s': %s", name, e.what()); } } template<> -std::string BaseSetting::to_string() const +std::string BaseSetting::to_string() const { return nlohmann::json(value).dump(); } -template<> -std::map BaseSetting>::parse(const std::string & str) const +template +T JSONSetting::parse(const std::string & str) const { try { - return nlohmann::json::parse(str).template get>(); + return nlohmann::json::parse(str).template get(); } catch (std::exception & e) { - throw UsageError("parsing setting '%s': %s", name, e.what()); + throw UsageError("parsing setting '%s': %s", BaseSetting::name, e.what()); } } -template<> -std::string BaseSetting>::to_string() const +template +std::string JSONSetting::to_string() const { - return nlohmann::json(value).dump(); + return nlohmann::json(BaseSetting::get()).dump(); } +template class JSONSetting; + template<> void BaseSetting::appendOrSet(PathsInChroot newValue, bool append) { @@ -446,6 +509,88 @@ void BaseSetting::appendOrSet(PathsInChroot newValue, bool append value.insert(std::make_move_iterator(newValue.begin()), std::make_move_iterator(newValue.end())); } +template<> +struct BaseSetting>::trait +{ + static constexpr bool appendable = true; +}; + +template<> +struct BaseSetting>::trait +{ + static constexpr bool appendable = true; +}; + +template<> +StoreReference BaseSetting::parse(const std::string & str) const +{ + return StoreReference::parse(str); +} + +template<> +std::string BaseSetting::to_string() const +{ + return value.render(); +} + +template<> +std::vector BaseSetting>::parse(const std::string & str) const +{ + std::vector res; + for (const auto & s : tokenizeString(str)) + res.push_back(StoreReference::parse(s)); + return res; +} + +template<> +std::string BaseSetting>::to_string() const +{ + Strings ss; + for (const auto & ref : value) + ss.push_back(ref.render()); + return concatStringsSep(" ", ss); +} + +template<> +void BaseSetting>::appendOrSet(std::vector newValue, bool append) +{ + if (append) + value.insert(value.end(), std::make_move_iterator(newValue.begin()), std::make_move_iterator(newValue.end())); + else + value = std::move(newValue); +} + +template<> +std::set BaseSetting>::parse(const std::string & str) const +{ + std::set res; + for (const auto & s : tokenizeString(str)) + res.insert(StoreReference::parse(s)); + return res; +} + +template<> +std::string BaseSetting>::to_string() const +{ + Strings ss; + for (const auto & ref : value) + ss.push_back(ref.render()); + return concatStringsSep(" ", ss); +} + +template<> +void BaseSetting>::appendOrSet(std::set newValue, bool append) +{ + if (append) + value.insert(std::make_move_iterator(newValue.begin()), std::make_move_iterator(newValue.end())); + else + value = std::move(newValue); +} + +template class BaseSetting; +template class BaseSetting>; +template class BaseSetting>; + static void preloadNSS() { /* builtin:fetchurl can trigger a DNS lookup, which with glibc can trigger a dynamic library load of diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index d4361264edf6..b3678ae4fdf1 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -2,8 +2,10 @@ #include "nix/store/filetransfer.hh" #include "nix/store/globals.hh" #include "nix/store/nar-info-disk-cache.hh" +#include "nix/store/sqlite.hh" #include "nix/util/callback.hh" #include "nix/store/store-registration.hh" +#include "nix/store/globals.hh" namespace nix { @@ -18,15 +20,13 @@ StringSet HttpBinaryCacheStoreConfig::uriSchemes() return ret; } -HttpBinaryCacheStoreConfig::HttpBinaryCacheStoreConfig( - std::string_view scheme, std::string_view _cacheUri, const Params & params) +HttpBinaryCacheStoreConfig::HttpBinaryCacheStoreConfig(ParsedURL _cacheUri, const Params & params) : StoreConfig(params) , BinaryCacheStoreConfig(params) - , cacheUri(parseURL( - std::string{scheme} + "://" - + (!_cacheUri.empty() ? _cacheUri - : throw UsageError("`%s` Store requires a non-empty authority in Store URL", scheme)))) + , cacheUri(std::move(_cacheUri)) { + if (!uriSchemes().contains("file") && (!cacheUri.authority || cacheUri.authority->host.empty())) + throw UsageError("`%s` Store requires a non-empty authority in Store URL", cacheUri.scheme); while (!cacheUri.path.empty() && cacheUri.path.back() == "") cacheUri.path.pop_back(); } @@ -50,12 +50,13 @@ std::string HttpBinaryCacheStoreConfig::doc() ; } -HttpBinaryCacheStore::HttpBinaryCacheStore(ref config) +HttpBinaryCacheStore::HttpBinaryCacheStore(ref config, ref fileTransfer) : Store{*config} // TODO it will actually mutate the configuration , BinaryCacheStore{*config} + , fileTransfer{fileTransfer} , config{config} { - diskCache = getNarInfoDiskCache(); + diskCache = NarInfoDiskCache::get(settings.getNarInfoDiskCacheSettings(), {.useWAL = settings.useSQLiteWAL}); } void HttpBinaryCacheStore::init() @@ -78,13 +79,13 @@ void HttpBinaryCacheStore::init() } } -std::optional HttpBinaryCacheStore::getCompressionMethod(const std::string & path) +std::optional HttpBinaryCacheStore::getCompressionMethod(const std::string & path) { - if (hasSuffix(path, ".narinfo") && !config->narinfoCompression.get().empty()) + if (hasSuffix(path, ".narinfo") && config->narinfoCompression.get()) return config->narinfoCompression; - else if (hasSuffix(path, ".ls") && !config->lsCompression.get().empty()) + else if (hasSuffix(path, ".ls") && config->lsCompression.get()) return config->lsCompression; - else if (hasPrefix(path, "log/") && !config->logCompression.get().empty()) + else if (hasPrefix(path, "log/") && config->logCompression.get()) return config->logCompression; else return std::nullopt; @@ -93,7 +94,7 @@ std::optional HttpBinaryCacheStore::getCompressionMethod(const std: void HttpBinaryCacheStore::maybeDisable() { auto state(_state.lock()); - if (state->enabled && settings.tryFallback) { + if (state->enabled && settings.getWorkerSettings().tryFallback) { int t = 60; printError("disabling binary cache '%s' for %s seconds", config->getHumanReadableURI(), t); state->enabled = false; @@ -121,7 +122,7 @@ bool HttpBinaryCacheStore::fileExists(const std::string & path) try { FileTransferRequest request(makeRequest(path)); request.method = HttpMethod::Head; - getFileTransfer()->download(request); + fileTransfer->download(request); return true; } catch (FileTransferError & e) { /* S3 buckets return 403 if a file doesn't exist and the @@ -151,7 +152,7 @@ void HttpBinaryCacheStore::upload( req.data = {sizeHint, source}; req.mimeType = mimeType; - getFileTransfer()->upload(req); + fileTransfer->upload(req); } void HttpBinaryCacheStore::upsertFile( @@ -160,7 +161,9 @@ void HttpBinaryCacheStore::upsertFile( try { if (auto compressionMethod = getCompressionMethod(path)) { CompressedSource compressed(source, *compressionMethod); - Headers headers = {{"Content-Encoding", *compressionMethod}}; + /* TODO: Validate that this is a valid content encoding. We probably shouldn't set non-standard values here. + */ + Headers headers = {{"Content-Encoding", showCompressionAlgo(*compressionMethod)}}; upload(path, compressed, compressed.size(), mimeType, std::move(headers)); } else { upload(path, source, sizeHint, mimeType, std::nullopt); @@ -193,7 +196,23 @@ FileTransferRequest HttpBinaryCacheStore::makeRequest(std::string_view path) result.query = config->cacheUri.query; } - return FileTransferRequest(result); + FileTransferRequest request(result); + + /* Only use the specified SSL certificate and private key if the resolved URL names the same + authority and uses the same protocol. */ + if (result.scheme == config->cacheUri.scheme && result.authority == config->cacheUri.authority) { + if (const auto & cert = config->tlsCert.get()) { + debug("using TLS client certificate %s for '%s'", PathFmt(*cert), request.uri); + request.tlsCert = *cert; + } + + if (const auto & key = config->tlsKey.get()) { + debug("using TLS client key '%s' for '%s'", PathFmt(*key), request.uri); + request.tlsKey = *key; + } + } + + return request; } void HttpBinaryCacheStore::getFile(const std::string & path, Sink & sink) @@ -201,7 +220,7 @@ void HttpBinaryCacheStore::getFile(const std::string & path, Sink & sink) checkEnabled(); auto request(makeRequest(path)); try { - getFileTransfer()->download(std::move(request), sink); + fileTransfer->download(std::move(request), sink); } catch (FileTransferError & e) { if (e.error == FileTransfer::NotFound || e.error == FileTransfer::Forbidden) throw NoSuchBinaryCacheFile( @@ -220,19 +239,19 @@ void HttpBinaryCacheStore::getFile(const std::string & path, CallbackenqueueFileTransfer(request, {[callbackPtr, this](std::future result) { - try { - (*callbackPtr)(std::move(result.get().data)); - } catch (FileTransferError & e) { - if (e.error == FileTransfer::NotFound - || e.error == FileTransfer::Forbidden) - return (*callbackPtr)({}); - maybeDisable(); - callbackPtr->rethrow(); - } catch (...) { - callbackPtr->rethrow(); - } - }}); + fileTransfer->enqueueFileTransfer(request, {[callbackPtr, this](std::future result) { + try { + (*callbackPtr)(std::move(result.get().data)); + } catch (FileTransferError & e) { + if (e.error == FileTransfer::NotFound + || e.error == FileTransfer::Forbidden) + return (*callbackPtr)({}); + maybeDisable(); + callbackPtr->rethrow(); + } catch (...) { + callbackPtr->rethrow(); + } + }}); } catch (...) { callbackPtr->rethrow(); @@ -243,7 +262,7 @@ void HttpBinaryCacheStore::getFile(const std::string & path, Callback HttpBinaryCacheStore::getNixCacheInfo() { try { - auto result = getFileTransfer()->download(makeRequest(cacheInfoFile)); + auto result = fileTransfer->download(makeRequest(cacheInfoFile)); return result.data; } catch (FileTransferError & e) { if (e.error == FileTransfer::NotFound) @@ -266,11 +285,17 @@ std::optional HttpBinaryCacheStore::isTrustedClient() return std::nullopt; } -ref HttpBinaryCacheStore::Config::openStore() const +ref HttpBinaryCacheStore::Config::openStore(ref fileTransfer) const { return make_ref( ref{// FIXME we shouldn't actually need a mutable config - std::const_pointer_cast(shared_from_this())}); + std::const_pointer_cast(shared_from_this())}, + fileTransfer); +} + +ref HttpBinaryCacheStoreConfig::openStore() const +{ + return openStore(getFileTransfer()); } static RegisterStoreImplementation regHttpBinaryCacheStore; diff --git a/src/libstore/include/nix/store/active-builds.hh b/src/libstore/include/nix/store/active-builds.hh index c8a40e137986..2dc914f35606 100644 --- a/src/libstore/include/nix/store/active-builds.hh +++ b/src/libstore/include/nix/store/active-builds.hh @@ -32,7 +32,7 @@ struct ActiveBuild pid_t mainPid; UserInfo mainUser; - std::optional cgroup; + std::optional cgroup; time_t startTime; @@ -88,6 +88,8 @@ struct TrackActiveBuildsStore } }; + virtual ~TrackActiveBuildsStore() = default; + virtual BuildHandle buildStarted(const ActiveBuild & build) = 0; virtual void buildFinished(const BuildHandle & handle) = 0; @@ -97,6 +99,8 @@ struct QueryActiveBuildsStore { inline static std::string operationName = "Querying active builds"; + virtual ~QueryActiveBuildsStore() = default; + virtual std::vector queryActiveBuilds() = 0; }; diff --git a/src/libstore/include/nix/store/async-path-writer.hh b/src/libstore/include/nix/store/async-path-writer.hh index d64418479bc6..695321ccb1f3 100644 --- a/src/libstore/include/nix/store/async-path-writer.hh +++ b/src/libstore/include/nix/store/async-path-writer.hh @@ -6,12 +6,13 @@ namespace nix { struct AsyncPathWriter { + virtual ~AsyncPathWriter() = default; + virtual StorePath addPath( std::string contents, std::string name, StorePathSet references, RepairFlag repair, - bool readOnly = false, std::shared_ptr provenance = {}) = 0; virtual void waitForPath(const StorePath & path) = 0; diff --git a/src/libstore/include/nix/store/aws-creds.hh b/src/libstore/include/nix/store/aws-creds.hh index 30f6592a06f8..0751757cb015 100644 --- a/src/libstore/include/nix/store/aws-creds.hh +++ b/src/libstore/include/nix/store/aws-creds.hh @@ -34,12 +34,13 @@ struct AwsCredentials } }; -class AwsAuthError : public Error +class AwsAuthError final : public CloneableError { std::optional errorCode; public: - using Error::Error; + using CloneableError::CloneableError; + AwsAuthError(int errorCode); std::optional getErrorCode() const diff --git a/src/libstore/include/nix/store/binary-cache-store.hh b/src/libstore/include/nix/store/binary-cache-store.hh index 4cb7b23f2d31..fbd347e7684f 100644 --- a/src/libstore/include/nix/store/binary-cache-store.hh +++ b/src/libstore/include/nix/store/binary-cache-store.hh @@ -1,7 +1,7 @@ #pragma once ///@file -#include "nix/util/signature/local-keys.hh" +#include "nix/util/compression-settings.hh" #include "nix/store/store-api.hh" #include "nix/store/log-store.hh" @@ -18,13 +18,19 @@ struct BinaryCacheStoreConfig : virtual StoreConfig { using StoreConfig::StoreConfig; - const Setting compression{ - this, "xz", "compression", "NAR compression method (`xz`, `bzip2`, `gzip`, `zstd`, or `none`)."}; + Setting compression{ + this, + CompressionAlgo::xz, + "compression", + R"( + NAR compression method. One of: `xz`, `bzip2`, `gzip`, `zstd`, `none`, `br`, `compress`, `grzip`, `lrzip`, `lz4`, `lzip`, `lzma` or `lzop`. + To use a particular compression method Nix has to be built with a version of libarchive that natively supports that compression algorithm. + )"}; - const Setting writeNARListing{ + Setting writeNARListing{ this, false, "write-nar-listing", "Whether to write a JSON file that lists the files in each NAR."}; - const Setting writeDebugInfo{ + Setting writeDebugInfo{ this, false, "index-debug-info", @@ -33,24 +39,25 @@ struct BinaryCacheStoreConfig : virtual StoreConfig fetch debug info on demand )"}; - const Setting secretKeyFile{this, "", "secret-key", "Path to the secret key used to sign the binary cache."}; + Setting secretKeyFile{ + this, "", "secret-key", "Path to the secret key used to sign the binary cache."}; - const Setting secretKeyFiles{ + Setting secretKeyFiles{ this, "", "secret-keys", "List of comma-separated paths to the secret keys used to sign the binary cache."}; - const Setting localNarCache{ + Setting> localNarCache{ this, - "", + std::nullopt, "local-nar-cache", "Path to a local cache of NARs fetched from this binary cache, used by commands such as `nix store cat`."}; - const Setting parallelCompression{ + Setting parallelCompression{ this, false, "parallel-compression", "Enable multi-threaded compression of NARs. This is currently only available for `xz` and `zstd`."}; - const Setting compressionLevel{ + Setting compressionLevel{ this, -1, "compression-level", @@ -159,10 +166,7 @@ private: void writeNarInfo(ref narInfo); ref addToStoreCommon( - Source & narSource, - RepairFlag repair, - CheckSigsFlag checkSigs, - std::function mkInfo); + Source & narSource, RepairFlag repair, CheckSigsFlag checkSigs, fun mkInfo); /** * Same as `getFSAccessor`, but with a more preceise return type. @@ -211,7 +215,7 @@ public: std::shared_ptr getFSAccessor(const StorePath &, bool requireValidPath = true) override; - void addSignatures(const StorePath & storePath, const StringSet & sigs) override; + void addSignatures(const StorePath & storePath, const std::set & sigs) override; std::optional getBuildLogExact(const StorePath & path) override; diff --git a/src/libstore/include/nix/store/build-result.hh b/src/libstore/include/nix/store/build-result.hh index e6cbd1f73ccd..ef2d5ecc817c 100644 --- a/src/libstore/include/nix/store/build-result.hh +++ b/src/libstore/include/nix/store/build-result.hh @@ -7,32 +7,131 @@ #include "nix/store/derived-path.hh" #include "nix/store/realisation.hh" +#include "nix/util/error.hh" +#include "nix/util/fmt.hh" #include "nix/util/json-impls.hh" namespace nix { struct Provenance; +/** + * Names must be disjoint with `BuildResultFailureStatus`. + * + * @note Prefer using `BuildResult::Success::Status`, this name is just + * for sake of forward declarations. + */ +enum struct BuildResultSuccessStatus : uint8_t { + Built, + Substituted, + AlreadyValid, + ResolvesToAlreadyValid, +}; + +/** + * Names must be disjoint with `BuildResultSuccessStatus`. + * + * @note Prefer using `BuildResult::Failure::Status`, this name is just + * for sake of forward declarations. + */ +enum struct BuildResultFailureStatus : uint8_t { + PermanentFailure, + InputRejected, + OutputRejected, + /// possibly transient + TransientFailure, + /// no longer used + CachedFailure, + TimedOut, + MiscFailure, + DependencyFailed, + LogLimitExceeded, + NotDeterministic, + NoSubstituters, + /// A certain type of `OutputRejected`. The protocols do not yet + /// know about this one, so change it back to `OutputRejected` + /// before serialization. + HashMismatch, + Cancelled, +}; + +/** + * Denotes a permanent build failure. + * + * This is both an exception type (inherits from Error) and serves as + * the failure variant in BuildResult::inner. + */ +struct BuildError : public CloneableError +{ + using Status = BuildResultFailureStatus; + using enum Status; + + Status status = MiscFailure; + + /** + * If timesBuilt > 1, whether some builds did not produce the same + * result. (Note that 'isNonDeterministic = false' does not mean + * the build is deterministic, just that we don't have evidence of + * non-determinism.) + */ + bool isNonDeterministic = false; + + /** + * The provenance of the derivation, if any. + */ + std::shared_ptr provenance; + +public: + /** + * Variadic constructor for throwing with format strings. + * Delegates to the string constructor after formatting. + */ + template + BuildError(Status status, const Args &... args) + : CloneableError(args...) + , status{status} + { + } + + struct Args + { + Status status; + HintFmt msg; + bool isNonDeterministic = false; + std::shared_ptr provenance; + }; + + /** + * Constructor taking a pre-formatted error message. + * Also used for deserialization. + */ + BuildError(Args args) + : CloneableError(std::move(args.msg)) + , status{args.status} + , isNonDeterministic{args.isNonDeterministic} + , provenance{args.provenance} + { + } + + /** + * Default constructor for deserialization. + */ + BuildError() + : CloneableError("") + { + } + + bool operator==(const BuildError &) const noexcept; + std::strong_ordering operator<=>(const BuildError &) const noexcept; +}; + struct BuildResult { struct Success { - /** - * @note This is directly used in the nix-store --serve protocol. - * That means we need to worry about compatibility across versions. - * Therefore, don't remove status codes, and only add new status - * codes at the end of the list. - * - * Must be disjoint with `Failure::Status`. - */ - enum Status : uint8_t { - Built = 0, - Substituted = 1, - AlreadyValid = 2, - ResolvesToAlreadyValid = 13, - } status; - - static std::string_view statusToString(Status status); + using Status = enum BuildResultSuccessStatus; + using enum Status; + Status status; /** * For derivations, a mapping from the names of the wanted outputs @@ -48,76 +147,12 @@ struct BuildResult bool operator==(const BuildResult::Success &) const noexcept; std::strong_ordering operator<=>(const BuildResult::Success &) const noexcept; - - static bool statusIs(uint8_t status) - { - return status == Built || status == Substituted || status == AlreadyValid - || status == ResolvesToAlreadyValid; - } }; - struct Failure - { - /** - * @note This is directly used in the nix-store --serve protocol. - * That means we need to worry about compatibility across versions. - * Therefore, don't remove status codes, and only add new status - * codes at the end of the list. - * - * Must be disjoint with `Success::Status`. - */ - enum Status : uint8_t { - PermanentFailure = 3, - InputRejected = 4, - OutputRejected = 5, - /// possibly transient - TransientFailure = 6, - /// no longer used - CachedFailure = 7, - TimedOut = 8, - MiscFailure = 9, - DependencyFailed = 10, - LogLimitExceeded = 11, - NotDeterministic = 12, - NoSubstituters = 14, - /// A certain type of `OutputRejected`. The protocols do not yet - /// know about this one, so change it back to `OutputRejected` - /// before serialization. - HashMismatch = 15, - Cancelled = 16, - } status = MiscFailure; - - static std::string_view statusToString(Status status); - - /** - * Information about the error if the build failed. - * - * @todo This should be an entire ErrorInfo object, not just a - * string, for richer information. - */ - std::string errorMsg; - - /** - * If timesBuilt > 1, whether some builds did not produce the same - * result. (Note that 'isNonDeterministic = false' does not mean - * the build is deterministic, just that we don't have evidence of - * non-determinism.) - */ - bool isNonDeterministic = false; - - /** - * The provenance of the derivation, if any. - */ - std::shared_ptr provenance; - - bool operator==(const BuildResult::Failure &) const noexcept; - std::strong_ordering operator<=>(const BuildResult::Failure &) const noexcept; - - [[noreturn]] void rethrow() const - { - throw Error("%s", errorMsg.empty() ? statusToString(status) : errorMsg); - } - }; + /** + * Failure is now an alias for BuildError. + */ + using Failure = BuildError; std::variant inner = Failure{}; @@ -141,6 +176,19 @@ struct BuildResult return std::get_if(&self.inner); } + /** + * Throw the build error if this result represents a failure. + * Optionally set the exit status on the error before throwing. + */ + void tryThrowBuildError(std::optional exitStatus = std::nullopt) + { + if (auto * failure = tryGetFailure()) { + if (exitStatus) + failure->withExitStatus(*exitStatus); + throw *failure; + } + } + /** * How many times this build was performed. */ @@ -168,20 +216,6 @@ struct BuildResult } }; -/** - * denotes a permanent build failure - */ -struct BuildError : public Error -{ - BuildResult::Failure::Status status; - - BuildError(BuildResult::Failure::Status status, auto &&... args) - : Error{args...} - , status{status} - { - } -}; - /** * A `BuildResult` together with its "primary key". */ @@ -200,6 +234,61 @@ struct KeyedBuildResult : BuildResult } }; +/** + * Flags tracking different types of build failures for exit status computation. + */ +struct ExitStatusFlags +{ + /** + * Set if at least one derivation had a BuildError (i.e. permanent + * failure). + */ + bool permanentFailure = false; + + /** + * Set if at least one derivation had a timeout. + */ + bool timedOut = false; + + /** + * Set if at least one derivation fails with a hash mismatch. + */ + bool hashMismatch = false; + + /** + * Set if at least one derivation is not deterministic in check mode. + */ + bool checkMismatch = false; + + /** + * Update flags based on a build failure status. + */ + void updateFromStatus(BuildResult::Failure::Status status); + + /** + * The exit status in case of failure. + * + * In the case of a build failure, returned value follows this + * bitmask: + * + * ``` + * 0b1100100 + * ^^^^ + * |||`- timeout + * ||`-- output hash mismatch + * |`--- build failure + * `---- not deterministic + * ``` + * + * In other words, the failure code is at least 100 (0b1100100), but + * might also be greater. + * + * Otherwise (no build failure, but some other sort of failure by + * assumption), this returned value is 1. + */ + unsigned int failingExitStatus() const; +}; + } // namespace nix JSON_IMPL(nix::BuildResult) diff --git a/src/libstore/include/nix/store/build/build-log.hh b/src/libstore/include/nix/store/build/build-log.hh new file mode 100644 index 000000000000..4fd71a65f5d2 --- /dev/null +++ b/src/libstore/include/nix/store/build/build-log.hh @@ -0,0 +1,85 @@ +#pragma once +///@file + +#include "nix/util/logging.hh" +#include "nix/util/serialise.hh" +#include "nix/util/ref.hh" + +#include +#include +#include + +namespace nix { + +/** + * Line buffering and log tracking for build output. + * + * This class handles: + * - Owning the build Activity for logging + * - Buffering partial lines (handling \r and \n) + * - Maintaining a tail of recent log lines (for error messages) + * - Processing JSON log messages via handleJSONLogMessage + * + * Implements Sink so it can be used as a data destination. + * I/O is handled separately by the caller. + */ +struct BuildLog : Sink +{ +private: + size_t maxTailLines; + + std::list logTail; + std::string currentLogLine; + size_t currentLogLinePos = 0; // to handle carriage return + + void flushLine(); + +public: + /** + * The build activity. + */ + ref act; + + /** + * Map for tracking nested activities from JSON messages. + */ + std::map builderActivities; + + /** + * @param maxTailLines Maximum number of tail lines to keep + * @param act Activity for this build + */ + BuildLog(size_t maxTailLines, ref act); + + /** + * Process output data from child process. + * Handles JSON log messages and emits regular lines to activity. + * @param data Raw output data from child + */ + void operator()(std::string_view data) override; + + /** + * Flush any remaining partial line. + * Call this when the child process exits. + */ + void flush(); + + /** + * Get the most recent log lines. + * Used for including in error messages. + */ + const std::list & getTail() const + { + return logTail; + } + + /** + * Check if there's an incomplete line buffered. + */ + bool hasPartialLine() const + { + return !currentLogLine.empty(); + } +}; + +} // namespace nix diff --git a/src/libstore/include/nix/store/build/derivation-builder.hh b/src/libstore/include/nix/store/build/derivation-builder.hh index 8ff8613b689e..3b5b8504966a 100644 --- a/src/libstore/include/nix/store/build/derivation-builder.hh +++ b/src/libstore/include/nix/store/build/derivation-builder.hh @@ -1,6 +1,7 @@ #pragma once ///@file +#include #include #include "nix/store/build-result.hh" @@ -19,14 +20,14 @@ namespace nix { * Denotes a build failure that stemmed from the builder exiting with a * failing exist status. */ -struct BuilderFailureError : BuildError +struct BuilderFailureError final : CloneableError { int builderStatus; std::string extraMsgAfter; BuilderFailureError(BuildResult::Failure::Status status, int builderStatus, std::string extraMsgAfter) - : BuildError{ + : CloneableError{ status, /* No message for now, because the caller will make for us, with extra context */ @@ -43,11 +44,11 @@ struct BuilderFailureError : BuildError */ struct ChrootPath { - Path source; + std::filesystem::path source; bool optional = false; }; -typedef std::map PathsInChroot; // maps target path to source path +typedef std::map PathsInChroot; // maps target path to source path /** * Parameters by (mostly) `const` reference for `DerivationBuilder`. @@ -107,7 +108,7 @@ struct DerivationBuilderParams /** * The activity corresponding to the build. */ - std::unique_ptr & act; + ref act; }; /** @@ -120,7 +121,7 @@ struct DerivationBuilderCallbacks /** * Open a log file and a pipe to it. */ - virtual Path openLogFile() = 0; + virtual void openLogFile() = 0; /** * Close the log file. @@ -192,10 +193,23 @@ struct DerivationBuilder : RestrictionContext virtual bool killChild() = 0; }; +/** + * Run a callback that may change process credentials (setuid, setgid, etc.) + * while preserving the parent-death signal. + * + * The parent-death signal setting is cleared by the Linux kernel upon changes + * to EUID, EGID. + * + * @note Does nothing on non-Linux systems. + * @see man PR_SET_PDEATHSIG + * @see https://github.com/golang/go/issues/9686 + */ +void preserveDeathSignal(fun setCredentials); + struct ExternalBuilder { StringSet systems; - Path program; + std::filesystem::path program; std::vector args; }; diff --git a/src/libstore/include/nix/store/build/derivation-building-goal.hh b/src/libstore/include/nix/store/build/derivation-building-goal.hh index 75ca43b41f91..615b5f04ea38 100644 --- a/src/libstore/include/nix/store/build/derivation-building-goal.hh +++ b/src/libstore/include/nix/store/build/derivation-building-goal.hh @@ -2,20 +2,21 @@ ///@file #include "nix/store/derivations.hh" +#include "nix/store/local-store.hh" #include "nix/store/parsed-derivations.hh" #include "nix/store/derivation-options.hh" #include "nix/store/build/derivation-building-misc.hh" -#include "nix/store/build/derivation-builder.hh" -#include "nix/store/outputs-spec.hh" #include "nix/store/store-api.hh" #include "nix/store/pathlocks.hh" #include "nix/store/build/goal.hh" +#include "nix/store/build/build-log.hh" namespace nix { using std::map; struct BuilderFailureError; +struct ExternalBuilder; #ifndef _WIN32 // TODO enable build hook on Windows struct HookInstance; struct DerivationBuilder; @@ -46,95 +47,52 @@ struct DerivationBuildingGoal : public Goal private: /** The path of the derivation. */ - StorePath drvPath; + const StorePath drvPath; /** * The derivation stored at drvPath. */ - std::unique_ptr drv; + const std::unique_ptr drv; /** * The remainder is state held during the build. */ - /** - * All input paths (that is, the union of FS closures of the - * immediate input paths). - */ - StorePathSet inputPaths; - - /** - * File descriptor for the log file. - */ - AutoCloseFD fdLogFile; - std::shared_ptr logFileSink, logSink; - - /** - * Number of bytes received from the builder's stdout/stderr. - */ - unsigned long logSize; - - /** - * The most recent log lines. - */ - std::list logTail; - - std::string currentLogLine; - size_t currentLogLinePos = 0; // to handle carriage return - - std::string currentHookLine; - -#ifndef _WIN32 // TODO enable build hook on Windows - /** - * The build hook. - */ - std::unique_ptr hook; - - DerivationBuilderUnique builder; -#endif - - BuildMode buildMode; + const BuildMode buildMode; std::unique_ptr> mcRunningBuilds; - std::unique_ptr act; - - std::map builderActivities; - - void timedOut(Error && ex) override; - std::string key() override; + struct LocalBuildCapability + { + LocalStore & localStore; + const ExternalBuilder * externalBuilder; + }; + /** * The states. */ Co gaveUpOnSubstitution(bool storeDerivation); - Co tryToBuild(); + Co tryToBuild(StorePathSet inputPaths); + Co buildWithHook( + StorePathSet inputPaths, + std::map initialOutputs, + DerivationOptions drvOptions, + PathLocks outputLocks); + Co buildLocally( + LocalBuildCapability localBuildCap, + StorePathSet inputPaths, + std::map initialOutputs, + DerivationOptions drvOptions, + PathLocks outputLocks); /** * Is the build hook willing to perform the build? */ - HookReply tryBuildHook( - const std::map & initialOutputs, const DerivationOptions & drvOptions); + HookReply tryBuildHook(const DerivationOptions & drvOptions); - /** - * Open a log file and a pipe to it. - */ - Path openLogFile(); - - /** - * Close the log file. - */ - void closeLogFile(); - - bool isReadDesc(Descriptor fd); - - /** - * Callback used by the worker to write to the log. - */ - void handleChildOutput(Descriptor fd, std::string_view data) override; - void handleEOF(Descriptor fd) override; - void flushLine(); + Done doneFailureLogTooLong(BuildLog & buildLog); /** * Wrappers around the corresponding Store methods that first consult the @@ -151,11 +109,6 @@ private: */ std::pair checkPathValidity(std::map & initialOutputs); - /** - * Forcibly kill the child process, if any. - */ - void killChild(); - Done doneSuccess( BuildResult::Success::Status status, SingleDrvOutputs builtOutputs, @@ -163,7 +116,7 @@ private: Done doneFailure(BuildError ex); - BuildError fixupBuilderFailureErrorMessage(BuilderFailureError msg); + BuildError fixupBuilderFailureErrorMessage(BuilderFailureError msg, BuildLog & buildLog); JobCategory jobCategory() const override { diff --git a/src/libstore/include/nix/store/build/derivation-goal.hh b/src/libstore/include/nix/store/build/derivation-goal.hh index 0fe610987fca..aaded75511f0 100644 --- a/src/libstore/include/nix/store/build/derivation-goal.hh +++ b/src/libstore/include/nix/store/build/derivation-goal.hh @@ -52,11 +52,6 @@ struct DerivationGoal : public Goal bool storeDerivation); ~DerivationGoal() = default; - void timedOut(Error && ex) override - { - unreachable(); - }; - std::string key() override; JobCategory jobCategory() const override diff --git a/src/libstore/include/nix/store/build/derivation-resolution-goal.hh b/src/libstore/include/nix/store/build/derivation-resolution-goal.hh index fb4c2a34635f..b79e6bbb79e4 100644 --- a/src/libstore/include/nix/store/build/derivation-resolution-goal.hh +++ b/src/libstore/include/nix/store/build/derivation-resolution-goal.hh @@ -43,8 +43,6 @@ struct DerivationResolutionGoal : public Goal */ std::unique_ptr> resolvedDrv; - void timedOut(Error && ex) override {} - private: /** diff --git a/src/libstore/include/nix/store/build/derivation-trampoline-goal.hh b/src/libstore/include/nix/store/build/derivation-trampoline-goal.hh index bfed67f63706..33276723a7e0 100644 --- a/src/libstore/include/nix/store/build/derivation-trampoline-goal.hh +++ b/src/libstore/include/nix/store/build/derivation-trampoline-goal.hh @@ -109,8 +109,6 @@ struct DerivationTrampolineGoal : public Goal virtual ~DerivationTrampolineGoal(); - void timedOut(Error && ex) override {} - std::string key() override; JobCategory jobCategory() const override diff --git a/src/libstore/include/nix/store/build/drv-output-substitution-goal.hh b/src/libstore/include/nix/store/build/drv-output-substitution-goal.hh index 6310e0d2ccca..5f36bbf06d8b 100644 --- a/src/libstore/include/nix/store/build/drv-output-substitution-goal.hh +++ b/src/libstore/include/nix/store/build/drv-output-substitution-goal.hh @@ -14,11 +14,14 @@ namespace nix { class Worker; /** - * Substitution of a derivation output. - * This is done in three steps: - * 1. Fetch the output info from a substituter - * 2. Substitute the corresponding output path - * 3. Register the output info + * Fetch a `Realisation` (drv ⨯ output name -> output path) from a + * substituter. + * + * If the output store object itself should also be substituted, that is + * the responsibility of the caller to do so. + * + * @todo rename this `BuidlTraceEntryGoal`, which will make sense + * especially once `Realisation` is renamed to `BuildTraceEntry`. */ class DrvOutputSubstitutionGoal : public Goal { @@ -31,17 +34,16 @@ class DrvOutputSubstitutionGoal : public Goal public: DrvOutputSubstitutionGoal(const DrvOutput & id, Worker & worker); - Co init(); + /** + * The realisation corresponding to the given output id. + * Will be filled once we can get it. + */ + std::shared_ptr outputInfo; - void timedOut(Error && ex) override - { - unreachable(); - }; + Co init(); std::string key() override; - void handleEOF(Descriptor fd) override; - JobCategory jobCategory() const override { return JobCategory::Substitution; diff --git a/src/libstore/include/nix/store/build/goal.hh b/src/libstore/include/nix/store/build/goal.hh index d26803532dbe..7706ad20b4d3 100644 --- a/src/libstore/include/nix/store/build/goal.hh +++ b/src/libstore/include/nix/store/build/goal.hh @@ -5,9 +5,18 @@ #include "nix/store/build-result.hh" #include +#include +#include namespace nix { +struct TimedOut final : CloneableError +{ + time_t maxDuration; + + TimedOut(time_t maxDuration); +}; + /** * Forward definition. */ @@ -109,7 +118,7 @@ public: /** * Build result. */ - BuildResult buildResult = {.inner = BuildResult::Failure{.status = BuildResult::Failure::Cancelled}}; + BuildResult buildResult{BuildError(BuildResult::Failure::Cancelled, "")}; /** * Suspend our goal and wait until we get `work`-ed again. @@ -138,6 +147,29 @@ public: friend Goal; }; + /** + * Event types for child process communication, delivered via coroutines. + */ + struct ChildOutput + { + Descriptor fd; + std::string data; + }; + + struct ChildEOF + { + Descriptor fd; + }; + + using ChildEvent = std::variant; + + /** + * Tag type for `co_await`-ing child events. + * Returns a `ChildEvent` when resumed. + */ + struct WaitForChildEvent + {}; + // forward declaration of promise_type, see below struct promise_type; @@ -276,6 +308,28 @@ public: */ bool alive = true; + class + { + /** + * Structured queue of child events: + * - outputs: stream of data from child + * - eof: optional end-of-stream marker + * - timeout: optional timeout that flushes/overrides other events + */ + std::queue childOutputs; + std::optional childEOF; + std::optional childTimeout; + + public: + + void pushChildEvent(ChildOutput event); + void pushChildEvent(ChildEOF event); + void pushChildEvent(TimedOut event); + bool hasChildEvent() const; + ChildEvent popChildEvent(); + + } childEvents; + /** * The awaiter used by @ref final_suspend. */ @@ -369,13 +423,66 @@ public: return static_cast(co); } + /** + * Awaiter for @ref Suspend. Always suspends, but asserts + * there are no pending child events (those should be + * consumed first via @ref WaitForChildEvent). + */ + struct SuspendAwaiter + { + promise_type & promise; + + bool await_ready() + { + assert(!promise.childEvents.hasChildEvent()); + return false; + } + + void await_suspend(handle_type) {} + + void await_resume() {} + }; + /** * Allows awaiting a @ref Suspend. * Always suspends. */ - std::suspend_always await_transform(Suspend) + SuspendAwaiter await_transform(Suspend) { - return {}; + return SuspendAwaiter{*this}; + }; + + /** + * Awaiter for child events. Suspends and returns the + * pending child event when resumed. + */ + struct ChildEventAwaiter + { + handle_type handle; + + bool await_ready() + { + return handle && handle.promise().childEvents.hasChildEvent(); + } + + void await_suspend(handle_type h) + { + handle = h; + } + + ChildEvent await_resume() + { + assert(handle); + return handle.promise().childEvents.popChildEvent(); + } + }; + + /** + * Allows awaiting child events (output, EOF, timeout). + */ + ChildEventAwaiter await_transform(WaitForChildEvent) + { + return ChildEventAwaiter{handle_type::from_promise(*this)}; }; }; @@ -397,7 +504,7 @@ protected: * Prefer using `doneSuccess` or `doneFailure` instead, which ensure * `buildResult` is set correctly. */ - Done amDone(ExitCode result, std::optional ex = {}); + Done amDone(ExitCode result); /** * Signals successful completion of the goal. @@ -411,15 +518,14 @@ protected: * * @param result The exit code (ecFailed or ecNoSubstituters) * @param failure The failure details including status and error message - * @param ex Optional exception to store/log */ - Done doneFailure(ExitCode result, BuildResult::Failure failure, std::optional ex = {}); + Done doneFailure(ExitCode result, BuildResult::Failure failure); public: virtual void cleanup() {} /** - * Hack to say that this goal should not log `ex`, but instead keep + * Hack to say that this goal should not log the failure, but instead keep * it around. Set by a waitee which sees itself as the designated * continuation of this goal, responsible for reporting its * successes or failures. @@ -427,12 +533,7 @@ public: * @todo this is yet another not-nice hack in the goal system that * we ought to get rid of. See #11927 */ - bool preserveException = false; - - /** - * Exception containing an error message, if any. - */ - std::optional ex; + bool preserveFailure = false; Goal(Worker & worker, Co init) : worker(worker) @@ -451,15 +552,23 @@ public: void work(); - virtual void handleChildOutput(Descriptor fd, std::string_view data) - { - unreachable(); - } + /** + * Called by the worker when data is received from a child process. + * Stores the event and resumes the coroutine. + */ + void handleChildOutput(Descriptor fd, std::string_view data); - virtual void handleEOF(Descriptor fd) - { - unreachable(); - } + /** + * Called by the worker when EOF is received from a child process. + * Stores the event and resumes the coroutine. + */ + void handleEOF(Descriptor fd); + + /** + * Called by the worker when a build times out. + * Stores the event and resumes the coroutine. + */ + void timedOut(TimedOut && ex); void trace(std::string_view s); @@ -468,13 +577,6 @@ public: return name; } - /** - * Callback in case of a timeout. It should wake up its waiters, - * get rid of any running child processes that are being monitored - * by the worker (important!), etc. - */ - virtual void timedOut(Error && ex) = 0; - /** * Used for comparisons. The order matters a bit for scheduling. We * want: diff --git a/src/libstore/include/nix/store/build/substitution-goal.hh b/src/libstore/include/nix/store/build/substitution-goal.hh index 1b7d956a1a2e..84ddcc8ab19c 100644 --- a/src/libstore/include/nix/store/build/substitution-goal.hh +++ b/src/libstore/include/nix/store/build/substitution-goal.hh @@ -41,10 +41,6 @@ struct PathSubstitutionGoal : public Goal */ std::optional ca; - Done doneSuccess(BuildResult::Success::Status status, std::shared_ptr provenance); - - Done doneFailure(ExitCode result, BuildResult::Failure::Status status, std::string errorMsg); - public: PathSubstitutionGoal( const StorePath & storePath, @@ -53,11 +49,6 @@ public: std::optional ca = std::nullopt); ~PathSubstitutionGoal(); - void timedOut(Error && ex) override - { - unreachable(); - }; - std::string key() override { return "a$" + std::string(storePath.name()) + "$" + worker.store.printStorePath(storePath); @@ -72,12 +63,6 @@ public: StorePath subPath, nix::ref sub, std::shared_ptr info, bool & substituterFailed); Co finished(); - /** - * Callback used by the worker to write to the log. - */ - void handleChildOutput(Descriptor fd, std::string_view data) override {}; - void handleEOF(Descriptor fd) override; - /* Called by destructor, can't be overridden */ void cleanup() override final; @@ -85,6 +70,8 @@ public: { return JobCategory::Substitution; }; + + Done doneFailure(ExitCode result, BuildResult::Failure failure); }; } // namespace nix diff --git a/src/libstore/include/nix/store/build/worker.hh b/src/libstore/include/nix/store/build/worker.hh index 173f7b222b74..baf5c5a4d63e 100644 --- a/src/libstore/include/nix/store/build/worker.hh +++ b/src/libstore/include/nix/store/build/worker.hh @@ -5,15 +5,18 @@ #include "nix/store/store-api.hh" #include "nix/store/derived-path-map.hh" #include "nix/store/build/goal.hh" +#include "nix/store/build-result.hh" #include "nix/store/realisation.hh" #include "nix/util/muxable-pipe.hh" +#include #include #include namespace nix { /* Forward definition. */ +struct WorkerSettings; struct DerivationTrampolineGoal; struct DerivationGoal; struct DerivationResolutionGoal; @@ -144,25 +147,9 @@ public: const Activity actSubstitutions; /** - * Set if at least one derivation had a BuildError (i.e. permanent - * failure). + * Tracks different types of build failures for exit status computation. */ - bool permanentFailure; - - /** - * Set if at least one derivation had a timeout. - */ - bool timedOut; - - /** - * Set if at least one derivation fails with a hash mismatch. - */ - bool hashMismatch; - - /** - * Set if at least one derivation is not deterministic in check mode. - */ - bool checkMismatch; + ExitStatusFlags exitStatusFlags; #ifdef _WIN32 AutoCloseFD ioport; @@ -170,6 +157,15 @@ public: Store & store; Store & evalStore; + const WorkerSettings & settings; + + /** + * Function to get the substituters to use for path substitution. + * + * Defaults to `getDefaultSubstituters`. This allows tests to + * inject custom substituters. + */ + fun>()> getSubstituters; #ifndef _WIN32 // TODO Enable building on Windows std::unique_ptr hook; @@ -286,9 +282,21 @@ public: * false if there is no sense in waking up goals that are sleeping * because they can't run yet (e.g., there is no free build slot, * or the hook would still say `postpone`). + * + * This overload requires `goal` to point to a fully constructed, + * valid goal object, as it calls `goal->jobCategory()`. */ void childTerminated(Goal * goal, bool wakeSleepers = true); + /** + * Unregisters a running child process, like the other overload. + * + * This overload only uses `goal` as a pointer for comparison with + * weak goal references, so it is safe to call from destructors + * where the goal object may be partially destroyed. + */ + void childTerminated(Goal * goal, JobCategory jobCategory, bool wakeSleepers = true); + /** * Put `goal` to sleep until a build slot becomes available (which * might be right away). @@ -319,29 +327,6 @@ public: */ void waitForInput(); - /*** - * The exit status in case of failure. - * - * In the case of a build failure, returned value follows this - * bitmask: - * - * ``` - * 0b1100100 - * ^^^^ - * |||`- timeout - * ||`-- output hash mismatch - * |`--- build failure - * `---- not deterministic - * ``` - * - * In other words, the failure code is at least 100 (0b1100100), but - * might also be greater. - * - * Otherwise (no build failure, but some other sort of failure by - * assumption), this returned value is 1. - */ - unsigned int failingExitStatus(); - /** * Check whether the given valid path exists and has the right * contents. diff --git a/src/libstore/include/nix/store/builtins.hh b/src/libstore/include/nix/store/builtins.hh index fee11e59e9f3..feeb6b49897c 100644 --- a/src/libstore/include/nix/store/builtins.hh +++ b/src/libstore/include/nix/store/builtins.hh @@ -17,10 +17,11 @@ struct StructuredAttrs; struct BuiltinBuilderContext { const BasicDerivation & drv; - std::map outputs; + std::map outputs; std::string netrcData; std::string caFileData; - Path tmpDirInSandbox; + Strings hashedMirrors; + std::filesystem::path tmpDirInSandbox; #if NIX_WITH_AWS_AUTH /** @@ -31,7 +32,7 @@ struct BuiltinBuilderContext #endif }; -using BuiltinBuilder = std::function; +using BuiltinBuilder = fun; struct RegisterBuiltinBuilder { @@ -39,9 +40,9 @@ struct RegisterBuiltinBuilder static BuiltinBuilders & builtinBuilders(); - RegisterBuiltinBuilder(const std::string & name, BuiltinBuilder && fun) + RegisterBuiltinBuilder(const std::string & name, BuiltinBuilder && builder) { - builtinBuilders().insert_or_assign(name, std::move(fun)); + builtinBuilders().insert_or_assign(name, std::move(builder)); } }; diff --git a/src/libstore/include/nix/store/builtins/buildenv.hh b/src/libstore/include/nix/store/builtins/buildenv.hh index c152ab00af5a..a0f0b3f24b99 100644 --- a/src/libstore/include/nix/store/builtins/buildenv.hh +++ b/src/libstore/include/nix/store/builtins/buildenv.hh @@ -2,6 +2,7 @@ ///@file #include "nix/store/store-api.hh" +#include "nix/util/fmt.hh" namespace nix { @@ -10,11 +11,11 @@ namespace nix { */ struct Package { - Path path; + std::filesystem::path path; bool active; int priority; - Package(const Path & path, bool active, int priority) + Package(const std::filesystem::path & path, bool active, int priority) : path{path} , active{active} , priority{priority} @@ -22,21 +23,21 @@ struct Package } }; -class BuildEnvFileConflictError : public Error +class BuildEnvFileConflictError final : public CloneableError { public: - const Path fileA; - const Path fileB; + const std::filesystem::path fileA; + const std::filesystem::path fileB; int priority; - BuildEnvFileConflictError(const Path fileA, const Path fileB, int priority) - : Error( + BuildEnvFileConflictError(const std::filesystem::path fileA, const std::filesystem::path fileB, int priority) + : CloneableError( "Unable to build profile. There is a conflict for the following files:\n" "\n" " %1%\n" " %2%", - fileA, - fileB) + PathFmt(fileA), + PathFmt(fileB)) , fileA(fileA) , fileB(fileB) , priority(priority) @@ -46,6 +47,6 @@ public: typedef std::vector Packages; -void buildProfile(const Path & out, Packages && pkgs); +void buildProfile(const std::filesystem::path & out, Packages && pkgs); } // namespace nix diff --git a/src/libstore/include/nix/store/common-protocol.hh b/src/libstore/include/nix/store/common-protocol.hh index 6139afc5d2ec..877e3754a3fc 100644 --- a/src/libstore/include/nix/store/common-protocol.hh +++ b/src/libstore/include/nix/store/common-protocol.hh @@ -3,6 +3,8 @@ #include "nix/util/serialise.hh" +#include + namespace nix { struct StoreDirConfig; @@ -13,6 +15,9 @@ class StorePath; struct ContentAddress; struct DrvOutput; struct Realisation; +struct Signature; +enum struct BuildResultSuccessStatus : uint8_t; +enum struct BuildResultFailureStatus : uint8_t; /** * Shared serializers between the worker protocol, serve protocol, and a @@ -74,6 +79,8 @@ template<> DECLARE_COMMON_SERIALISER(DrvOutput); template<> DECLARE_COMMON_SERIALISER(Realisation); +template<> +DECLARE_COMMON_SERIALISER(Signature); #define COMMA_ , template @@ -85,7 +92,6 @@ DECLARE_COMMON_SERIALISER(std::tuple); template DECLARE_COMMON_SERIALISER(std::map); -#undef COMMA_ /** * These use the empty string for the null case, relying on the fact @@ -106,4 +112,14 @@ DECLARE_COMMON_SERIALISER(std::optional); template<> DECLARE_COMMON_SERIALISER(std::optional); +/** + * The success and failure codes never overlay in enum tag values in the wire formats + */ +using BuildResultStatus = std::variant; + +template<> +DECLARE_COMMON_SERIALISER(BuildResultStatus); + +#undef COMMA_ + } // namespace nix diff --git a/src/libstore/include/nix/store/common-ssh-store-config.hh b/src/libstore/include/nix/store/common-ssh-store-config.hh index bbd81835d4f5..42b3415b777a 100644 --- a/src/libstore/include/nix/store/common-ssh-store-config.hh +++ b/src/libstore/include/nix/store/common-ssh-store-config.hh @@ -12,18 +12,17 @@ struct CommonSSHStoreConfig : virtual StoreConfig { using StoreConfig::StoreConfig; - CommonSSHStoreConfig(std::string_view scheme, const ParsedURL::Authority & authority, const Params & params); - CommonSSHStoreConfig(std::string_view scheme, std::string_view authority, const Params & params); + CommonSSHStoreConfig(const ParsedURL::Authority & authority, const Params & params); - const Setting sshKey{ + Setting sshKey{ this, "", "ssh-key", "Path to the SSH private key used to authenticate to the remote machine."}; - const Setting sshPublicHostKey{ + Setting sshPublicHostKey{ this, "", "base64-ssh-public-host-key", "The public host key of the remote machine."}; - const Setting compress{this, false, "compress", "Whether to enable SSH compression."}; + Setting compress{this, false, "compress", "Whether to enable SSH compression."}; - const Setting remoteStore{ + Setting remoteStore{ this, "", "remote-store", diff --git a/src/libstore/include/nix/store/derivation-options.hh b/src/libstore/include/nix/store/derivation-options.hh index d733df1599bd..4e6ec22f9ccd 100644 --- a/src/libstore/include/nix/store/derivation-options.hh +++ b/src/libstore/include/nix/store/derivation-options.hh @@ -10,10 +10,10 @@ #include "nix/util/json-impls.hh" #include "nix/store/store-dir-config.hh" #include "nix/store/downstream-placeholder.hh" +#include "nix/store/worker-settings.hh" namespace nix { -class Store; struct StoreDirConfig; struct BasicDerivation; struct StructuredAttrs; @@ -183,17 +183,7 @@ struct DerivationOptions */ StringSet getRequiredSystemFeatures(const BasicDerivation & drv) const; - /** - * @param drv See note on `getRequiredSystemFeatures` - */ - bool canBuildLocally(Store & localStore, const BasicDerivation & drv) const; - - /** - * @param drv See note on `getRequiredSystemFeatures` - */ - bool willBuildLocally(Store & localStore, const BasicDerivation & drv) const; - - bool substitutesAllowed() const; + bool substitutesAllowed(const WorkerSettings & workerSettings) const; /** * @param drv See note on `getRequiredSystemFeatures` @@ -238,7 +228,7 @@ DerivationOptions derivationOptionsFromStructuredAttrs( */ std::optional> tryResolve( const DerivationOptions & drvOptions, - std::function(ref drvPath, const std::string & outputName)> + fun(ref drvPath, const std::string & outputName)> queryResolutionChain); }; // namespace nix diff --git a/src/libstore/include/nix/store/derivations.hh b/src/libstore/include/nix/store/derivations.hh index 3b07072d99d4..c3ee6bd0e06a 100644 --- a/src/libstore/include/nix/store/derivations.hh +++ b/src/libstore/include/nix/store/derivations.hh @@ -17,7 +17,6 @@ namespace nix { struct StoreDirConfig; -struct AsyncPathWriter; struct Provenance; /* Abstract syntax of derivations. */ @@ -276,7 +275,10 @@ struct BasicDerivation */ StorePathSet inputSrcs; std::string platform; - Path builder; + /** + * Probably should be an absolute path in the path format that `platform` uses + */ + std::string builder; Strings args; /** * Must not contain the key `__json`, at least in order to serialize to ATerm. @@ -342,6 +344,19 @@ struct Derivation : BasicDerivation bool maskOutputs, DerivedPathMap::ChildNode::Map * actualInputs = nullptr) const; + /** + * Determine whether this derivation should be resolved before building. + * + * Resolution is needed when: + * - Input-addressed derivations are deferred (depend on CA derivations) + * - Content-addressed derivations have input drvs and are either: + * - Floating (non-fixed), which must always be resolved + * - Fixed, which can optionally be resolved when ca-derivations is enabled + * - Impure derivations always need resolution + * - Any input derivations have outputs from dynamic derivations + */ + bool shouldResolve() const; + /** * Return the underlying basic derivation but with these changes: * @@ -360,7 +375,7 @@ struct Derivation : BasicDerivation */ std::optional tryResolve( Store & store, - std::function(ref drvPath, const std::string & outputName)> + fun(ref drvPath, const std::string & outputName)> queryResolutionChain) const; /** @@ -455,25 +470,11 @@ struct Derivation : BasicDerivation class Store; /** - * Write a derivation to the Nix store, and return its path. - */ -StorePath writeDerivation( - Store & store, - const Derivation & drv, - RepairFlag repair = NoRepair, - bool readOnly = false, - std::shared_ptr provenance = nullptr); - -/** - * Asynchronously write a derivation to the Nix store, and return its path. + * Compute the store path that would be used for a derivation without writing it. + * + * This is a pure computation based on the derivation content and store directory. */ -StorePath writeDerivation( - Store & store, - AsyncPathWriter & asyncPathWriter, - const Derivation & drv, - RepairFlag repair = NoRepair, - bool readOnly = false, - std::shared_ptr provenance = nullptr); +StorePath computeStorePath(const StoreDirConfig & store, const Derivation & drv); /** * Read a derivation from a file. diff --git a/src/libstore/include/nix/store/dummy-store.hh b/src/libstore/include/nix/store/dummy-store.hh index febf351c975c..59d1b1fddb22 100644 --- a/src/libstore/include/nix/store/dummy-store.hh +++ b/src/libstore/include/nix/store/dummy-store.hh @@ -35,6 +35,8 @@ struct DummyStoreConfig : public std::enable_shared_from_this, No additional memory will be used, because no information needs to be stored. )"}; + bool getReadOnly() const override; + static const std::string name() { return "Dummy Store"; diff --git a/src/libstore/include/nix/store/filetransfer.hh b/src/libstore/include/nix/store/filetransfer.hh index fa8a649e2b36..4994477bdcb4 100644 --- a/src/libstore/include/nix/store/filetransfer.hh +++ b/src/libstore/include/nix/store/filetransfer.hh @@ -19,8 +19,16 @@ namespace nix { +const std::filesystem::path & nixConfDir(); + struct FileTransferSettings : Config { +private: + static std::filesystem::path getDefaultSSLCertFile(); + +public: + FileTransferSettings(); + Setting enableHttp2{this, true, "http2", "Whether to enable HTTP/2 support."}; Setting userAgentSuffix{ @@ -77,6 +85,64 @@ struct FileTransferSettings : Config not processed quickly enough to exceed the size of this buffer, downloads may stall. The default is 1048576 (1 MiB). )"}; + + Setting downloadSpeed{ + this, + 0, + "download-speed", + R"( + Specify the maximum transfer rate in kilobytes per second you want + Nix to use for downloads. + )"}; + + Setting netrcFile{ + this, + nixConfDir() / "netrc", + "netrc-file", + R"( + If set to an absolute path to a `netrc` file, Nix uses the HTTP + authentication credentials in this file when trying to download from + a remote host through HTTP or HTTPS. Defaults to + `$NIX_CONF_DIR/netrc`. + + The `netrc` file consists of a list of accounts in the following + format: + + machine my-machine + login my-username + password my-password + + For the exact syntax, see [the `curl` + documentation](https://ec.haxx.se/usingcurl-netrc.html). + + > **Note** + > + > This must be an absolute path, and `~` is not resolved. For + > example, `~/.netrc` won't resolve to your home directory's + > `.netrc`. + )"}; + + Setting> caFile{ + this, + getDefaultSSLCertFile(), + "ssl-cert-file", + R"( + The path of a file containing CA certificates used to + authenticate `https://` downloads. Nix by default uses + the first of the following files that exists: + + 1. `/etc/ssl/certs/ca-certificates.crt` + 2. `/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt` + + The path can be overridden by the following environment + variables, in order of precedence: + + 1. `NIX_SSL_CERT_FILE` + 2. `SSL_CERT_FILE` + )", + {}, + // Don't document the machine-specific default value + false}; }; extern FileTransferSettings fileTransferSettings; @@ -116,11 +182,20 @@ struct FileTransferRequest Headers headers; std::string expectedETag; HttpMethod method = HttpMethod::Get; - size_t tries = fileTransferSettings.tries; unsigned int baseRetryTimeMs = RETRY_TIME_MS_DEFAULT; ActivityId parentAct; bool decompress = true; + /** + * Optional path to the client certificate in "PEM" format. Only used for TLS-based protocols. + */ + std::optional tlsCert; + + /** + * Optional path to the client private key in "PEM" format. Only used for TLS-based protocols. + */ + std::optional tlsKey; + struct UploadData { UploadData(StringSource & s) @@ -326,11 +401,11 @@ ref getFileTransfer(); * * Prefer getFileTransfer() to this; see its docs for why. */ -ref makeFileTransfer(); +ref makeFileTransfer(const FileTransferSettings & settings = fileTransferSettings); std::shared_ptr resetFileTransfer(); -class FileTransferError : public Error +class FileTransferError final : public CloneableError { public: FileTransfer::Error error; diff --git a/src/libstore/include/nix/store/gc-store.hh b/src/libstore/include/nix/store/gc-store.hh index de7a71382f92..d3d42470e2c9 100644 --- a/src/libstore/include/nix/store/gc-store.hh +++ b/src/libstore/include/nix/store/gc-store.hh @@ -76,11 +76,10 @@ struct GCResults * Depending on the action, the GC roots, or the paths that would * be or have been deleted. */ - PathSet paths; + StringSet paths; /** - * For `gcReturnDead`, `gcDeleteDead` and `gcDeleteSpecific`, the - * number of bytes that would be or was freed. + * For `gcDeleteDead` and `gcDeleteSpecific`, the number of bytes that were freed. */ uint64_t bytesFreed = 0; }; diff --git a/src/libstore/include/nix/store/global-paths.hh b/src/libstore/include/nix/store/global-paths.hh new file mode 100644 index 000000000000..4eb4cdef2c71 --- /dev/null +++ b/src/libstore/include/nix/store/global-paths.hh @@ -0,0 +1,33 @@ +#pragma once +///@file + +#include +#include + +namespace nix { + +/** + * The directory where system configuration files are stored. + * + * This is needed very early during initialization, before a main + * `Settings` object can be constructed. + */ +const std::filesystem::path & nixConfDir(); + +/** + * The path to the system configuration file (`nix.conf`). + */ +static inline std::filesystem::path nixConfFile() +{ + return nixConfDir() / "nix.conf"; +} + +/** + * A list of user configuration files to load. + * + * This is needed very early during initialization, before a main + * `Settings` object can be constructed. + */ +const std::vector & nixUserConfFiles(); + +} // namespace nix diff --git a/src/libstore/include/nix/store/globals.hh b/src/libstore/include/nix/store/globals.hh index d0a134b9c4e1..38bf5d0a7041 100644 --- a/src/libstore/include/nix/store/globals.hh +++ b/src/libstore/include/nix/store/globals.hh @@ -1,121 +1,197 @@ #pragma once ///@file -#include -#include - #include #include "nix/util/types.hh" #include "nix/util/configuration.hh" #include "nix/util/environment-variables.hh" -#include "nix/util/experimental-features.hh" -#include "nix/util/users.hh" #include "nix/store/build/derivation-builder.hh" +#include "nix/store/local-settings.hh" +#include "nix/store/store-reference.hh" +#include "nix/store/worker-settings.hh" #include "nix/store/config.hh" namespace nix { -typedef enum { smEnabled, smRelaxed, smDisabled } SandboxMode; - template<> -SandboxMode BaseSetting::parse(const std::string & str) const; +StoreReference BaseSetting::parse(const std::string & str) const; template<> -std::string BaseSetting::to_string() const; +std::string BaseSetting::to_string() const; template<> -PathsInChroot BaseSetting::parse(const std::string & str) const; +std::set BaseSetting>::parse(const std::string & str) const; template<> -std::string BaseSetting::to_string() const; +std::string BaseSetting>::to_string() const; -template<> -struct BaseSetting::trait +struct ProfileDirsOptions; + +struct LogFileSettings : public virtual Config { - static constexpr bool appendable = true; -}; + /** + * The directory where we log various operations. + */ + std::filesystem::path nixLogDir; -template<> -void BaseSetting::appendOrSet(PathsInChroot newValue, bool append); +protected: + LogFileSettings(); -struct MaxBuildJobsSetting : public BaseSetting -{ - MaxBuildJobsSetting( - Config * options, - unsigned int def, - const std::string & name, - const std::string & description, - const StringSet & aliases = {}) - : BaseSetting(def, true, name, description, aliases) - { - options->addSetting(this); - } +public: + Setting keepLog{ + this, + true, + "keep-build-log", + R"( + If set to `true` (the default), Nix writes the build log of a + derivation (i.e. the standard output and error of its builder) to + the directory `/nix/var/log/nix/drvs`. The build log can be + retrieved using the command `nix-store -l path`. + )", + {"build-keep-log"}}; - unsigned int parse(const std::string & str) const override; + Setting compressLog{ + this, + true, + "compress-build-log", + R"( + If set to `true` (the default), build logs written to + `/nix/var/log/nix/drvs` are compressed on the fly using bzip2. + Otherwise, they are not compressed. + )", + {"build-compress-log"}}; }; -const uint32_t maxIdsPerBuild = -#ifdef __linux__ - 1 << 16 -#else - 1 -#endif - ; - -class Settings : public Config +struct NarInfoDiskCacheSettings : public virtual Config { + Setting ttlNegative{ + this, + 3600, + "narinfo-cache-negative-ttl", + R"( + The TTL in seconds for negative lookups. + If a store path is queried from a [substituter](#conf-substituters) but was not found, a negative lookup is cached in the local disk cache database for the specified duration. + + Set to `0` to force updating the lookup cache. + + To wipe the lookup cache completely: + + ```shell-session + $ rm $HOME/.cache/nix/binary-cache-v*.sqlite* + # rm /root/.cache/nix/binary-cache-v*.sqlite* + ``` + )"}; + Setting ttlPositive{ + this, + 30 * 24 * 3600, + "narinfo-cache-positive-ttl", + R"( + The TTL in seconds for positive lookups. If a store path is queried + from a substituter, the result of the query is cached in the + local disk cache database including some of the NAR metadata. The + default TTL is a month, setting a shorter TTL for positive lookups + can be useful for binary caches that have frequent garbage + collection, in which case having a more frequent cache invalidation + would prevent trying to pull the path again and failing with a hash + mismatch if the build isn't reproducible. + )"}; + + Setting ttlMeta{ + this, + 7 * 24 * 3600, + "narinfo-cache-meta-ttl", + R"( + The TTL in seconds for caching binary cache metadata (i.e. + `/nix-cache-info`). This determines how long information about a + binary cache (such as its store directory, priority, and whether it + wants mass queries) is considered valid before being refreshed. + )"}; +}; + +class Settings : public virtual Config, + private LocalSettings, + private LogFileSettings, + private WorkerSettings, + private NarInfoDiskCacheSettings +{ StringSet getDefaultSystemFeatures(); StringSet getDefaultExtraPlatforms(); bool isWSL1(); - Path getDefaultSSLCertFile(); - public: Settings(); - static unsigned int getDefaultCores(); - - Path nixPrefix; - /** - * The directory where we store sources and derived files. + * Get the local store settings. */ - Path nixStore; + LocalSettings & getLocalSettings() + { + return *this; + } - Path nixDataDir; /* !!! fix */ + const LocalSettings & getLocalSettings() const + { + return *this; + } /** - * The directory where we log various operations. + * Get the log file settings. */ - Path nixLogDir; + LogFileSettings & getLogFileSettings() + { + return *this; + } + + const LogFileSettings & getLogFileSettings() const + { + return *this; + } /** - * The directory where state is stored. + * Get the worker settings. */ - Path nixStateDir; + WorkerSettings & getWorkerSettings() + { + return *this; + } + + const WorkerSettings & getWorkerSettings() const + { + return *this; + } /** - * The directory where system configuration files are stored. + * Get the NAR info disk cache settings. */ - std::filesystem::path nixConfDir; + NarInfoDiskCacheSettings & getNarInfoDiskCacheSettings() + { + return *this; + } + + const NarInfoDiskCacheSettings & getNarInfoDiskCacheSettings() const + { + return *this; + } + + static unsigned int getDefaultCores(); /** - * A list of user configuration files to load. + * The directory where state is stored. */ - std::vector nixUserConfFiles; + std::filesystem::path nixStateDir; /** * File name of the socket the daemon listens to. */ - Path nixDaemonSocketFile; + std::filesystem::path nixDaemonSocketFile; - Setting storeUri{ + Setting storeUri{ this, - getEnv("NIX_REMOTE").value_or("auto"), + StoreReference::parse(getEnv("NIX_REMOTE").value_or("auto")), "store", R"( The [URL of the Nix store](@docroot@/store/types/index.md#store-url-format) @@ -125,88 +201,15 @@ public: section of the manual for supported store types and settings. )"}; - Setting keepFailed{this, false, "keep-failed", "Whether to keep temporary directories of failed builds."}; - - Setting keepGoing{ - this, false, "keep-going", "Whether to keep building derivations when another build fails."}; + Setting useSQLiteWAL{this, !isWSL1(), "use-sqlite-wal", "Whether SQLite should use WAL mode."}; - Setting tryFallback{ - this, - false, - "fallback", - R"( - If set to `true`, Nix falls back to building from source if a - binary substitute fails. This is equivalent to the `--fallback` - flag. The default is `false`. - )", - {"build-fallback"}}; + Setting keepFailed{this, false, "keep-failed", "Whether to keep temporary directories of failed builds."}; /** * Whether to show build log output in real time. */ bool verboseBuild = true; - Setting logLines{ - this, - 25, - "log-lines", - "The number of lines of the tail of " - "the log to show if a build fails."}; - - MaxBuildJobsSetting maxBuildJobs{ - this, - 1, - "max-jobs", - R"( - Maximum number of jobs that Nix tries to build locally in parallel. - - The special value `auto` causes Nix to use the number of CPUs in your system. - Use `0` to disable local builds and directly use the remote machines specified in [`builders`](#conf-builders). - This doesn't affect derivations that have [`preferLocalBuild = true`](@docroot@/language/advanced-attributes.md#adv-attr-preferLocalBuild), which are always built locally. - - > **Note** - > - > The number of CPU cores to use for each build job is independently determined by the [`cores`](#conf-cores) setting. - - - The setting can be overridden using the `--max-jobs` (`-j`) command line switch. - )", - {"build-max-jobs"}}; - - Setting maxSubstitutionJobs{ - this, - 16, - "max-substitution-jobs", - R"( - This option defines the maximum number of substitution jobs that Nix - tries to run in parallel. The default is `16`. The minimum value - one can choose is `1` and lower values are interpreted as `1`. - )", - {"substitution-max-jobs"}}; - - Setting buildCores{ - this, - 0, - "cores", - R"( - Sets the value of the `NIX_BUILD_CORES` environment variable in the [invocation of the `builder` executable](@docroot@/store/building.md#builder-execution) of a derivation. - The `builder` executable can use this variable to control its own maximum amount of parallelism. - - - For instance, in Nixpkgs, if the attribute `enableParallelBuilding` for the `mkDerivation` build helper is set to `true`, it passes the `-j${NIX_BUILD_CORES}` flag to GNU Make. - - If set to `0`, nix will detect the number of CPU cores and pass this number via `NIX_BUILD_CORES`. - - > **Note** - > - > The number of parallel local Nix build jobs is independently controlled with the [`max-jobs`](#conf-max-jobs) setting. - )", - {"build-cores"}}; - /** * Read-only mode. Don't copy stuff to the store, don't change * the database. @@ -238,620 +241,6 @@ public: configuration option is set as the empty string. )"}; - Setting maxSilentTime{ - this, - 0, - "max-silent-time", - R"( - This option defines the maximum number of seconds that a builder can - go without producing any data on standard output or standard error. - This is useful (for instance in an automated build system) to catch - builds that are stuck in an infinite loop, or to catch remote builds - that are hanging due to network problems. It can be overridden using - the `--max-silent-time` command line switch. - - The value `0` means that there is no timeout. This is also the - default. - )", - {"build-max-silent-time"}}; - - Setting buildTimeout{ - this, - 0, - "timeout", - R"( - This option defines the maximum number of seconds that a builder can - run. This is useful (for instance in an automated build system) to - catch builds that are stuck in an infinite loop but keep writing to - their standard output or standard error. It can be overridden using - the `--timeout` command line switch. - - The value `0` means that there is no timeout. This is also the - default. - )", - {"build-timeout"}}; - - Setting buildHook{ - this, - {"nix", "__build-remote"}, - "build-hook", - R"( - The path to the helper program that executes remote builds. - - Nix communicates with the build hook over `stdio` using a custom protocol to request builds that cannot be performed directly by the Nix daemon. - The default value is the internal Nix binary that implements remote building. - - > **Important** - > - > Change this setting only if you really know what you’re doing. - )"}; - - Setting builders{ - this, - "@" + nixConfDir.string() + "/machines", - "builders", - R"( - A semicolon- or newline-separated list of build machines. - - In addition to the [usual ways of setting configuration options](@docroot@/command-ref/conf-file.md), the value can be read from a file by prefixing its absolute path with `@`. - - > **Example** - > - > This is the default setting: - > - > ``` - > builders = @/etc/nix/machines - > ``` - - Each machine specification consists of the following elements, separated by spaces. - Only the first element is required. - To leave a field at its default, set it to `-`. - - 1. The URI of the remote store in the format `ssh://[username@]hostname[:port]`. - - > **Example** - > - > `ssh://nix@mac` - - For backward compatibility, `ssh://` may be omitted. - The hostname may be an alias defined in `~/.ssh/config`. - - 2. A comma-separated list of [Nix system types](@docroot@/development/building.md#system-type). - If omitted, this defaults to the local platform type. - - > **Example** - > - > `aarch64-darwin` - - It is possible for a machine to support multiple platform types. - - > **Example** - > - > `i686-linux,x86_64-linux` - - 3. The SSH identity file to be used to log in to the remote machine. - If omitted, SSH uses its regular identities. - - > **Example** - > - > `/home/user/.ssh/id_mac` - - 4. The maximum number of builds that Nix executes in parallel on the machine. - Typically this should be equal to the number of CPU cores. - - 5. The “speed factor”, indicating the relative speed of the machine as a positive integer. - If there are multiple machines of the right type, Nix prefers the fastest, taking load into account. - - 6. A comma-separated list of supported [system features](#conf-system-features). - - A machine is only used to build a derivation if all the features in the derivation's [`requiredSystemFeatures`](@docroot@/language/advanced-attributes.html#adv-attr-requiredSystemFeatures) attribute are supported by that machine. - - 7. A comma-separated list of required [system features](#conf-system-features). - - A machine is only used to build a derivation if all of the machine’s required features appear in the derivation’s [`requiredSystemFeatures`](@docroot@/language/advanced-attributes.html#adv-attr-requiredSystemFeatures) attribute. - - 8. The (base64-encoded) public host key of the remote machine. - If omitted, SSH uses its regular `known_hosts` file. - - The value for this field can be obtained via `base64 -w0`. - - > **Example** - > - > Multiple builders specified on the command line: - > - > ```console - > --builders 'ssh://mac x86_64-darwin ; ssh://beastie x86_64-freebsd' - > ``` - - > **Example** - > - > This specifies several machines that can perform `i686-linux` builds: - > - > ``` - > nix@scratchy.labs.cs.uu.nl i686-linux /home/nix/.ssh/id_scratchy 8 1 kvm - > nix@itchy.labs.cs.uu.nl i686-linux /home/nix/.ssh/id_scratchy 8 2 - > nix@poochie.labs.cs.uu.nl i686-linux /home/nix/.ssh/id_scratchy 1 2 kvm benchmark - > ``` - > - > However, `poochie` only builds derivations that have the attribute - > - > ```nix - > requiredSystemFeatures = [ "benchmark" ]; - > ``` - > - > or - > - > ```nix - > requiredSystemFeatures = [ "benchmark" "kvm" ]; - > ``` - > - > `itchy` cannot do builds that require `kvm`, but `scratchy` does support such builds. - > For regular builds, `itchy` is preferred over `scratchy` because it has a higher speed factor. - - For Nix to use substituters, the calling user must be in the [`trusted-users`](#conf-trusted-users) list. - - > **Note** - > - > A build machine must be accessible via SSH and have Nix installed. - > `nix` must be available in `$PATH` for the user connecting over SSH. - - > **Warning** - > - > If you are building via the Nix daemon (default), the Nix daemon user account on the local machine (that is, `root`) requires access to a user account on the remote machine (not necessarily `root`). - > - > If you can’t or don’t want to configure `root` to be able to access the remote machine, set [`store`](#conf-store) to any [local store](@docroot@/store/types/local-store.html), e.g. by passing `--store /tmp` to the command on the local machine. - - To build only on remote machines and disable local builds, set [`max-jobs`](#conf-max-jobs) to 0. - - If you want the remote machines to use substituters, set [`builders-use-substitutes`](#conf-builders-use-substitutes) to `true`. - )", - {}, - false}; - - Setting alwaysAllowSubstitutes{ - this, - false, - "always-allow-substitutes", - R"( - If set to `true`, Nix ignores the [`allowSubstitutes`](@docroot@/language/advanced-attributes.md) attribute in derivations and always attempt to use [available substituters](#conf-substituters). - )"}; - - Setting buildersUseSubstitutes{ - this, - false, - "builders-use-substitutes", - R"( - If set to `true`, Nix instructs [remote build machines](#conf-builders) to use their own [`substituters`](#conf-substituters) if available. - - It means that remote build hosts fetch as many dependencies as possible from their own substituters (e.g, from `cache.nixos.org`) instead of waiting for the local machine to upload them all. - This can drastically reduce build times if the network connection between the local machine and the remote build host is slow. - )"}; - - Setting reservedSize{ - this, 8 * 1024 * 1024, "gc-reserved-space", "Amount of reserved disk space for the garbage collector."}; - - Setting fsyncMetadata{ - this, - true, - "fsync-metadata", - R"( - If set to `true`, changes to the Nix store metadata (in - `/nix/var/nix/db`) are synchronously flushed to disk. This improves - robustness in case of system crashes, but reduces performance. The - default is `true`. - )"}; - - Setting fsyncStorePaths{ - this, - false, - "fsync-store-paths", - R"( - Whether to call `fsync()` on store paths before registering them, to - flush them to disk. This improves robustness in case of system crashes, - but reduces performance. The default is `false`. - )"}; - - Setting useSQLiteWAL{this, !isWSL1(), "use-sqlite-wal", "Whether SQLite should use WAL mode."}; - -#ifndef _WIN32 - // FIXME: remove this option, `fsync-store-paths` is faster. - Setting syncBeforeRegistering{ - this, false, "sync-before-registering", "Whether to call `sync()` before registering a path as valid."}; -#endif - - Setting useSubstitutes{ - this, - true, - "substitute", - R"( - If set to `true` (default), Nix uses binary substitutes if - available. This option can be disabled to force building from - source. - )", - {"build-use-substitutes"}}; - - Setting buildUsersGroup{ - this, - "", - "build-users-group", - R"( - This options specifies the Unix group containing the Nix build user - accounts. In multi-user Nix installations, builds should not be - performed by the Nix account since that would allow users to - arbitrarily modify the Nix store and database by supplying specially - crafted builders; and they cannot be performed by the calling user - since that would allow him/her to influence the build result. - - Therefore, if this option is non-empty and specifies a valid group, - builds are performed under the user accounts that are a member - of the group specified here (as listed in `/etc/group`). Those user - accounts should not be used for any other purpose\! - - Nix never runs two builds under the same user account at the - same time. This is to prevent an obvious security hole: a malicious - user writing a Nix expression that modifies the build result of a - legitimate Nix expression being built by another user. Therefore it - is good to have as many Nix build user accounts as you can spare. - (Remember: uids are cheap.) - - The build users should have permission to create files in the Nix - store, but not delete them. Therefore, `/nix/store` should be owned - by the Nix account, its group should be the group specified here, - and its mode should be `1775`. - - If the build users group is empty, builds are performed under - the uid of the Nix process (that is, the uid of the caller if - `NIX_REMOTE` is empty, the uid under which the Nix daemon runs if - `NIX_REMOTE` is `daemon`). Obviously, this should not be used - with a nix daemon accessible to untrusted clients. - - Defaults to `nixbld` when running as root, *empty* otherwise. - )", - {}, - false}; - - Setting autoAllocateUids{ - this, - false, - "auto-allocate-uids", - R"( - Whether to select UIDs for builds automatically, instead of using the - users in `build-users-group`. - - UIDs are allocated starting at 872415232 (0x34000000) on Linux and 56930 on macOS. - )", - {}, - true, - Xp::AutoAllocateUids}; - - Setting startId{ - this, -#ifdef __linux__ - 0x34000000, -#else - 56930, -#endif - "start-id", - "The first UID and GID to use for dynamic ID allocation."}; - - Setting uidCount{ - this, -#ifdef __linux__ - maxIdsPerBuild * 128, -#else - 128, -#endif - "id-count", - "The number of UIDs/GIDs to use for dynamic ID allocation."}; - -#ifdef __linux__ - Setting useCgroups{ - this, - false, - "use-cgroups", - R"( - Whether to execute builds inside cgroups. - This is only supported on Linux. - - Cgroups are required and enabled automatically for derivations - that require the `uid-range` system feature. - )"}; -#endif - - Setting impersonateLinux26{ - this, - false, - "impersonate-linux-26", - "Whether to impersonate a Linux 2.6 machine on newer kernels.", - {"build-impersonate-linux-26"}}; - - Setting keepLog{ - this, - true, - "keep-build-log", - R"( - If set to `true` (the default), Nix writes the build log of a - derivation (i.e. the standard output and error of its builder) to - the directory `/nix/var/log/nix/drvs`. The build log can be - retrieved using the command `nix-store -l path`. - )", - {"build-keep-log"}}; - - Setting compressLog{ - this, - true, - "compress-build-log", - R"( - If set to `true` (the default), build logs written to - `/nix/var/log/nix/drvs` are compressed on the fly using bzip2. - Otherwise, they are not compressed. - )", - {"build-compress-log"}}; - - Setting maxLogSize{ - this, - 0, - "max-build-log-size", - R"( - This option defines the maximum number of bytes that a builder can - write to its stdout/stderr. If the builder exceeds this limit, it’s - killed. A value of `0` (the default) means that there is no limit. - )", - {"build-max-log-size"}}; - - Setting pollInterval{this, 5, "build-poll-interval", "How often (in seconds) to poll for locks."}; - - Setting gcKeepOutputs{ - this, - false, - "keep-outputs", - R"( - If `true`, the garbage collector keeps the outputs of - non-garbage derivations. If `false` (default), outputs are - deleted unless they are GC roots themselves (or reachable from other - roots). - - In general, outputs must be registered as roots separately. However, - even if the output of a derivation is registered as a root, the - collector still deletes store paths that are used only at build - time (e.g., the C compiler, or source tarballs downloaded from the - network). To prevent it from doing so, set this option to `true`. - )", - {"gc-keep-outputs"}}; - - Setting gcKeepDerivations{ - this, - true, - "keep-derivations", - R"( - If `true` (default), the garbage collector keeps the derivations - from which non-garbage store paths were built. If `false`, they are - deleted unless explicitly registered as a root (or reachable from - other roots). - - Keeping derivation around is useful for querying and traceability - (e.g., it allows you to ask with what dependencies or options a - store path was built), so by default this option is on. Turn it off - to save a bit of disk space (or a lot if `keep-outputs` is also - turned on). - )", - {"gc-keep-derivations"}}; - - Setting autoOptimiseStore{ - this, - false, - "auto-optimise-store", - R"( - If set to `true`, Nix automatically detects files in the store - that have identical contents, and replaces them with hard links to - a single copy. This saves disk space. If set to `false` (the - default), you can still run `nix-store --optimise` to get rid of - duplicate files. - )"}; - - Setting envKeepDerivations{ - this, - false, - "keep-env-derivations", - R"( - If `false` (default), derivations are not stored in Nix user - environments. That is, the derivations of any build-time-only - dependencies may be garbage-collected. - - If `true`, when you add a Nix derivation to a user environment, the - path of the derivation is stored in the user environment. Thus, the - derivation isn't garbage-collected until the user environment - generation is deleted (`nix-env --delete-generations`). To prevent - build-time-only dependencies from being collected, you should also - turn on `keep-outputs`. - - The difference between this option and `keep-derivations` is that - this one is “sticky”: it applies to any user environment created - while this option was enabled, while `keep-derivations` only applies - at the moment the garbage collector is run. - )", - {"env-keep-derivations"}}; - - Setting sandboxMode{ - this, -#ifdef __linux__ - smEnabled -#else - smDisabled -#endif - , - "sandbox", - R"( - If set to `true`, builds are performed in a *sandboxed - environment*, i.e., they’re isolated from the normal file system - hierarchy and only see their dependencies in the Nix store, - the temporary build directory, private versions of `/proc`, - `/dev`, `/dev/shm` and `/dev/pts` (on Linux), and the paths - configured with the `sandbox-paths` option. This is useful to - prevent undeclared dependencies on files in directories such as - `/usr/bin`. In addition, on Linux, builds run in private PID, - mount, network, IPC and UTS namespaces to isolate them from other - processes in the system (except that fixed-output derivations do - not run in private network namespace to ensure they can access the - network). - - Currently, sandboxing only work on Linux and macOS. The use of a - sandbox requires that Nix is run as root (so you should use the - “build users” feature to perform the actual builds under different - users than root). - - If this option is set to `relaxed`, then fixed-output derivations - and derivations that have the `__noChroot` attribute set to `true` - do not run in sandboxes. - - The default is `true` on Linux and `false` on all other platforms. - )", - {"build-use-chroot", "build-use-sandbox"}}; - - Setting sandboxPaths{ - this, - {}, - "sandbox-paths", - R"( - A list of paths bind-mounted into Nix sandbox environments. You can - use the syntax `target=source` to mount a path in a different - location in the sandbox; for instance, `/bin=/nix-bin` mounts - the path `/nix-bin` as `/bin` inside the sandbox. If *source* is - followed by `?`, then it is not an error if *source* does not exist; - for example, `/dev/nvidiactl?` specifies that `/dev/nvidiactl` - only be mounted in the sandbox if it exists in the host filesystem. - - If the source is in the Nix store, then its closure is added to - the sandbox as well. - - Depending on how Nix was built, the default value for this option - may be empty or provide `/bin/sh` as a bind-mount of `bash`. - )", - {"build-chroot-dirs", "build-sandbox-paths"}}; - - Setting sandboxFallback{ - this, true, "sandbox-fallback", "Whether to disable sandboxing when the kernel doesn't allow it."}; - -#ifndef _WIN32 - Setting requireDropSupplementaryGroups{ - this, - isRootUser(), - "require-drop-supplementary-groups", - R"( - Following the principle of least privilege, - Nix attempts to drop supplementary groups when building with sandboxing. - - However this can fail under some circumstances. - For example, if the user lacks the `CAP_SETGID` capability. - Search `setgroups(2)` for `EPERM` to find more detailed information on this. - - If you encounter such a failure, setting this option to `false` enables you to ignore it and continue. - But before doing so, you should consider the security implications carefully. - Not dropping supplementary groups means the build sandbox is less restricted than intended. - - This option defaults to `true` when the user is root - (since `root` usually has permissions to call setgroups) - and `false` otherwise. - )"}; -#endif - -#ifdef __linux__ - Setting sandboxShmSize{ - this, - "50%", - "sandbox-dev-shm-size", - R"( - *Linux only* - - This option determines the maximum size of the `tmpfs` filesystem - mounted on `/dev/shm` in Linux sandboxes. For the format, see the - description of the `size` option of `tmpfs` in mount(8). The default - is `50%`. - )"}; -#endif - -#if defined(__linux__) || defined(__FreeBSD__) - Setting sandboxBuildDir{ - this, - "/build", - "sandbox-build-dir", - R"( - *Linux only* - - The build directory inside the sandbox. - - This directory is backed by [`build-dir`](#conf-build-dir) on the host. - )"}; -#endif - - Setting> buildDir{ - this, - std::nullopt, - "build-dir", - R"( - Override the `build-dir` store setting for all stores that have this setting. - - See also the per-store [`build-dir`](@docroot@/store/types/local-store.md#store-local-store-build-dir) setting. - )"}; - - Setting allowedImpureHostPrefixes{ - this, - {}, - "allowed-impure-host-deps", - "Which prefixes to allow derivations to ask for access to (primarily for Darwin)."}; - -#ifdef __APPLE__ - Setting darwinLogSandboxViolations{ - this, - false, - "darwin-log-sandbox-violations", - "Whether to log Darwin sandbox access violations to the system log."}; -#endif - - Setting runDiffHook{ - this, - false, - "run-diff-hook", - R"( - If true, enable the execution of the `diff-hook` program. - - When using the Nix daemon, `run-diff-hook` must be set in the - `nix.conf` configuration file, and cannot be passed at the command - line. - )"}; - - OptionalPathSetting diffHook{ - this, - std::nullopt, - "diff-hook", - R"( - Absolute path to an executable capable of diffing build - results. The hook is executed if `run-diff-hook` is true, and the - output of a build is known to not be the same. This program is not - executed to determine if two results are the same. - - The diff hook is executed by the same user and group who ran the - build. However, the diff hook does not have write access to the - store path just built. - - The diff hook program receives three parameters: - - 1. A path to the previous build's results - - 2. A path to the current build's results - - 3. The path to the build's derivation - - 4. The path to the build's scratch directory. This directory - exists only if the build was run with `--keep-failed`. - - The stderr and stdout output from the diff hook isn't displayed - to the user. Instead, it prints to the nix-daemon's log. - - When using the Nix daemon, `diff-hook` must be set in the `nix.conf` - configuration file, and cannot be passed at the command line. - )"}; - Setting trustedPublicKeys{ this, {"cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY="}, @@ -882,25 +271,6 @@ public: can add it to `trusted-public-keys` in their `nix.conf`. )"}; - Setting tarballTtl{ - this, - 60 * 60, - "tarball-ttl", - R"( - The number of seconds a downloaded tarball is considered fresh. If - the cached tarball is stale, Nix checks whether it is still up - to date using the ETag header. Nix downloads a new version if - the ETag header is unsupported, or the cached ETag doesn't match. - - Setting the TTL to `0` forces Nix to always check if the tarball is - up to date. - - Nix caches tarballs in `$XDG_CACHE_HOME/nix/tarballs`. - - Files fetched via `NIX_PATH`, `fetchGit`, `fetchMercurial`, - `fetchTarball`, and `fetchurl` respect this TTL. - )"}; - Setting requireSigs{ this, true, @@ -991,28 +361,10 @@ public: // Don't document the machine-specific default value false}; - Setting substituters{ - this, - Strings{"https://cache.nixos.org/"}, - "substituters", - R"( - A list of [URLs of Nix stores](@docroot@/store/types/index.md#store-url-format) to be used as substituters, separated by whitespace. - A substituter is an additional [store](@docroot@/glossary.md#gloss-store) from which Nix can obtain [store objects](@docroot@/store/store-object.md) instead of building them. - - Substituters are tried based on their priority value, which each substituter can set independently. - Lower value means higher priority. - The default is `https://cache.nixos.org`, which has a priority of 40. - - At least one of the following conditions must be met for Nix to use a substituter: - - - The substituter is in the [`trusted-substituters`](#conf-trusted-substituters) list - - The user calling Nix is in the [`trusted-users`](#conf-trusted-users) list - - In addition, each store path should be trusted as described in [`trusted-public-keys`](#conf-trusted-public-keys) - )", - {"binary-caches"}}; - - Setting trustedSubstituters{ + // move to daemonsettings in another pass + // + // we'll add a another parameter to processConnection to thread it through + Setting> trustedSubstituters{ this, {}, "trusted-substituters", @@ -1024,289 +376,10 @@ public: )", {"trusted-binary-caches"}}; - Setting ttlNegativeNarInfoCache{ - this, - 3600, - "narinfo-cache-negative-ttl", - R"( - The TTL in seconds for negative lookups. - If a store path is queried from a [substituter](#conf-substituters) but was not found, a negative lookup is cached in the local disk cache database for the specified duration. - - Set to `0` to force updating the lookup cache. - - To wipe the lookup cache completely: - - ```shell-session - $ rm $HOME/.cache/nix/binary-cache-v*.sqlite* - # rm /root/.cache/nix/binary-cache-v*.sqlite* - ``` - )"}; - - Setting ttlPositiveNarInfoCache{ - this, - 30 * 24 * 3600, - "narinfo-cache-positive-ttl", - R"( - The TTL in seconds for positive lookups. If a store path is queried - from a substituter, the result of the query is cached in the - local disk cache database including some of the NAR metadata. The - default TTL is a month, setting a shorter TTL for positive lookups - can be useful for binary caches that have frequent garbage - collection, in which case having a more frequent cache invalidation - would prevent trying to pull the path again and failing with a hash - mismatch if the build isn't reproducible. - )"}; - - Setting ttlNarInfoCacheMeta{ - this, - 7 * 24 * 3600, - "narinfo-cache-meta-ttl", - R"( - The TTL in seconds for caching binary cache metadata (i.e. - `/nix-cache-info`). This determines how long information about a - binary cache (such as its store directory, priority, and whether it - wants mass queries) is considered valid before being refreshed. - )"}; - + // move it out in the 2nd pass Setting printMissing{ this, true, "print-missing", "Whether to print what paths need to be built or downloaded."}; - Setting preBuildHook{ - this, - "", - "pre-build-hook", - R"( - If set, the path to a program that can set extra derivation-specific - settings for this system. This is used for settings that can't be - captured by the derivation model itself and are too variable between - different versions of the same system to be hard-coded into nix. - - The hook is passed the derivation path and, if sandboxes are - enabled, the sandbox directory. It can then modify the sandbox and - send a series of commands to modify various settings to stdout. The - currently recognized commands are: - - - `extra-sandbox-paths`\ - Pass a list of files and directories to be included in the - sandbox for this build. One entry per line, terminated by an - empty line. Entries have the same format as `sandbox-paths`. - )"}; - - Setting postBuildHook{ - this, - "", - "post-build-hook", - R"( - Optional. The path to a program to execute after each build. - - This option is only settable in the global `nix.conf`, or on the - command line by trusted users. - - When using the nix-daemon, the daemon executes the hook as `root`. - If the nix-daemon is not involved, the hook runs as the user - executing the nix-build. - - - The hook executes after an evaluation-time build. - - - The hook does not execute on substituted paths. - - - The hook's output always goes to the user's terminal. - - - If the hook fails, the build succeeds but no further builds - execute. - - - The hook executes synchronously, and blocks other builds from - progressing while it runs. - - The program executes with no arguments. The program's environment - contains the following environment variables: - - - `DRV_PATH` - The derivation for the built paths. - - Example: - `/nix/store/5nihn1a7pa8b25l9zafqaqibznlvvp3f-bash-4.4-p23.drv` - - - `OUT_PATHS` - Output paths of the built derivation, separated by a space - character. - - Example: - `/nix/store/l88brggg9hpy96ijds34dlq4n8fan63g-bash-4.4-p23-dev - /nix/store/vch71bhyi5akr5zs40k8h2wqxx69j80l-bash-4.4-p23-doc - /nix/store/c5cxjywi66iwn9dcx5yvwjkvl559ay6p-bash-4.4-p23-info - /nix/store/scz72lskj03ihkcn42ias5mlp4i4gr1k-bash-4.4-p23-man - /nix/store/a724znygmd1cac856j3gfsyvih3lw07j-bash-4.4-p23`. - )"}; - - Setting downloadSpeed{ - this, - 0, - "download-speed", - R"( - Specify the maximum transfer rate in kilobytes per second you want - Nix to use for downloads. - )"}; - - Setting netrcFile{ - this, - (nixConfDir / "netrc").string(), - "netrc-file", - R"( - If set to an absolute path to a `netrc` file, Nix uses the HTTP - authentication credentials in this file when trying to download from - a remote host through HTTP or HTTPS. Defaults to - `$NIX_CONF_DIR/netrc`. - - The `netrc` file consists of a list of accounts in the following - format: - - machine my-machine - login my-username - password my-password - - For the exact syntax, see [the `curl` - documentation](https://ec.haxx.se/usingcurl-netrc.html). - - > **Note** - > - > This must be an absolute path, and `~` is not resolved. For - > example, `~/.netrc` won't resolve to your home directory's - > `.netrc`. - )"}; - - Setting caFile{ - this, - getDefaultSSLCertFile(), - "ssl-cert-file", - R"( - The path of a file containing CA certificates used to - authenticate `https://` downloads. Nix by default uses - the first of the following files that exists: - - 1. `/etc/ssl/certs/ca-certificates.crt` - 2. `/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt` - - The path can be overridden by the following environment - variables, in order of precedence: - - 1. `NIX_SSL_CERT_FILE` - 2. `SSL_CERT_FILE` - )", - {}, - // Don't document the machine-specific default value - false}; - -#ifdef __linux__ - Setting filterSyscalls{ - this, - true, - "filter-syscalls", - R"( - Whether to prevent certain dangerous system calls, such as - creation of setuid/setgid files or adding ACLs or extended - attributes. Only disable this if you're aware of the - security implications. - )"}; - - Setting allowNewPrivileges{ - this, - false, - "allow-new-privileges", - R"( - (Linux-specific.) By default, builders on Linux cannot acquire new - privileges by calling setuid/setgid programs or programs that have - file capabilities. For example, programs such as `sudo` or `ping` - should fail. (Note that in sandbox builds, no such programs are - available unless you bind-mount them into the sandbox via the - `sandbox-paths` option.) You can allow the use of such programs by - enabling this option. This is impure and usually undesirable, but - may be useful in certain scenarios (e.g. to spin up containers or - set up userspace network interfaces in tests). - )"}; -#endif - -#if NIX_SUPPORT_ACL - Setting ignoredAcls{ - this, - {"security.selinux", "system.nfs4_acl", "security.csm"}, - "ignored-acls", - R"( - A list of ACLs that should be ignored, normally Nix attempts to - remove all ACLs from files and directories in the Nix store, but - some ACLs like `security.selinux` or `system.nfs4_acl` can't be - removed even by root. Therefore it's best to just ignore them. - )"}; -#endif - - Setting hashedMirrors{ - this, - {}, - "hashed-mirrors", - R"( - A list of web servers used by `builtins.fetchurl` to obtain files by - hash. Given a hash algorithm *ha* and a base-16 hash *h*, Nix tries to - download the file from *hashed-mirror*/*ha*/*h*. This allows files to - be downloaded even if they have disappeared from their original URI. - For example, given an example mirror `http://tarballs.nixos.org/`, - when building the derivation - - ```nix - builtins.fetchurl { - url = "https://example.org/foo-1.2.3.tar.xz"; - sha256 = "2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae"; - } - ``` - - Nix will attempt to download this file from - `http://tarballs.nixos.org/sha256/2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae` - first. If it is not available there, it tries the original URI. - )"}; - - Setting minFree{ - this, - 0, - "min-free", - R"( - When free disk space in `/nix/store` drops below `min-free` during a - build, Nix performs a garbage-collection until `max-free` bytes are - available or there is no more garbage. A value of `0` (the default) - disables this feature. - )"}; - - Setting maxFree{// n.b. this is deliberately int64 max rather than uint64 max because - // this goes through the Nix language JSON parser and thus needs to be - // representable in Nix language integers. - this, - std::numeric_limits::max(), - "max-free", - R"( - When a garbage collection is triggered by the `min-free` option, it - stops as soon as `max-free` bytes are available. The default is - infinity (i.e. delete all garbage). - )"}; - - Setting minFreeCheckInterval{ - this, 5, "min-free-check-interval", "Number of seconds between checking free disk space."}; - - Setting narBufferSize{ - this, 32 * 1024 * 1024, "nar-buffer-size", "Maximum size of NARs before spilling them to disk."}; - - Setting allowSymlinkedStore{ - this, - false, - "allow-symlinked-store", - R"( - If set to `true`, Nix stops complaining if the store directory - (typically `/nix/store`) contains symlink components. - - This risks making some builds "impure" because builders sometimes - "canonicalise" paths by resolving all symlink components. Problems - occur if those builds are then deployed to machines where /nix/store - resolves to a different location from that of the build machine. You - can enable this setting if you are sure you're not going to do that. - )"}; - Setting useXDGBaseDirectories{ this, false, @@ -1341,37 +414,6 @@ public: ``` )"}; - Setting impureEnv{ - this, - {}, - "impure-env", - R"( - A list of items, each in the format of: - - - `name=value`: Set environment variable `name` to `value`. - - If the user is trusted (see `trusted-users` option), when building - a fixed-output derivation, environment variables set in this option - is passed to the builder if they are listed in [`impureEnvVars`](@docroot@/language/advanced-attributes.md#adv-attr-impureEnvVars). - - This option is useful for, e.g., setting `https_proxy` for - fixed-output derivations and in a multi-user Nix installation, or - setting private access tokens when fetching a private repository. - )", - {}, // aliases - true, // document default - Xp::ConfigurableImpureEnv}; - - Setting upgradeNixStorePathUrl{ - this, - "", - "upgrade-nix-store-path-url", - R"( - Deprecated. This option was used to configure how `nix upgrade-nix` operated. - - Using this setting has no effect. It will be removed in a future release of Determinate Nix. - )"}; - Setting warnLargePathThreshold{ this, 0, @@ -1383,98 +425,16 @@ public: Set it to 1 to warn on all paths. )"}; - using ExternalBuilders = std::vector; - - Setting externalBuilders{ - this, - {}, - "external-builders", - R"( - Helper programs that execute derivations. - - The program is passed a JSON document that describes the build environment as the final argument. - The JSON document looks like this: - - { - "args": [ - "-e", - "/nix/store/vj1c3wf9…-source-stdenv.sh", - "/nix/store/shkw4qm9…-default-builder.sh" - ], - "builder": "/nix/store/s1qkj0ph…-bash-5.2p37/bin/bash", - "env": { - "HOME": "/homeless-shelter", - "builder": "/nix/store/s1qkj0ph…-bash-5.2p37/bin/bash", - "nativeBuildInputs": "/nix/store/l31j72f1…-version-check-hook", - "out": "/nix/store/2yx2prgx…-hello-2.12.2" - … - }, - "inputPaths": [ - "/nix/store/14dciax3…-glibc-2.32-54-dev", - "/nix/store/1azs5s8z…-gettext-0.21", - … - ], - "outputs": { - "out": "/nix/store/2yx2prgx…-hello-2.12.2" - }, - "realStoreDir": "/nix/store", - "storeDir": "/nix/store", - "system": "aarch64-linux", - "tmpDir": "/private/tmp/nix-build-hello-2.12.2.drv-0/build", - "tmpDirInSandbox": "/build", - "topTmpDir": "/private/tmp/nix-build-hello-2.12.2.drv-0", - "version": 1 - } - )", - {}, // aliases - true, // document default - // NOTE(cole-h): even though we can make the experimental feature required here, the errors - // are not as good (it just becomes a warning if you try to use this setting without the - // experimental feature) - // - // With this commented out: - // - // error: experimental Nix feature 'external-builders' is disabled; add '--extra-experimental-features - // external-builders' to enable it - // - // With this uncommented: - // - // warning: Ignoring setting 'external-builders' because experimental feature 'external-builders' is not enabled - // error: Cannot build '/nix/store/vwsp4qd8…-opentofu-1.10.2.drv'. - // Reason: required system or feature not available - // Required system: 'aarch64-linux' with features {} - // Current system: 'aarch64-darwin' with features {apple-virt, benchmark, big-parallel, nixos-test} - // Xp::ExternalBuilders - }; - /** - * Finds the first external derivation builder that supports this - * derivation, or else returns a null pointer. + * Get the options needed for profile directory functions. */ - const ExternalBuilder * findExternalDerivationBuilderIfSupported(const Derivation & drv); - - Setting hostName{ - this, - "", - "host-name", - R"( - The name of this host for recording build provenance. If unset, the Unix host name is used. - )"}; + ProfileDirsOptions getProfileDirsOptions() const; - std::optional getHostName(); - - Setting> buildProvenanceTags{ - this, - {}, - "build-provenance-tags", - R"( - Arbitrary name/value pairs that are recorded in the build provenance of store paths built by this machine. - This can be used to tag builds with metadata such as the CI job URL, build cluster name, etc. - )"}; + const ExternalBuilder * findExternalDerivationBuilderIfSupported(const Derivation & drv); }; // FIXME: don't use a global variable. -extern Settings settings; +extern nix::Settings settings; /** * Load the configuration (from `nix.conf`, `NIX_CONFIG`, etc.) into the @@ -1484,9 +444,6 @@ extern Settings settings; */ void loadConfFile(AbstractConfig & config); -// Used by the Settings constructor -std::vector getUserConfigFiles(); - /** * The version of Nix itself. * diff --git a/src/libstore/include/nix/store/http-binary-cache-store.hh b/src/libstore/include/nix/store/http-binary-cache-store.hh index ea3d77b7987b..765eb6dd5135 100644 --- a/src/libstore/include/nix/store/http-binary-cache-store.hh +++ b/src/libstore/include/nix/store/http-binary-cache-store.hh @@ -16,19 +16,19 @@ struct HttpBinaryCacheStoreConfig : std::enable_shared_from_this narinfoCompression{ - this, "", "narinfo-compression", "Compression method for `.narinfo` files."}; + Setting> narinfoCompression{ + this, std::nullopt, "narinfo-compression", "Compression method for `.narinfo` files."}; - const Setting lsCompression{this, "", "ls-compression", "Compression method for `.ls` files."}; + Setting> lsCompression{ + this, std::nullopt, "ls-compression", "Compression method for `.ls` files."}; - const Setting logCompression{ + Setting> logCompression{ this, - "", + std::nullopt, "log-compression", R"( Compression method for `log/*` files. It is recommended to @@ -36,6 +36,12 @@ struct HttpBinaryCacheStoreConfig : std::enable_shared_from_this> tlsCert{ + this, std::nullopt, "tls-certificate", "Path to an optional TLS client certificate in PEM format."}; + + Setting> tlsKey{ + this, std::nullopt, "tls-private-key", "Path to an optional TLS client certificate private key in PEM format."}; + static const std::string name() { return "HTTP Binary Cache Store"; @@ -45,6 +51,8 @@ struct HttpBinaryCacheStoreConfig : std::enable_shared_from_this openStore(ref fileTransfer) const; + ref openStore() const override; StoreReference getReference() const override; @@ -60,19 +68,23 @@ class HttpBinaryCacheStore : public virtual BinaryCacheStore Sync _state; +protected: + + ref fileTransfer; + public: using Config = HttpBinaryCacheStoreConfig; ref config; - HttpBinaryCacheStore(ref config); + HttpBinaryCacheStore(ref config, ref fileTransfer = getFileTransfer()); void init() override; protected: - std::optional getCompressionMethod(const std::string & path); + std::optional getCompressionMethod(const std::string & path); void maybeDisable(); diff --git a/src/libstore/include/nix/store/indirect-root-store.hh b/src/libstore/include/nix/store/indirect-root-store.hh index c39e8ea69f70..d477a320aa79 100644 --- a/src/libstore/include/nix/store/indirect-root-store.hh +++ b/src/libstore/include/nix/store/indirect-root-store.hh @@ -55,7 +55,7 @@ struct IndirectRootStore : public virtual LocalFSStore * The implementation of this method is concrete, but it delegates * to `addIndirectRoot()` which is abstract. */ - Path addPermRoot(const StorePath & storePath, const Path & gcRoot) override final; + std::filesystem::path addPermRoot(const StorePath & storePath, const std::filesystem::path & gcRoot) override final; /** * Add an indirect root, which is a weak reference to the @@ -66,10 +66,10 @@ struct IndirectRootStore : public virtual LocalFSStore * * The form this weak-reference takes is implementation-specific. */ - virtual void addIndirectRoot(const Path & path) = 0; + virtual void addIndirectRoot(const std::filesystem::path & path) = 0; protected: - void makeSymlink(const Path & link, const Path & target); + void makeSymlink(const std::filesystem::path & link, const std::filesystem::path & target); }; } // namespace nix diff --git a/src/libstore/include/nix/store/legacy-ssh-store.hh b/src/libstore/include/nix/store/legacy-ssh-store.hh index 6fd077796044..5706aec300f0 100644 --- a/src/libstore/include/nix/store/legacy-ssh-store.hh +++ b/src/libstore/include/nix/store/legacy-ssh-store.hh @@ -14,21 +14,21 @@ struct LegacySSHStoreConfig : std::enable_shared_from_this { using CommonSSHStoreConfig::CommonSSHStoreConfig; - LegacySSHStoreConfig(std::string_view scheme, std::string_view authority, const Params & params); + LegacySSHStoreConfig(const ParsedURL::Authority & authority, const Params & params); #ifndef _WIN32 // Hack for getting remote build log output. // Intentionally not in `LegacySSHStoreConfig` so that it doesn't appear in // the documentation - const Setting logFD{this, INVALID_DESCRIPTOR, "log-fd", "file descriptor to which SSH's stderr is connected"}; + Setting logFD{this, INVALID_DESCRIPTOR, "log-fd", "file descriptor to which SSH's stderr is connected"}; #else Descriptor logFD = INVALID_DESCRIPTOR; #endif - const Setting remoteProgram{ + Setting remoteProgram{ this, {"nix-store"}, "remote-program", "Path to the `nix-store` executable on the remote machine."}; - const Setting maxConnections{this, 1, "max-connections", "Maximum number of concurrent SSH connections."}; + Setting maxConnections{this, 1, "max-connections", "Maximum number of concurrent SSH connections."}; /** * Hack for hydra @@ -95,7 +95,7 @@ struct LegacySSHStore : public virtual Store * * This is exposed for sake of Hydra. */ - void narFromPath(const StorePath & path, std::function fun); + void narFromPath(const StorePath & path, fun receiveNar); std::optional queryPathFromHashPart(const std::string & hashPart) override { @@ -142,7 +142,7 @@ public: * * @todo Use C++23 `std::move_only_function`. */ - std::function buildDerivationAsync( + fun buildDerivationAsync( const StorePath & drvPath, const BasicDerivation & drv, const ServeProto::BuildOptions & options); void buildPaths( @@ -220,6 +220,12 @@ public: { unsupported("queryRealisation"); } + + StorePathSet querySubstitutablePaths(const StorePathSet & paths) override + { + // not supported + return {}; + } }; } // namespace nix diff --git a/src/libstore/include/nix/store/local-binary-cache-store.hh b/src/libstore/include/nix/store/local-binary-cache-store.hh index 2846a9225c78..8c4e68d26398 100644 --- a/src/libstore/include/nix/store/local-binary-cache-store.hh +++ b/src/libstore/include/nix/store/local-binary-cache-store.hh @@ -12,9 +12,9 @@ struct LocalBinaryCacheStoreConfig : std::enable_shared_from_this defaultValue) + static Setting> + makeRootDirSetting(LocalFSStoreConfig & self, std::optional defaultValue) { return { &self, @@ -30,9 +31,9 @@ public: * * @todo Make this less error-prone with new store settings system. */ - LocalFSStoreConfig(PathView path, const Params & params); + LocalFSStoreConfig(const std::filesystem::path & path, const Params & params); - OptionalPathSetting rootDir = makeRootDirSetting(*this, std::nullopt); + Setting> rootDir = makeRootDirSetting(*this, std::nullopt); private: @@ -40,30 +41,36 @@ private: * An indirection so that we don't need to refer to global settings * in headers. */ - static Path getDefaultStateDir(); + static std::filesystem::path getDefaultStateDir(); /** * An indirection so that we don't need to refer to global settings * in headers. */ - static Path getDefaultLogDir(); + static std::filesystem::path getDefaultLogDir(); public: - PathSetting stateDir{ + Setting stateDir{ this, - rootDir.get() ? *rootDir.get() + "/nix/var/nix" : getDefaultStateDir(), + rootDir.get() ? *rootDir.get() / "nix" / "var" / "nix" : getDefaultStateDir(), "state", - "Directory where Nix stores state."}; + "Directory where Nix stores state.", + }; - PathSetting logDir{ + Setting logDir{ this, - rootDir.get() ? *rootDir.get() + "/nix/var/log/nix" : getDefaultLogDir(), + rootDir.get() ? *rootDir.get() / "nix" / "var" / "log" / "nix" : getDefaultLogDir(), "log", - "directory where Nix stores log files."}; + "directory where Nix stores log files.", + }; - PathSetting realStoreDir{ - this, rootDir.get() ? *rootDir.get() + "/nix/store" : storeDir, "real", "Physical path of the Nix store."}; + Setting realStoreDir{ + this, + rootDir.get() ? *rootDir.get() / "nix" / "store" : std::filesystem::path{storeDir}, + "real", + "Physical path of the Nix store.", + }; }; struct alignas(8) /* Work around ASAN failures on i686-linux. */ @@ -77,7 +84,7 @@ struct alignas(8) /* Work around ASAN failures on i686-linux. */ inline static std::string operationName = "Local Filesystem Store"; - const static std::string drvsLogDir; + const static std::filesystem::path drvsLogDir; LocalFSStore(const Config & params); @@ -93,27 +100,21 @@ struct alignas(8) /* Work around ASAN failures on i686-linux. */ * @param gcRoot The location of the symlink. * * @param storePath The store object being rooted. The symlink will - * point to `toRealPath(store.printStorePath(storePath))`. + * point to `toRealPath(storePath)`. * * How the permanent GC root corresponding to this symlink is * managed is implementation-specific. */ - virtual Path addPermRoot(const StorePath & storePath, const Path & gcRoot) = 0; + virtual std::filesystem::path addPermRoot(const StorePath & storePath, const std::filesystem::path & gcRoot) = 0; - virtual Path getRealStoreDir() + virtual std::filesystem::path getRealStoreDir() { return config.realStoreDir; } - Path toRealPath(const StorePath & storePath) - { - return toRealPath(printStorePath(storePath)); - } - - Path toRealPath(const Path & storePath) + std::filesystem::path toRealPath(const StorePath & storePath) { - assert(isInStore(storePath)); - return getRealStoreDir() + "/" + std::string(storePath, storeDir.size() + 1); + return getRealStoreDir() / storePath.to_string(); } std::optional getBuildLogExact(const StorePath & path) override; diff --git a/src/libstore/include/nix/store/local-gc.hh b/src/libstore/include/nix/store/local-gc.hh new file mode 100644 index 000000000000..d609626e2f56 --- /dev/null +++ b/src/libstore/include/nix/store/local-gc.hh @@ -0,0 +1,20 @@ +#include "nix/store/gc-store.hh" +#include +#include + +namespace nix { + +/** + * Finds a list of "runtime roots", i.e. store paths currently open, + * mapped, or in the environment of a process and should not be deleted. + * + * This function does not attempt to check the nix database and find if paths are + * valid. It may return paths in the store that look like nix paths, + * but are not known to the nix daemon or may not even exist. + * + * @param config Configuration for the store, needed to find the store dir + * @return a map from store paths to processes that are using them + */ +Roots findRuntimeRootsUnchecked(const StoreDirConfig & config); + +} // namespace nix diff --git a/src/libstore/include/nix/store/local-overlay-store.hh b/src/libstore/include/nix/store/local-overlay-store.hh index 1d69d3417086..5d8ec1b4571a 100644 --- a/src/libstore/include/nix/store/local-overlay-store.hh +++ b/src/libstore/include/nix/store/local-overlay-store.hh @@ -8,18 +8,18 @@ namespace nix { struct LocalOverlayStoreConfig : virtual LocalStoreConfig { LocalOverlayStoreConfig(const StringMap & params) - : LocalOverlayStoreConfig("local-overlay", "", params) + : LocalOverlayStoreConfig("", params) { } - LocalOverlayStoreConfig(std::string_view scheme, PathView path, const Params & params) + LocalOverlayStoreConfig(const std::filesystem::path & path, const Params & params) : StoreConfig(params) , LocalFSStoreConfig(path, params) - , LocalStoreConfig(scheme, path, params) + , LocalStoreConfig(path, params) { } - const Setting lowerStoreUri{ + Setting lowerStoreUri{ (StoreConfig *) this, "", "lower-store", @@ -31,7 +31,7 @@ struct LocalOverlayStoreConfig : virtual LocalStoreConfig Must be used as OverlayFS lower layer for this store's store dir. )"}; - const PathSetting upperLayer{ + const Setting upperLayer{ (StoreConfig *) this, "", "upper-layer", @@ -53,7 +53,7 @@ struct LocalOverlayStoreConfig : virtual LocalStoreConfig default, but can be disabled if needed. )"}; - const PathSetting remountHook{ + const Setting remountHook{ (StoreConfig *) this, "", "remount-hook", @@ -99,7 +99,7 @@ protected: * at that file path. It might be stored in the lower layer instead, * or it might not be part of this store at all. */ - Path toUpperPath(const StorePath & path) const; + std::filesystem::path toUpperPath(const StorePath & path) const; friend struct LocalOverlayStore; }; @@ -184,7 +184,7 @@ private: * Check which layers the store object exists in to try to avoid * needing to remount. */ - void deleteStorePath(const Path & path, uint64_t & bytesFreed) override; + void deleteStorePath(const std::filesystem::path & path, uint64_t & bytesFreed, bool isKnownPath) override; /** * Deduplicate by removing store objects from the upper layer that diff --git a/src/libstore/include/nix/store/local-settings.hh b/src/libstore/include/nix/store/local-settings.hh new file mode 100644 index 000000000000..9d28c3706482 --- /dev/null +++ b/src/libstore/include/nix/store/local-settings.hh @@ -0,0 +1,726 @@ +#pragma once +///@file + +#include "nix/util/types.hh" +#include "nix/util/configuration.hh" +#include "nix/util/experimental-features.hh" +#include "nix/util/users.hh" +#include "nix/store/build/derivation-builder.hh" + +#include "nix/store/config.hh" + +#include +#include +#include +#include + +namespace nix { + +typedef enum { smEnabled, smRelaxed, smDisabled } SandboxMode; + +template<> +SandboxMode BaseSetting::parse(const std::string & str) const; +template<> +std::string BaseSetting::to_string() const; + +template<> +PathsInChroot BaseSetting::parse(const std::string & str) const; +template<> +std::string BaseSetting::to_string() const; + +template<> +struct BaseSetting::trait +{ + static constexpr bool appendable = true; +}; + +template<> +void BaseSetting::appendOrSet(PathsInChroot newValue, bool append); + +template<> +std::map BaseSetting::toJSONObject() const; + +template<> +std::vector BaseSetting>::parse(const std::string & str) const; +template<> +std::string BaseSetting>::to_string() const; + +struct GCSettings : public virtual Config +{ + Setting reservedSize{ + this, + 8 * 1024 * 1024, + "gc-reserved-space", + "Amount of reserved disk space for the garbage collector.", + }; + + Setting keepOutputs{ + this, + false, + "keep-outputs", + R"( + If `true`, the garbage collector keeps the outputs of + non-garbage derivations. If `false` (default), outputs are + deleted unless they are GC roots themselves (or reachable from other + roots). + + In general, outputs must be registered as roots separately. However, + even if the output of a derivation is registered as a root, the + collector still deletes store paths that are used only at build + time (e.g., the C compiler, or source tarballs downloaded from the + network). To prevent it from doing so, set this option to `true`. + )", + {"gc-keep-outputs"}, + }; + + Setting keepDerivations{ + this, + true, + "keep-derivations", + R"( + If `true` (default), the garbage collector keeps the derivations + from which non-garbage store paths were built. If `false`, they are + deleted unless explicitly registered as a root (or reachable from + other roots). + + Keeping derivation around is useful for querying and traceability + (e.g., it allows you to ask with what dependencies or options a + store path was built), so by default this option is on. Turn it off + to save a bit of disk space (or a lot if `keep-outputs` is also + turned on). + )", + {"gc-keep-derivations"}, + }; + + Setting minFree{ + this, + 0, + "min-free", + R"( + When free disk space in `/nix/store` drops below `min-free` during a + build, Nix performs a garbage-collection until `max-free` bytes are + available or there is no more garbage. A value of `0` (the default) + disables this feature. + )", + }; + + // n.b. this is deliberately int64 max rather than uint64 max because + // this goes through the Nix language JSON parser and thus needs to be + // representable in Nix language integers. + Setting maxFree{ + this, + std::numeric_limits::max(), + "max-free", + R"( + When a garbage collection is triggered by the `min-free` option, it + stops as soon as `max-free` bytes are available. The default is + infinity (i.e. delete all garbage). + )", + }; + + Setting minFreeCheckInterval{ + this, + 5, + "min-free-check-interval", + "Number of seconds between checking free disk space.", + }; +}; + +const uint32_t maxIdsPerBuild = +#ifdef __linux__ + 1 << 16 +#else + 1 +#endif + ; + +struct AutoAllocateUidSettings : public virtual Config +{ + Setting startId{ + this, +#ifdef __linux__ + 0x34000000, +#else + 56930, +#endif + "start-id", + "The first UID and GID to use for dynamic ID allocation."}; + + Setting uidCount{ + this, +#ifdef __linux__ + maxIdsPerBuild * 128, +#else + 128, +#endif + "id-count", + "The number of UIDs/GIDs to use for dynamic ID allocation."}; +}; + +/** + * Either about local store or local building + * + * These are things that should not be part of the global settings, but + * should be per-local-store at a minimum. We expose them from + * `settings` with `settings.getLocalSettings()` for now, but we also + * have `localStore.config->getLocalSettings()` as a way to get them + * too. Even though both ways will actually draw from the same global + * variable, we would much prefer if you use the second one, because + * this will prepare the code base to making these *actual*, rather than + * pretend, per-store settings. + */ +struct LocalSettings : public virtual Config, public GCSettings, public AutoAllocateUidSettings +{ + /** + * Get the GC settings. + */ + GCSettings & getGCSettings() + { + return *this; + } + + const GCSettings & getGCSettings() const + { + return *this; + } + + /** + * Get AutoAllocateUidSettings if auto-allocate-uids is enabled. + * @return Pointer to settings if enabled, nullptr otherwise. + */ + const AutoAllocateUidSettings * getAutoAllocateUidSettings() const + { + return autoAllocateUids ? this : nullptr; + } + + Setting buildCores{ + this, + 0, + "cores", + R"( + Sets the value of the `NIX_BUILD_CORES` environment variable in the [invocation of the `builder` executable](@docroot@/store/building.md#builder-execution) of a derivation. + The `builder` executable can use this variable to control its own maximum amount of parallelism. + + + For instance, in Nixpkgs, if the attribute `enableParallelBuilding` for the `mkDerivation` build helper is set to `true`, it passes the `-j${NIX_BUILD_CORES}` flag to GNU Make. + + If set to `0`, nix will detect the number of CPU cores and pass this number via `NIX_BUILD_CORES`. + + > **Note** + > + > The number of parallel local Nix build jobs is independently controlled with the [`max-jobs`](#conf-max-jobs) setting. + )", + {"build-cores"}}; + + Setting fsyncMetadata{ + this, + true, + "fsync-metadata", + R"( + If set to `true`, changes to the Nix store metadata (in + `/nix/var/nix/db`) are synchronously flushed to disk. This improves + robustness in case of system crashes, but reduces performance. The + default is `true`. + )"}; + + Setting fsyncStorePaths{ + this, + false, + "fsync-store-paths", + R"( + Whether to call `fsync()` on store paths before registering them, to + flush them to disk. This improves robustness in case of system crashes, + but reduces performance. The default is `false`. + )"}; + +#ifndef _WIN32 + // FIXME: remove this option, `fsync-store-paths` is faster. + Setting syncBeforeRegistering{ + this, false, "sync-before-registering", "Whether to call `sync()` before registering a path as valid."}; +#endif + + Setting autoOptimiseStore{ + this, + false, + "auto-optimise-store", + R"( + If set to `true`, Nix automatically detects files in the store + that have identical contents, and replaces them with hard links to + a single copy. This saves disk space. If set to `false` (the + default), you can still run `nix-store --optimise` to get rid of + duplicate files. + )"}; + + Setting narBufferSize{ + this, 32 * 1024 * 1024, "nar-buffer-size", "Maximum size of NARs before spilling them to disk."}; + + Setting allowSymlinkedStore{ + this, + false, + "allow-symlinked-store", + R"( + If set to `true`, Nix stops complaining if the store directory + (typically `/nix/store`) contains symlink components. + + This risks making some builds "impure" because builders sometimes + "canonicalise" paths by resolving all symlink components. Problems + occur if those builds are then deployed to machines where /nix/store + resolves to a different location from that of the build machine. You + can enable this setting if you are sure you're not going to do that. + )"}; + + Setting buildUsersGroup{ + this, + "", + "build-users-group", + R"( + This options specifies the Unix group containing the Nix build user + accounts. In multi-user Nix installations, builds should not be + performed by the Nix account since that would allow users to + arbitrarily modify the Nix store and database by supplying specially + crafted builders; and they cannot be performed by the calling user + since that would allow him/her to influence the build result. + + Therefore, if this option is non-empty and specifies a valid group, + builds are performed under the user accounts that are a member + of the group specified here (as listed in `/etc/group`). Those user + accounts should not be used for any other purpose\! + + Nix never runs two builds under the same user account at the + same time. This is to prevent an obvious security hole: a malicious + user writing a Nix expression that modifies the build result of a + legitimate Nix expression being built by another user. Therefore it + is good to have as many Nix build user accounts as you can spare. + (Remember: uids are cheap.) + + The build users should have permission to create files in the Nix + store, but not delete them. Therefore, `/nix/store` should be owned + by the Nix account, its group should be the group specified here, + and its mode should be `1775`. + + If the build users group is empty, builds are performed under + the uid of the Nix process (that is, the uid of the caller if + `NIX_REMOTE` is empty, the uid under which the Nix daemon runs if + `NIX_REMOTE` is `daemon`). Obviously, this should not be used + with a nix daemon accessible to untrusted clients. + + Defaults to `nixbld` when running as root, *empty* otherwise. + )", + {}, + false}; + + Setting autoAllocateUids{ + this, + false, + "auto-allocate-uids", + R"( + Whether to select UIDs for builds automatically, instead of using the + users in `build-users-group`. + + UIDs are allocated starting at 872415232 (0x34000000) on Linux and 56930 on macOS. + )", + {}, + true, + Xp::AutoAllocateUids}; + +#ifdef __linux__ + Setting useCgroups{ + this, + false, + "use-cgroups", + R"( + Whether to execute builds inside cgroups. + This is only supported on Linux. + + Cgroups are required and enabled automatically for derivations + that require the `uid-range` system feature. + )"}; +#endif + + Setting impersonateLinux26{ + this, + false, + "impersonate-linux-26", + "Whether to impersonate a Linux 2.6 machine on newer kernels.", + {"build-impersonate-linux-26"}}; + + Setting sandboxMode{ + this, +#ifdef __linux__ + smEnabled +#else + smDisabled +#endif + , + "sandbox", + R"( + If set to `true`, builds are performed in a *sandboxed + environment*, i.e., they're isolated from the normal file system + hierarchy and only see their dependencies in the Nix store, + the temporary build directory, private versions of `/proc`, + `/dev`, `/dev/shm` and `/dev/pts` (on Linux), and the paths + configured with the `sandbox-paths` option. This is useful to + prevent undeclared dependencies on files in directories such as + `/usr/bin`. In addition, on Linux, builds run in private PID, + mount, network, IPC and UTS namespaces to isolate them from other + processes in the system (except that fixed-output derivations do + not run in private network namespace to ensure they can access the + network). + + Currently, sandboxing only work on Linux and macOS. The use of a + sandbox requires that Nix is run as root (so you should use the + "build users" feature to perform the actual builds under different + users than root). + + If this option is set to `relaxed`, then fixed-output derivations + and derivations that have the `__noChroot` attribute set to `true` + do not run in sandboxes. + + The default is `true` on Linux and `false` on all other platforms. + )", + {"build-use-chroot", "build-use-sandbox"}}; + + Setting sandboxPaths{ + this, + {}, + "sandbox-paths", + R"( + A list of paths bind-mounted into Nix sandbox environments. You can + use the syntax `target=source` to mount a path in a different + location in the sandbox; for instance, `/bin=/nix-bin` mounts + the path `/nix-bin` as `/bin` inside the sandbox. If *source* is + followed by `?`, then it is not an error if *source* does not exist; + for example, `/dev/nvidiactl?` specifies that `/dev/nvidiactl` + only be mounted in the sandbox if it exists in the host filesystem. + + If the source is in the Nix store, then its closure is added to + the sandbox as well. + + Depending on how Nix was built, the default value for this option + may be empty or provide `/bin/sh` as a bind-mount of `bash`. + )", + {"build-chroot-dirs", "build-sandbox-paths"}}; + + Setting sandboxFallback{ + this, true, "sandbox-fallback", "Whether to disable sandboxing when the kernel doesn't allow it."}; + +#ifndef _WIN32 + Setting requireDropSupplementaryGroups{ + this, + isRootUser(), + "require-drop-supplementary-groups", + R"( + Following the principle of least privilege, + Nix attempts to drop supplementary groups when building with sandboxing. + + However this can fail under some circumstances. + For example, if the user lacks the `CAP_SETGID` capability. + Search `setgroups(2)` for `EPERM` to find more detailed information on this. + + If you encounter such a failure, setting this option to `false` enables you to ignore it and continue. + But before doing so, you should consider the security implications carefully. + Not dropping supplementary groups means the build sandbox is less restricted than intended. + + This option defaults to `true` when the user is root + (since `root` usually has permissions to call setgroups) + and `false` otherwise. + )"}; +#endif + +#ifdef __linux__ + Setting sandboxShmSize{ + this, + "50%", + "sandbox-dev-shm-size", + R"( + *Linux only* + + This option determines the maximum size of the `tmpfs` filesystem + mounted on `/dev/shm` in Linux sandboxes. For the format, see the + description of the `size` option of `tmpfs` in mount(8). The default + is `50%`. + )"}; +#endif + +#if defined(__linux__) || defined(__FreeBSD__) + Setting sandboxBuildDir{ + this, + "/build", + "sandbox-build-dir", + R"( + *Linux only* + + The build directory inside the sandbox. + + This directory is backed by [`build-dir`](#conf-build-dir) on the host. + )"}; +#endif + + Setting> buildDir{ + this, + std::nullopt, + "build-dir", + R"( + Override the `build-dir` store setting for all stores that have this setting. + + See also the per-store [`build-dir`](@docroot@/store/types/local-store.md#store-local-store-build-dir) setting. + )"}; + + Setting> allowedImpureHostPrefixes{ + this, + {}, + "allowed-impure-host-deps", + "Which prefixes to allow derivations to ask for access to (primarily for Darwin)."}; + +#ifdef __APPLE__ + Setting darwinLogSandboxViolations{ + this, + false, + "darwin-log-sandbox-violations", + "Whether to log Darwin sandbox access violations to the system log."}; +#endif + + Setting runDiffHook{ + this, + false, + "run-diff-hook", + R"( + If true, enable the execution of the `diff-hook` program. + + When using the Nix daemon, `run-diff-hook` must be set in the + `nix.conf` configuration file, and cannot be passed at the command + line. + )"}; + +private: + + Setting> diffHook{ + this, + std::nullopt, + "diff-hook", + R"( + Absolute path to an executable capable of diffing build + results. The hook is executed if `run-diff-hook` is true, and the + output of a build is known to not be the same. This program is not + executed to determine if two results are the same. + + The diff hook is executed by the same user and group who ran the + build. However, the diff hook does not have write access to the + store path just built. + + The diff hook program receives three parameters: + + 1. A path to the previous build's results + + 2. A path to the current build's results + + 3. The path to the build's derivation + + 4. The path to the build's scratch directory. This directory + exists only if the build was run with `--keep-failed`. + + The stderr and stdout output from the diff hook isn't displayed + to the user. Instead, it prints to the nix-daemon's log. + + When using the Nix daemon, `diff-hook` must be set in the `nix.conf` + configuration file, and cannot be passed at the command line. + )"}; + +public: + + /** + * Get the diff hook path if run-diff-hook is enabled. + * @return Pointer to path if enabled, nullptr otherwise. + */ + const std::filesystem::path * getDiffHook() const + { + if (!runDiffHook.get()) { + return nullptr; + } + return get(diffHook.get()); + } + + Setting preBuildHook{ + this, + "", + "pre-build-hook", + R"( + If set, the path to a program that can set extra derivation-specific + settings for this system. This is used for settings that can't be + captured by the derivation model itself and are too variable between + different versions of the same system to be hard-coded into nix. + + The hook is passed the derivation path and, if sandboxes are + enabled, the sandbox directory. It can then modify the sandbox and + send a series of commands to modify various settings to stdout. The + currently recognized commands are: + + - `extra-sandbox-paths`\ + Pass a list of files and directories to be included in the + sandbox for this build. One entry per line, terminated by an + empty line. Entries have the same format as `sandbox-paths`. + )"}; + +#ifdef __linux__ + Setting filterSyscalls{ + this, + true, + "filter-syscalls", + R"( + Whether to prevent certain dangerous system calls, such as + creation of setuid/setgid files or adding ACLs or extended + attributes. Only disable this if you're aware of the + security implications. + )"}; + + Setting allowNewPrivileges{ + this, + false, + "allow-new-privileges", + R"( + (Linux-specific.) By default, builders on Linux cannot acquire new + privileges by calling setuid/setgid programs or programs that have + file capabilities. For example, programs such as `sudo` or `ping` + should fail. (Note that in sandbox builds, no such programs are + available unless you bind-mount them into the sandbox via the + `sandbox-paths` option.) You can allow the use of such programs by + enabling this option. This is impure and usually undesirable, but + may be useful in certain scenarios (e.g. to spin up containers or + set up userspace network interfaces in tests). + )"}; +#endif + +#if NIX_SUPPORT_ACL + Setting ignoredAcls{ + this, + {"security.selinux", "system.nfs4_acl", "security.csm"}, + "ignored-acls", + R"( + A list of ACLs that should be ignored, normally Nix attempts to + remove all ACLs from files and directories in the Nix store, but + some ACLs like `security.selinux` or `system.nfs4_acl` can't be + removed even by root. Therefore it's best to just ignore them. + )"}; +#endif + + Setting impureEnv{ + this, + {}, + "impure-env", + R"( + A list of items, each in the format of: + + - `name=value`: Set environment variable `name` to `value`. + + If the user is trusted (see `trusted-users` option), when building + a fixed-output derivation, environment variables set in this option + is passed to the builder if they are listed in [`impureEnvVars`](@docroot@/language/advanced-attributes.md#adv-attr-impureEnvVars). + + This option is useful for, e.g., setting `https_proxy` for + fixed-output derivations and in a multi-user Nix installation, or + setting private access tokens when fetching a private repository. + )", + {}, // aliases + true, // document default + Xp::ConfigurableImpureEnv}; + + Setting hashedMirrors{ + this, + {}, + "hashed-mirrors", + R"( + A list of web servers used by `builtins.fetchurl` to obtain files by + hash. Given a hash algorithm *ha* and a base-16 hash *h*, Nix tries to + download the file from *hashed-mirror*/*ha*/*h*. This allows files to + be downloaded even if they have disappeared from their original URI. + For example, given an example mirror `http://tarballs.nixos.org/`, + when building the derivation + + ```nix + builtins.fetchurl { + url = "https://example.org/foo-1.2.3.tar.xz"; + sha256 = "2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae"; + } + ``` + + Nix will attempt to download this file from + `http://tarballs.nixos.org/sha256/2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae` + first. If it is not available there, it tries the original URI. + )"}; + + using ExternalBuilders = std::vector; + + Setting externalBuilders{ + this, + {}, + "external-builders", + R"( + Helper programs that execute derivations. + + The program is passed a JSON document that describes the build environment as the final argument. + The JSON document looks like this: + + { + "args": [ + "-e", + "/nix/store/vj1c3wf9…-source-stdenv.sh", + "/nix/store/shkw4qm9…-default-builder.sh" + ], + "builder": "/nix/store/s1qkj0ph…-bash-5.2p37/bin/bash", + "env": { + "HOME": "/homeless-shelter", + "builder": "/nix/store/s1qkj0ph…-bash-5.2p37/bin/bash", + "nativeBuildInputs": "/nix/store/l31j72f1…-version-check-hook", + "out": "/nix/store/2yx2prgx…-hello-2.12.2" + … + }, + "inputPaths": [ + "/nix/store/14dciax3…-glibc-2.32-54-dev", + "/nix/store/1azs5s8z…-gettext-0.21", + … + ], + "outputs": { + "out": "/nix/store/2yx2prgx…-hello-2.12.2" + }, + "realStoreDir": "/nix/store", + "storeDir": "/nix/store", + "system": "aarch64-linux", + "tmpDir": "/private/tmp/nix-build-hello-2.12.2.drv-0/build", + "tmpDirInSandbox": "/build", + "topTmpDir": "/private/tmp/nix-build-hello-2.12.2.drv-0", + "version": 1 + } + )", + {}, // aliases + true, // document default + // NOTE(cole-h): even though we can make the experimental feature required here, the errors + // are not as good (it just becomes a warning if you try to use this setting without the + // experimental feature) + // + // With this commented out: + // + // error: experimental Nix feature 'external-builders' is disabled; add '--extra-experimental-features + // external-builders' to enable it + // + // With this uncommented: + // + // warning: Ignoring setting 'external-builders' because experimental feature 'external-builders' is not enabled + // error: Cannot build '/nix/store/vwsp4qd8…-opentofu-1.10.2.drv'. + // Reason: required system or feature not available + // Required system: 'aarch64-linux' with features {} + // Current system: 'aarch64-darwin' with features {apple-virt, benchmark, big-parallel, nixos-test} + // Xp::ExternalBuilders + }; + + /** + * Finds the first external derivation builder that supports this + * derivation, or else returns a null pointer. + */ + const ExternalBuilder * findExternalDerivationBuilderIfSupported(const Derivation & drv); +}; + +} // namespace nix diff --git a/src/libstore/include/nix/store/local-store.hh b/src/libstore/include/nix/store/local-store.hh index 40aa7c699e78..64786aa3c848 100644 --- a/src/libstore/include/nix/store/local-store.hh +++ b/src/libstore/include/nix/store/local-store.hh @@ -32,6 +32,8 @@ struct OptimiseStats uint64_t bytesFreed = 0; }; +struct LocalSettings; + struct LocalBuildStoreConfig : virtual LocalFSStoreConfig { @@ -39,7 +41,7 @@ private: /** Input for computing the build directory. See `getBuildDir()`. */ - Setting> buildDir{ + Setting> buildDir{ this, std::nullopt, "build-dir", @@ -66,7 +68,13 @@ private: See also the global [`build-dir`](@docroot@/command-ref/conf-file.md#conf-build-dir) setting. )"}; public: - Path getBuildDir() const; + /** + * For now, this just grabs the global local settings, but by having this method we get ready for these being + * per-store settings instead. + */ + const LocalSettings & getLocalSettings() const; + + std::filesystem::path getBuildDir() const; }; struct LocalStoreConfig : std::enable_shared_from_this, @@ -75,7 +83,7 @@ struct LocalStoreConfig : std::enable_shared_from_this, { using LocalFSStoreConfig::LocalFSStoreConfig; - LocalStoreConfig(std::string_view scheme, std::string_view authority, const Params & params); + LocalStoreConfig(const std::filesystem::path & path, const Params & params); private: @@ -86,7 +94,6 @@ private: bool getDefaultRequireSigs(); public: - Setting requireSigs{ this, getDefaultRequireSigs(), @@ -111,6 +118,43 @@ public: > While the filesystem the database resides on might appear to be read-only, consider whether another user or system might have write access to it. )"}; + bool getReadOnly() const override; + + Setting ignoreGcDeleteFailure{ + this, + false, + "ignore-gc-delete-failure", + R"( + Whether to ignore failures when deleting items with the garbage collector. + + Normally the garbage collector will fail with an error if the nix daemon cannot delete a file, with this setting such errors will only be printed as warnings. + )", + {}, + true, + Xp::LocalOverlayStore, + }; + + Setting useRootsDaemon{ + this, + false, + "use-roots-daemon", + R"( + Whether to request garbage collector roots from an external daemon. + + When enabled, the garbage collector connects to a Unix domain socket + at [``](@docroot@/store/types/local-store.md#store-option-state)`/gc-roots-socket/socket` to discover additional roots + that should not be collected. This is useful when the Nix daemon runs + without root privileges and cannot scan `/proc` for runtime roots. + + The daemon can be started with [`nix store roots-daemon`](@docroot@/command-ref/new-cli/nix3-store-roots-daemon.md). + )", + {}, + true, + Xp::LocalOverlayStore, + }; + + std::filesystem::path getRootsSocketPath() const; + static const std::string name() { return "Local Store"; @@ -188,12 +232,12 @@ private: public: - const Path dbDir; - const Path linksDir; - const Path reservedPath; - const Path schemaPath; - const Path tempRootsDir; - const Path fnTempRoots; + const std::filesystem::path dbDir; + const std::filesystem::path linksDir; + const std::filesystem::path reservedPath; + const std::filesystem::path schemaPath; + const std::filesystem::path tempRootsDir; + const std::filesystem::path fnTempRoots; private: @@ -236,8 +280,6 @@ public: std::optional queryPathFromHashPart(const std::string & hashPart) override; - StorePathSet querySubstitutablePaths(const StorePathSet & paths) override; - bool pathInfoIsUntrusted(const ValidPathInfo &) override; bool realisationIsUntrusted(const Realisation &) override; @@ -282,7 +324,7 @@ public: * The weak reference merely is a symlink to `path' from * /nix/var/nix/gcroots/auto/. */ - void addIndirectRoot(const Path & path) override; + void addIndirectRoot(const std::filesystem::path & path) override; private: @@ -312,8 +354,11 @@ public: * Called by `collectGarbage` to recursively delete a path. * The default implementation simply calls `deletePath`, but it can be * overridden by stores that wish to provide their own deletion behaviour. + * + * @param isKnownPath true if this is a known store path, false if it's + * garbage/unknown content found in the store directory */ - virtual void deleteStorePath(const Path & path, uint64_t & bytesFreed); + virtual void deleteStorePath(const std::filesystem::path & path, uint64_t & bytesFreed, bool isKnownPath); /** * Optimise the disk space usage of the Nix store by hard-linking @@ -327,7 +372,7 @@ public: * Optimise a single store path. Optionally, test the encountered * symlinks for corruption. */ - void optimisePath(const Path & path, RepairFlag repair); + void optimisePath(const std::filesystem::path & path, RepairFlag repair); bool verifyStore(bool checkContents, RepairFlag repair) override; @@ -375,7 +420,7 @@ public: void vacuumDB(); - void addSignatures(const StorePath & storePath, const StringSet & sigs) override; + void addSignatures(const StorePath & storePath, const std::set & sigs) override; /** * If free disk space in /nix/store if below minFree, delete @@ -403,7 +448,7 @@ protected: void verifyPath( const StorePath & path, - std::function existsInStoreDir, + fun existsInStoreDir, StorePathSet & done, StorePathSet & validPaths, RepairFlag repair, @@ -425,7 +470,7 @@ private: uint64_t queryValidPathId(State & state, const StorePath & path); - uint64_t addValidPath(State & state, const ValidPathInfo & info, bool checkOutputs = true); + uint64_t addValidPath(State & state, const ValidPathInfo & info); void invalidatePath(State & state, const StorePath & path); @@ -438,10 +483,7 @@ private: void updatePathInfo(State & state, const ValidPathInfo & info); - PathSet queryValidPathsOld(); - ValidPathInfo queryPathInfoOld(const Path & path); - - void findRoots(const Path & path, std::filesystem::file_type type, Roots & roots); + void findRoots(const std::filesystem::path & path, std::filesystem::file_type type, Roots & roots); void findRootsNoTemp(Roots & roots, bool censor); @@ -452,9 +494,13 @@ private: typedef boost::unordered_flat_set InodeHash; InodeHash loadInodeHash(); - Strings readDirectoryIgnoringInodes(const Path & path, const InodeHash & inodeHash); - void - optimisePath_(Activity * act, OptimiseStats & stats, const Path & path, InodeHash & inodeHash, RepairFlag repair); + Strings readDirectoryIgnoringInodes(const std::filesystem::path & path, const InodeHash & inodeHash); + void optimisePath_( + Activity * act, + OptimiseStats & stats, + const std::filesystem::path & path, + InodeHash & inodeHash, + RepairFlag repair); // Internal versions that are not wrapped in retry_sqlite. bool isValidPath_(State & state, const StorePath & path); diff --git a/src/libstore/include/nix/store/machines.hh b/src/libstore/include/nix/store/machines.hh index 1f7bb669ab53..6b3484699491 100644 --- a/src/libstore/include/nix/store/machines.hh +++ b/src/libstore/include/nix/store/machines.hh @@ -17,7 +17,7 @@ struct Machine const StoreReference storeUri; const StringSet systemTypes; - const std::string sshKey; + const std::filesystem::path sshKey; const unsigned int maxJobs; const float speedFactor; const StringSet supportedFeatures; @@ -79,11 +79,4 @@ struct Machine static Machines parseConfig(const StringSet & defaultSystems, const std::string & config); }; -/** - * Parse machines from the global config - * - * @todo Remove, globals are bad. - */ -Machines getMachines(); - } // namespace nix diff --git a/src/libstore/include/nix/store/meson.build b/src/libstore/include/nix/store/meson.build index f99cf39040cf..9900e64c67a0 100644 --- a/src/libstore/include/nix/store/meson.build +++ b/src/libstore/include/nix/store/meson.build @@ -15,6 +15,7 @@ headers = [ config_pub_h ] + files( 'aws-creds.hh', 'binary-cache-store.hh', 'build-result.hh', + 'build/build-log.hh', 'build/derivation-builder.hh', 'build/derivation-building-goal.hh', 'build/derivation-building-misc.hh', @@ -43,6 +44,7 @@ headers = [ config_pub_h ] + files( 'export-import.hh', 'filetransfer.hh', 'gc-store.hh', + 'global-paths.hh', 'globals.hh', 'http-binary-cache-store.hh', 'indirect-root-store.hh', @@ -51,7 +53,9 @@ headers = [ config_pub_h ] + files( 'length-prefixed-protocol-helper.hh', 'local-binary-cache-store.hh', 'local-fs-store.hh', + 'local-gc.hh', 'local-overlay-store.hh', + 'local-settings.hh', 'local-store.hh', 'log-store.hh', 'machines.hh', @@ -94,4 +98,5 @@ headers = [ config_pub_h ] + files( 'worker-protocol-connection.hh', 'worker-protocol-impl.hh', 'worker-protocol.hh', + 'worker-settings.hh', ) diff --git a/src/libstore/include/nix/store/nar-info-disk-cache.hh b/src/libstore/include/nix/store/nar-info-disk-cache.hh index 253487b30333..a30c5a553b96 100644 --- a/src/libstore/include/nix/store/nar-info-disk-cache.hh +++ b/src/libstore/include/nix/store/nar-info-disk-cache.hh @@ -7,14 +7,26 @@ namespace nix { -class NarInfoDiskCache +struct SQLiteSettings; +struct NarInfoDiskCacheSettings; + +struct NarInfoDiskCache { -public: + using Settings = NarInfoDiskCacheSettings; + + const Settings & settings; + + NarInfoDiskCache(const Settings & settings) + : settings(settings) + { + } + typedef enum { oValid, oInvalid, oUnknown } Outcome; virtual ~NarInfoDiskCache() {} - virtual int createCache(const std::string & uri, const Path & storeDir, bool wantMassQuery, int priority) = 0; + virtual int + createCache(const std::string & uri, const std::string & storeDir, bool wantMassQuery, int priority) = 0; struct CacheInfo { @@ -35,14 +47,20 @@ public: virtual void upsertAbsentRealisation(const std::string & uri, const DrvOutput & id) = 0; virtual std::pair> lookupRealisation(const std::string & uri, const DrvOutput & id) = 0; -}; -/** - * Return a singleton cache object that can be used concurrently by - * multiple threads. - */ -ref getNarInfoDiskCache(); + /** + * Return a singleton cache object that can be used concurrently by + * multiple threads. + * + * @note the parameters are only used to initialize this the first time this is called. + * In subsequent calls, these arguments are ignored. + * + * @todo Probably should instead create a memo table so multiple settings -> multiple instances, + * but this is not yet a problem in practice. + */ + static ref get(const Settings & settings, SQLiteSettings); -ref getTestNarInfoDiskCache(Path dbPath); + static ref getTest(const Settings & settings, SQLiteSettings, std::filesystem::path dbPath); +}; } // namespace nix diff --git a/src/libstore/include/nix/store/nar-info.hh b/src/libstore/include/nix/store/nar-info.hh index ac25f75c2cdf..7403dc1d4452 100644 --- a/src/libstore/include/nix/store/nar-info.hh +++ b/src/libstore/include/nix/store/nar-info.hh @@ -1,6 +1,7 @@ #pragma once ///@file +#include "nix/util/compression-algo.hh" #include "nix/util/types.hh" #include "nix/util/hash.hh" #include "nix/store/path-info.hh" @@ -12,7 +13,7 @@ struct StoreDirConfig; struct UnkeyedNarInfo : virtual UnkeyedValidPathInfo { std::string url; - std::string compression; + std::string compression; // FIXME: Use CompressionAlgo std::optional fileHash; uint64_t fileSize = 0; @@ -42,7 +43,7 @@ struct NarInfo : ValidPathInfo, UnkeyedNarInfo /* Later copies from `*this` are pointless. The argument is only there so the constructors can also call `UnkeyedValidPathInfo`, but this won't happen since the base - class is virtual. Only this counstructor (assuming it is most + class is virtual. Only this constructor (assuming it is most derived) will initialize that virtual base class. */ , ValidPathInfo{info.path, static_cast(*this)} , UnkeyedNarInfo{static_cast(*this)} diff --git a/src/libstore/include/nix/store/path-info.hh b/src/libstore/include/nix/store/path-info.hh index 374b90a26ae0..98f9ebe7aeda 100644 --- a/src/libstore/include/nix/store/path-info.hh +++ b/src/libstore/include/nix/store/path-info.hh @@ -103,7 +103,7 @@ struct UnkeyedValidPathInfo */ bool ultimate = false; - StringSet sigs; // note: not necessarily verified + std::set sigs; /** * If non-empty, an assertion that the path is content-addressed, @@ -207,7 +207,7 @@ struct ValidPathInfo : virtual UnkeyedValidPathInfo /** * Verify a single signature. */ - bool checkSignature(const StoreDirConfig & store, const PublicKeys & publicKeys, const std::string & sig) const; + bool checkSignature(const StoreDirConfig & store, const PublicKeys & publicKeys, const Signature & sig) const; /** * References as store path basenames, including a self reference if it has one. diff --git a/src/libstore/include/nix/store/path-references.hh b/src/libstore/include/nix/store/path-references.hh index 6aa506da4a33..2aa656adbef7 100644 --- a/src/libstore/include/nix/store/path-references.hh +++ b/src/libstore/include/nix/store/path-references.hh @@ -10,7 +10,7 @@ namespace nix { -StorePathSet scanForReferences(Sink & toTee, const Path & path, const StorePathSet & refs); +StorePathSet scanForReferences(Sink & toTee, const std::filesystem::path & path, const StorePathSet & refs); class PathRefScanSink : public RefScanSink { @@ -59,7 +59,7 @@ void scanForReferencesDeep( SourceAccessor & accessor, const CanonPath & rootPath, const StorePathSet & refs, - std::function callback); + fun callback); /** * Scan a store path tree and return which references appear in which files. diff --git a/src/libstore/include/nix/store/pathlocks.hh b/src/libstore/include/nix/store/pathlocks.hh index 7e27bec4cc10..dbb75a676976 100644 --- a/src/libstore/include/nix/store/pathlocks.hh +++ b/src/libstore/include/nix/store/pathlocks.hh @@ -33,6 +33,22 @@ private: public: PathLocks(); PathLocks(const std::set & paths, const std::string & waitMsg = ""); + + PathLocks(PathLocks && other) noexcept + : fds(std::exchange(other.fds, {})) + , deletePaths(other.deletePaths) + { + } + + PathLocks & operator=(PathLocks && other) noexcept + { + fds = std::exchange(other.fds, {}); + deletePaths = other.deletePaths; + return *this; + } + + PathLocks(const PathLocks &) = delete; + PathLocks & operator=(const PathLocks &) = delete; bool lockPaths(const std::set & _paths, const std::string & waitMsg = "", bool wait = true); ~PathLocks(); void unlock(); @@ -45,6 +61,10 @@ struct FdLock bool acquired = false; FdLock(Descriptor desc, LockType lockType, bool wait, std::string_view waitMsg); + FdLock(const FdLock &) = delete; + FdLock & operator=(const FdLock &) = delete; + FdLock(FdLock &&) = delete; + FdLock & operator=(FdLock &&) = delete; ~FdLock() { diff --git a/src/libstore/include/nix/store/posix-fs-canonicalise.hh b/src/libstore/include/nix/store/posix-fs-canonicalise.hh index 629759cfec3d..8a8b0eca309a 100644 --- a/src/libstore/include/nix/store/posix-fs-canonicalise.hh +++ b/src/libstore/include/nix/store/posix-fs-canonicalise.hh @@ -4,14 +4,51 @@ #include #include +#include + #include "nix/util/types.hh" #include "nix/util/error.hh" +#include "nix/store/config.hh" namespace nix { typedef std::pair Inode; typedef std::set InodesSeen; +struct CanonicalizePathMetadataOptions +{ +#ifndef _WIN32 + /** + * If uidRange is not empty, this function will throw an error if it + * encounters files owned by a user outside of the closed interval + * [uidRange->first, uidRange->second]. + */ + std::optional> uidRange; +#endif + +#if NIX_SUPPORT_ACL + /** + * A list of ACLs that should be ignored when canonicalising. + * Normally Nix attempts to remove all ACLs from files and directories + * in the Nix store, but some ACLs like `security.selinux` or + * `system.nfs4_acl` can't be removed even by root. + */ + const StringSet & ignoredAcls; +#endif +}; + +/** + * Makes it easier to cope with conditionally-available fields. + * + * @todo Switch to a better way, as having a macro is not the nicest. + * This will be easier to do after further settings refactors. + */ +#if NIX_SUPPORT_ACL +# define NIX_WHEN_SUPPORT_ACLS(ARG) .ignoredAcls = ARG, +#else +# define NIX_WHEN_SUPPORT_ACLS(ARG) +#endif + /** * "Fix", or canonicalise, the meta-data of the files in a store path * after it has been built. In particular: @@ -24,27 +61,13 @@ typedef std::set InodesSeen; * * - the owner and group are set to the Nix user and group, if we're * running as root. (Unix only.) - * - * If uidRange is not empty, this function will throw an error if it - * encounters files owned by a user outside of the closed interval - * [uidRange->first, uidRange->second]. */ void canonicalisePathMetaData( - const Path & path, -#ifndef _WIN32 - std::optional> uidRange, -#endif - InodesSeen & inodesSeen); + const std::filesystem::path & path, CanonicalizePathMetadataOptions options, InodesSeen & inodesSeen); -void canonicalisePathMetaData( - const Path & path -#ifndef _WIN32 - , - std::optional> uidRange = std::nullopt -#endif -); +void canonicalisePathMetaData(const std::filesystem::path & path, CanonicalizePathMetadataOptions options); -void canonicaliseTimestampAndPermissions(const Path & path); +void canonicaliseTimestampAndPermissions(const std::filesystem::path & path); MakeError(PathInUse, Error); diff --git a/src/libstore/include/nix/store/profiles.hh b/src/libstore/include/nix/store/profiles.hh index 1cc306744f79..11a3fbe89f01 100644 --- a/src/libstore/include/nix/store/profiles.hh +++ b/src/libstore/include/nix/store/profiles.hh @@ -209,32 +209,38 @@ void lockProfile(PathLocks & lock, const std::filesystem::path & profile); */ std::string optimisticLockProfile(const std::filesystem::path & profile); +struct ProfileDirsOptions +{ + const std::filesystem::path & nixStateDir; + bool useXDGBaseDirectories; +}; + /** * Create and return the path to a directory suitable for storing the user’s * profiles. */ -std::filesystem::path profilesDir(); +std::filesystem::path profilesDir(ProfileDirsOptions opts); /** * Return the path to the profile directory for root (but don't try creating it) */ -std::filesystem::path rootProfilesDir(); +std::filesystem::path rootProfilesDir(ProfileDirsOptions opts); /** * Create and return the path to the file used for storing the users's channels */ -std::filesystem::path defaultChannelsDir(); +std::filesystem::path defaultChannelsDir(ProfileDirsOptions opts); /** * Return the path to the channel directory for root (but don't try creating it) */ -std::filesystem::path rootChannelsDir(); +std::filesystem::path rootChannelsDir(ProfileDirsOptions opts); /** * Resolve the default profile (~/.nix-profile by default, * $XDG_STATE_HOME/nix/profile if XDG Base Directory Support is enabled), * and create if doesn't exist */ -std::filesystem::path getDefaultProfile(); +std::filesystem::path getDefaultProfile(ProfileDirsOptions opts); } // namespace nix diff --git a/src/libstore/include/nix/store/provenance.hh b/src/libstore/include/nix/store/provenance.hh index c3ae4f8a6b23..5f9773e724ea 100644 --- a/src/libstore/include/nix/store/provenance.hh +++ b/src/libstore/include/nix/store/provenance.hh @@ -1,6 +1,7 @@ #pragma once #include "nix/util/provenance.hh" +#include "nix/util/types.hh" #include "nix/store/path.hh" #include "nix/store/outputs-spec.hh" @@ -26,7 +27,7 @@ struct BuildProvenance : Provenance /** * User-defined tags from the build host. */ - std::map tags; + StringMap tags; /** * The system type of the derivation. @@ -44,7 +45,7 @@ struct BuildProvenance : Provenance const StorePath & drvPath, const OutputName & output, std::optional buildHost, - std::map tags, + StringMap tags, std::string system, std::shared_ptr next); diff --git a/src/libstore/include/nix/store/realisation.hh b/src/libstore/include/nix/store/realisation.hh index af0e4aefd8a2..670f8ce499ec 100644 --- a/src/libstore/include/nix/store/realisation.hh +++ b/src/libstore/include/nix/store/realisation.hh @@ -54,21 +54,13 @@ struct UnkeyedRealisation { StorePath outPath; - StringSet signatures; - - /** - * The realisations that are required for the current one to be valid. - * - * When importing this realisation, the store will first check that all its - * dependencies exist, and map to the correct output path - */ - std::map dependentRealisations; + std::set signatures; std::string fingerprint(const DrvOutput & key) const; void sign(const DrvOutput & key, const Signer &); - bool checkSignature(const DrvOutput & key, const PublicKeys & publicKeys, const std::string & sig) const; + bool checkSignature(const DrvOutput & key, const PublicKeys & publicKeys, const Signature & sig) const; size_t checkSignatures(const DrvOutput & key, const PublicKeys & publicKeys) const; @@ -87,10 +79,6 @@ struct Realisation : UnkeyedRealisation bool isCompatibleWith(const UnkeyedRealisation & other) const; - static std::set closure(Store &, const std::set &); - - static void closure(Store &, const std::set &, std::set & res); - bool operator==(const Realisation &) const = default; auto operator<=>(const Realisation &) const = default; }; @@ -154,15 +142,11 @@ struct RealisedPath */ const StorePath & path() const &; - void closure(Store & store, Set & ret) const; - static void closure(Store & store, const Set & startPaths, Set & ret); - Set closure(Store & store) const; - bool operator==(const RealisedPath &) const = default; auto operator<=>(const RealisedPath &) const = default; }; -class MissingRealisation : public Error +class MissingRealisation final : public CloneableError { public: MissingRealisation(DrvOutput & outputId) @@ -171,7 +155,7 @@ public: } MissingRealisation(std::string_view drv, OutputName outputName) - : Error( + : CloneableError( "cannot operate on output '%s' of the " "unbuilt derivation '%s'", outputName, diff --git a/src/libstore/include/nix/store/remote-fs-accessor.hh b/src/libstore/include/nix/store/remote-fs-accessor.hh index 9e1999cc0611..9a84e1e97891 100644 --- a/src/libstore/include/nix/store/remote-fs-accessor.hh +++ b/src/libstore/include/nix/store/remote-fs-accessor.hh @@ -3,6 +3,7 @@ #include "nix/util/source-accessor.hh" #include "nix/util/ref.hh" +#include "nix/util/nar-cache.hh" #include "nix/store/store-api.hh" namespace nix { @@ -11,20 +12,21 @@ class RemoteFSAccessor : public SourceAccessor { ref store; - std::map> nars; + /** + * Map from store path hash part to NAR hash. Used to then look up + * in the NAR cache. The indirection allows avoiding opening multiple + * redundant NAR accessors for the same NAR. + */ + std::map> narHashes; - bool requireValidPath; + NarCache narCache; - Path cacheDir; + bool requireValidPath; std::pair, CanonPath> fetch(const CanonPath & path); friend struct BinaryCacheStore; - Path makeCacheFile(std::string_view hashPart, const std::string & ext); - - ref addToCache(std::string_view hashPart, std::string && nar); - public: /** @@ -32,14 +34,15 @@ public: */ std::shared_ptr accessObject(const StorePath & path); - RemoteFSAccessor( - ref store, bool requireValidPath = true, const /* FIXME: use std::optional */ Path & cacheDir = ""); + RemoteFSAccessor(ref store, bool requireValidPath = true, std::optional cacheDir = {}); std::optional maybeLstat(const CanonPath & path) override; DirEntries readDirectory(const CanonPath & path) override; - std::string readFile(const CanonPath & path) override; + void readFile(const CanonPath & path, Sink & sink, fun sizeCallback) override; + + using SourceAccessor::readFile; std::string readLink(const CanonPath & path) override; }; diff --git a/src/libstore/include/nix/store/remote-store-connection.hh b/src/libstore/include/nix/store/remote-store-connection.hh index c2010818c4dc..16a90c3f66fe 100644 --- a/src/libstore/include/nix/store/remote-store-connection.hh +++ b/src/libstore/include/nix/store/remote-store-connection.hh @@ -59,7 +59,7 @@ struct RemoteStore::ConnectionHandle void processStderr(Sink * sink = 0, Source * source = 0, bool flush = true, bool block = true); - void withFramedSink(std::function fun); + void withFramedSink(fun sendData); }; } // namespace nix diff --git a/src/libstore/include/nix/store/remote-store.hh b/src/libstore/include/nix/store/remote-store.hh index 3644c55f206e..289d694aedd3 100644 --- a/src/libstore/include/nix/store/remote-store.hh +++ b/src/libstore/include/nix/store/remote-store.hh @@ -2,9 +2,12 @@ ///@file #include +#include #include #include "nix/store/store-api.hh" +#include "nix/util/sync.hh" +#include "nix/util/file-descriptor.hh" #include "nix/store/gc-store.hh" #include "nix/store/log-store.hh" #include "nix/store/active-builds.hh" @@ -23,10 +26,10 @@ struct RemoteStoreConfig : virtual StoreConfig { using StoreConfig::StoreConfig; - const Setting maxConnections{ + Setting maxConnections{ this, 64, "max-connections", "Maximum number of concurrent connections to the Nix daemon."}; - const Setting maxConnectionAge{ + Setting maxConnectionAge{ this, std::numeric_limits::max(), "max-connection-age", @@ -100,8 +103,6 @@ struct RemoteStore : public virtual Store, void addToStore(const ValidPathInfo & info, Source & nar, RepairFlag repair, CheckSigsFlag checkSigs) override; - void addMultipleToStore(Source & source, RepairFlag repair, CheckSigsFlag checkSigs) override; - void addMultipleToStore(PathsSource && pathsToCopy, Activity & act, RepairFlag repair, CheckSigsFlag checkSigs) override; @@ -143,7 +144,7 @@ struct RemoteStore : public virtual Store, unsupported("repairPath"); } - void addSignatures(const StorePath & storePath, const StringSet & sigs) override; + void addSignatures(const StorePath & storePath, const std::set & sigs) override; MissingPaths queryMissing(const std::vector & targets) override; @@ -161,6 +162,13 @@ struct RemoteStore : public virtual Store, void flushBadConnections(); + /** + * Shutdown all connections (both idle and in-use) to break any blocking I/O. + * This is called on interrupt to allow graceful termination when the client + * disconnects during a long-running operation. + */ + void shutdownConnections(); + struct Connection; ref openConnectionWrapper(); @@ -199,6 +207,12 @@ private: std::atomic_bool failed{false}; + /** + * Track all active connection file descriptors (both idle and in-use). + * Used by shutdownConnections() to break blocking I/O on interrupt. + */ + Sync> connectionFds; + void copyDrvsFromEvalStore(const std::vector & paths, std::shared_ptr evalStore); }; diff --git a/src/libstore/include/nix/store/restricted-store.hh b/src/libstore/include/nix/store/restricted-store.hh index 62cac3856752..ca4e0b8536a7 100644 --- a/src/libstore/include/nix/store/restricted-store.hh +++ b/src/libstore/include/nix/store/restricted-store.hh @@ -59,6 +59,8 @@ struct RestrictionContext addDependencyImpl(path); } + virtual ~RestrictionContext() = default; + protected: /** diff --git a/src/libstore/include/nix/store/s3-binary-cache-store.hh b/src/libstore/include/nix/store/s3-binary-cache-store.hh index e679035e461b..0edb1ea3547a 100644 --- a/src/libstore/include/nix/store/s3-binary-cache-store.hh +++ b/src/libstore/include/nix/store/s3-binary-cache-store.hh @@ -11,9 +11,11 @@ struct S3BinaryCacheStoreConfig : HttpBinaryCacheStoreConfig { using HttpBinaryCacheStoreConfig::HttpBinaryCacheStoreConfig; - S3BinaryCacheStoreConfig(std::string_view uriScheme, std::string_view bucketName, const Params & params); + S3BinaryCacheStoreConfig(ParsedURL cacheUri, const Params & params); - const Setting profile{ + S3BinaryCacheStoreConfig(std::string_view bucketName, const Params & params); + + Setting profile{ this, "default", "profile", @@ -22,7 +24,7 @@ struct S3BinaryCacheStoreConfig : HttpBinaryCacheStoreConfig Nix uses the `default` profile. )"}; - const Setting region{ + Setting region{ this, "us-east-1", "region", @@ -32,7 +34,7 @@ struct S3BinaryCacheStoreConfig : HttpBinaryCacheStoreConfig parameter. )"}; - const Setting scheme{ + Setting scheme{ this, "https", "scheme", @@ -47,7 +49,7 @@ struct S3BinaryCacheStoreConfig : HttpBinaryCacheStoreConfig > information. )"}; - const Setting endpoint{ + Setting endpoint{ this, "", "endpoint", @@ -71,7 +73,7 @@ struct S3BinaryCacheStoreConfig : HttpBinaryCacheStoreConfig contain dots). )"}; - const Setting multipartUpload{ + Setting multipartUpload{ this, false, "multipart-upload", @@ -82,7 +84,7 @@ struct S3BinaryCacheStoreConfig : HttpBinaryCacheStoreConfig can improve performance and reliability for large uploads. )"}; - const Setting multipartChunkSize{ + Setting multipartChunkSize{ this, 5 * 1024 * 1024, "multipart-chunk-size", @@ -93,7 +95,7 @@ struct S3BinaryCacheStoreConfig : HttpBinaryCacheStoreConfig )", {"buffer-size"}}; - const Setting multipartThreshold{ + Setting multipartThreshold{ this, 100 * 1024 * 1024, "multipart-threshold", @@ -103,7 +105,7 @@ struct S3BinaryCacheStoreConfig : HttpBinaryCacheStoreConfig Default is 100 MiB. Only takes effect when multipart-upload is enabled. )"}; - const Setting> storageClass{ + Setting> storageClass{ this, std::nullopt, "storage-class", diff --git a/src/libstore/include/nix/store/serve-protocol-connection.hh b/src/libstore/include/nix/store/serve-protocol-connection.hh index fa50132c88be..21fd09c24962 100644 --- a/src/libstore/include/nix/store/serve-protocol-connection.hh +++ b/src/libstore/include/nix/store/serve-protocol-connection.hh @@ -81,9 +81,9 @@ struct ServeProto::BasicClientConnection */ BuildResult getBuildDerivationResponse(const StoreDirConfig & store); - void narFromPath(const StoreDirConfig & store, const StorePath & path, std::function fun); + void narFromPath(const StoreDirConfig & store, const StorePath & path, fun receiveNar); - void importPaths(const StoreDirConfig & store, std::function fun); + void importPaths(const StoreDirConfig & store, fun sendPaths); }; struct ServeProto::BasicServerConnection diff --git a/src/libstore/include/nix/store/serve-protocol.hh b/src/libstore/include/nix/store/serve-protocol.hh index 974bf42d58d2..dba05a34548f 100644 --- a/src/libstore/include/nix/store/serve-protocol.hh +++ b/src/libstore/include/nix/store/serve-protocol.hh @@ -8,10 +8,6 @@ namespace nix { #define SERVE_MAGIC_1 0x390c9deb #define SERVE_MAGIC_2 0x5452eecb -#define SERVE_PROTOCOL_VERSION (2 << 8 | 7) -#define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) -#define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) - struct StoreDirConfig; struct Source; @@ -37,7 +33,38 @@ struct ServeProto * * @todo Convert to struct with separate major vs minor fields. */ - using Version = unsigned int; + struct Version + { + unsigned int major; + uint8_t minor; + + constexpr auto operator<=>(const Version &) const = default; + + /** + * Convert to wire format for protocol compatibility. + * Format: (major << 8) | minor + */ + constexpr unsigned int toWire() const + { + return (major << 8) | minor; + } + + /** + * Convert from wire format. + */ + static constexpr Version fromWire(unsigned int wire) + { + return { + .major = (wire & 0xff00) >> 8, + .minor = static_cast(wire & 0x00ff), + }; + } + }; + + static constexpr Version latest = { + .major = 2, + .minor = 7, + }; /** * A unidirectional read connection, to be used by the read half of the diff --git a/src/libstore/include/nix/store/sqlite.hh b/src/libstore/include/nix/store/sqlite.hh index 3495c0bd1439..789e82174627 100644 --- a/src/libstore/include/nix/store/sqlite.hh +++ b/src/libstore/include/nix/store/sqlite.hh @@ -33,6 +33,12 @@ enum class SQLiteOpenMode { Immutable, }; +struct SQLiteSettings +{ + SQLiteOpenMode mode = SQLiteOpenMode::Normal; + bool useWAL; +}; + /** * RAII wrapper to close a SQLite database automatically. */ @@ -42,7 +48,9 @@ struct SQLite SQLite() {} - SQLite(const std::filesystem::path & path, SQLiteOpenMode mode = SQLiteOpenMode::Normal); + using Settings = SQLiteSettings; + + SQLite(const std::filesystem::path & path, Settings && settings); SQLite(const SQLite & from) = delete; SQLite & operator=(const SQLite & from) = delete; @@ -158,7 +166,7 @@ struct SQLiteTxn ~SQLiteTxn(); }; -struct SQLiteError : Error +struct SQLiteError : CloneableError { std::string path; std::string errMsg; @@ -201,7 +209,7 @@ void handleSQLiteBusy(const SQLiteBusy & e, time_t & nextWarning); template T retrySQLite(F && fun) { - time_t nextWarning = time(0) + 1; + time_t nextWarning = time(nullptr) + 1; while (true) { try { diff --git a/src/libstore/include/nix/store/ssh-store.hh b/src/libstore/include/nix/store/ssh-store.hh index 9584a1a862c1..f10068ee4e2e 100644 --- a/src/libstore/include/nix/store/ssh-store.hh +++ b/src/libstore/include/nix/store/ssh-store.hh @@ -15,9 +15,9 @@ struct SSHStoreConfig : std::enable_shared_from_this, using CommonSSHStoreConfig::CommonSSHStoreConfig; using RemoteStoreConfig::RemoteStoreConfig; - SSHStoreConfig(std::string_view scheme, std::string_view authority, const Params & params); + SSHStoreConfig(const ParsedURL::Authority & authority, const Params & params); - const Setting remoteProgram{ + Setting remoteProgram{ this, {"nix-daemon"}, "remote-program", "Path to the `nix-daemon` executable on the remote machine."}; static const std::string name() @@ -40,7 +40,7 @@ struct SSHStoreConfig : std::enable_shared_from_this, struct MountedSSHStoreConfig : virtual SSHStoreConfig, virtual LocalFSStoreConfig { MountedSSHStoreConfig(StringMap params); - MountedSSHStoreConfig(std::string_view scheme, std::string_view host, StringMap params); + MountedSSHStoreConfig(const ParsedURL::Authority & authority, StringMap params); static const std::string name() { diff --git a/src/libstore/include/nix/store/ssh.hh b/src/libstore/include/nix/store/ssh.hh index 574cb5cf4142..c64e9b1cca93 100644 --- a/src/libstore/include/nix/store/ssh.hh +++ b/src/libstore/include/nix/store/ssh.hh @@ -9,7 +9,7 @@ namespace nix { -Strings getNixSshOpts(); +OsStrings getNixSshOpts(); class SSHMaster { @@ -18,7 +18,7 @@ private: ParsedURL::Authority authority; std::string hostnameAndUser; bool fakeSSH; - const std::string keyFile; + const std::filesystem::path keyFile; /** * Raw bytes, not Base64 encoding. */ @@ -34,23 +34,23 @@ private: #ifndef _WIN32 // TODO re-enable on Windows, once we can start processes. Pid sshMaster; #endif - Path socketPath; + std::filesystem::path socketPath; }; Sync state_; - void addCommonSSHOpts(Strings & args); + void addCommonSSHOpts(OsStrings & args); bool isMasterRunning(); #ifndef _WIN32 // TODO re-enable on Windows, once we can start processes. - Path startMaster(); + std::filesystem::path startMaster(); #endif public: SSHMaster( const ParsedURL::Authority & authority, - std::string_view keyFile, + std::filesystem::path keyFile, std::string_view sshPublicHostKey, bool useMaster, bool compress, @@ -83,7 +83,7 @@ public: * execute). Will not be used when "fake SSHing" to the local * machine. */ - std::unique_ptr startCommand(Strings && command, Strings && extraSshArgs = {}); + std::unique_ptr startCommand(OsStrings && command, OsStrings && extraSshArgs = {}); }; } // namespace nix diff --git a/src/libstore/include/nix/store/store-api.hh b/src/libstore/include/nix/store/store-api.hh index 133e2d5df650..89dd231a0cf4 100644 --- a/src/libstore/include/nix/store/store-api.hh +++ b/src/libstore/include/nix/store/store-api.hh @@ -40,9 +40,10 @@ struct BasicDerivation; struct Derivation; struct SourceAccessor; -class NarInfoDiskCache; +struct NarInfoDiskCache; +struct NarInfoDiskCacheSettings; class Store; - +struct AsyncPathWriter; struct Provenance; typedef std::map OutputPathMap; @@ -73,6 +74,32 @@ struct MissingPaths uint64_t narSize{0}; }; +/** + * A setting for the Nix store directory. Automatically canonicalises the + * path and rejects the empty string. Stored as `std::string` because + * store directory are valid file paths on *some* OS, but not neccessarily the OS of this build of Nix. + * + * (For example, consider `SSHStore` from Linux to Windows, or vice versa, the foreign path will not be a valid + * `std::filesystem::path`.) + */ +class StoreDirSetting : public BaseSetting +{ +public: + StoreDirSetting( + Config * options, + const std::string & def, + const std::string & name, + const std::string & description, + const StringSet & aliases = {}); + + std::string parse(const std::string & str) const override; + + void operator=(const std::string & v) + { + this->assign(v); + } +}; + /** * Need to make this a separate class so I can get the right * initialization order in the constructor for `StoreConfig`. @@ -84,14 +111,14 @@ struct StoreConfigBase : Config private: /** - * An indirection so that we don't need to refer to global settings - * in headers. + * Compute the default Nix store directory from environment variables + * (`NIX_STORE_DIR`, `NIX_STORE`) or the compile-time default. */ - static Path getDefaultNixStoreDir(); + static std::string getDefaultNixStoreDir(); public: - const PathSetting storeDir_{ + StoreDirSetting storeDir_{ this, getDefaultNixStoreDir(), "store", @@ -220,6 +247,12 @@ struct StoreConfig : public StoreConfigBase, public StoreDirConfig // Don't document the machine-specific default value false}; + /** + * Whether we're allowed to write to this store, also takes into account + * global `readOnly`'s mode setting, not just any per-store settings. + */ + virtual bool getReadOnly() const; + /** * Open a store of the type corresponding to this configuration * type. @@ -296,7 +329,7 @@ protected: * Whether the value is valid as a cache entry. The path may not * exist. */ - bool isKnownNow(); + bool isKnownNow(const NarInfoDiskCacheSettings & settings); /** * Past tense, because a path can only be assumed to exists when @@ -330,7 +363,7 @@ public: /** * Follow symlinks until we end up with a path in the Nix store. */ - Path followLinksToStore(std::string_view path) const; + std::filesystem::path followLinksToStore(std::string_view path) const; /** * Same as followLinksToStore(), but apply toStorePath() to the @@ -508,10 +541,7 @@ public: /** * Query which of the given paths have substitutes. */ - virtual StorePathSet querySubstitutablePaths(const StorePathSet & paths) - { - return {}; - }; + virtual StorePathSet querySubstitutablePaths(const StorePathSet & paths); /** * Query substitute info (i.e. references, derivers and download @@ -541,8 +571,6 @@ public: /** * Import multiple paths into the store. */ - virtual void addMultipleToStore(Source & source, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs); - virtual void addMultipleToStore( PathsSource && pathsToCopy, Activity & act, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs); @@ -770,7 +798,7 @@ public: * Add signatures to the specified store path. The signatures are * not verified. */ - virtual void addSignatures(const StorePath & storePath, const StringSet & sigs) + virtual void addSignatures(const StorePath & storePath, const std::set & sigs) { unsupported("addSignatures"); } @@ -797,6 +825,15 @@ public: virtual StorePath writeDerivation( const Derivation & drv, RepairFlag repair = NoRepair, std::shared_ptr provenance = nullptr); + /** + * Asynchronously write a derivation to the Nix store, and return its path. + */ + StorePath writeDerivation( + AsyncPathWriter & asyncPathWriter, + const Derivation & drv, + RepairFlag repair = NoRepair, + std::shared_ptr provenance = nullptr); + /** * Read a derivation (which must already be valid). */ @@ -1014,26 +1051,11 @@ void removeTempRoots(); StorePath resolveDerivedPath(Store &, const SingleDerivedPath &, Store * evalStore = nullptr); OutputPathMap resolveDerivedPath(Store &, const DerivedPath::Built &, Store * evalStore = nullptr); -/** - * Display a set of paths in human-readable form (i.e., between quotes - * and separated by commas). - */ -std::string showPaths(const PathSet & paths); - -/** - * Display a set of paths in human-readable form (i.e., between quotes - * and separated by commas). - */ -std::string showPaths(const std::set paths); - std::optional decodeValidPathInfo(const Store & store, std::istream & str, std::optional hashGiven = std::nullopt); const ContentAddress * getDerivationCA(const BasicDerivation & drv); -std::map -drvOutputReferences(Store & store, const Derivation & drv, const StorePath & outputPath, Store * evalStore = nullptr); - template<> struct json_avoids_null : std::true_type {}; diff --git a/src/libstore/include/nix/store/store-dir-config.hh b/src/libstore/include/nix/store/store-dir-config.hh index 34e928182ad6..d6d97b4ff6ae 100644 --- a/src/libstore/include/nix/store/store-dir-config.hh +++ b/src/libstore/include/nix/store/store-dir-config.hh @@ -1,6 +1,7 @@ #pragma once #include "nix/store/path.hh" +#include "nix/util/canon-path.hh" #include "nix/util/hash.hh" #include "nix/store/content-address.hh" #include "nix/util/configuration.hh" @@ -29,7 +30,7 @@ MakeError(BadStorePathName, BadStorePath); */ struct StoreDirConfig { - const Path & storeDir; + const std::string & storeDir; // pure methods @@ -44,21 +45,19 @@ struct StoreDirConfig * * \todo remove */ - StorePathSet parseStorePathSet(const PathSet & paths) const; + StorePathSet parseStorePathSet(const StringSet & paths) const; - PathSet printStorePathSet(const StorePathSet & path) const; + StringSet printStorePathSet(const StorePathSet & path) const; /** * Display a set of paths in human-readable form (i.e., between quotes * and separated by commas). */ - std::string showPaths(const StorePathSet & paths) const; - /** * @return true if *path* is in the Nix store (but not the Nix * store itself). */ - bool isInStore(PathView path) const; + bool isInStore(std::string_view path) const; /** * @return true if *path* is a store path, i.e. a direct child of the @@ -70,7 +69,7 @@ struct StoreDirConfig * Split a path like `/nix/store/-/` into * `/nix/store/-` and `/`. */ - std::pair toStorePath(PathView path) const; + std::pair toStorePath(std::string_view path) const; /** * Constructs a unique store path name. diff --git a/src/libstore/include/nix/store/store-reference.hh b/src/libstore/include/nix/store/store-reference.hh index dc34500d9cbe..37ff2cdee3d5 100644 --- a/src/libstore/include/nix/store/store-reference.hh +++ b/src/libstore/include/nix/store/store-reference.hh @@ -4,6 +4,8 @@ #include #include "nix/util/types.hh" +#include "nix/util/json-impls.hh" +#include "nix/util/json-non-null.hh" namespace nix { @@ -54,6 +56,7 @@ struct StoreReference /** * General case, a regular `scheme://authority` URL. + * @todo Consider making this pluggable instead of passing through the encoded authority + path. */ struct Specified { @@ -93,6 +96,7 @@ struct StoreReference Params params; bool operator==(const StoreReference & rhs) const = default; + auto operator<=>(const StoreReference & rhs) const = default; /** * Render the whole store reference as a URI, optionally including parameters. @@ -120,4 +124,10 @@ static inline std::ostream & operator<<(std::ostream & os, const StoreReference */ std::pair splitUriAndParams(const std::string & uri); +template<> +struct json_avoids_null : std::true_type +{}; + } // namespace nix + +JSON_IMPL(StoreReference) diff --git a/src/libstore/include/nix/store/store-registration.hh b/src/libstore/include/nix/store/store-registration.hh index 8b0f344ba38f..51db33e12854 100644 --- a/src/libstore/include/nix/store/store-registration.hh +++ b/src/libstore/include/nix/store/store-registration.hh @@ -14,6 +14,7 @@ */ #include "nix/store/store-api.hh" +#include "nix/util/url.hh" namespace nix { @@ -39,8 +40,7 @@ struct StoreFactory * The `authorityPath` parameter is `/`, or really * whatever comes after `://` and before `?`. */ - std::function( - std::string_view scheme, std::string_view authorityPath, const Store::Config::Params & params)> + fun(std::string_view scheme, std::string_view authorityPath, const Store::Config::Params & params)> parseConfig; /** @@ -48,7 +48,7 @@ struct StoreFactory * because it means we cannot require fields to be manually * specified so easily. */ - std::function()> getConfig; + fun()> getConfig; }; struct Implementations @@ -65,7 +65,16 @@ struct Implementations .uriSchemes = TConfig::uriSchemes(), .experimentalFeature = TConfig::experimentalFeature(), .parseConfig = ([](auto scheme, auto uri, auto & params) -> ref { - return make_ref(scheme, uri, params); + if constexpr (std::is_constructible_v) { + std::filesystem::path path = percentDecode(uri); + return make_ref(path.empty() ? std::filesystem::path{} : canonPath(path), params); + } else if constexpr (std::is_constructible_v) { + return make_ref(parseURL(concatStrings(scheme, "://", uri)), params); + } else if constexpr (std::is_constructible_v) { + return make_ref(ParsedURL::Authority::parse(uri), params); + } else { + return make_ref(scheme, uri, params); + } }), .getConfig = ([]() -> ref { return make_ref(Store::Config::Params{}); }), }; diff --git a/src/libstore/include/nix/store/uds-remote-store.hh b/src/libstore/include/nix/store/uds-remote-store.hh index 764e8768a32b..64672ee3adfa 100644 --- a/src/libstore/include/nix/store/uds-remote-store.hh +++ b/src/libstore/include/nix/store/uds-remote-store.hh @@ -11,15 +11,10 @@ struct UDSRemoteStoreConfig : std::enable_shared_from_this virtual LocalFSStoreConfig, virtual RemoteStoreConfig { - // TODO(fzakaria): Delete this constructor once moved over to the factory pattern - // outlined in https://github.com/NixOS/nix/issues/10766 using LocalFSStoreConfig::LocalFSStoreConfig; using RemoteStoreConfig::RemoteStoreConfig; - /** - * @param authority is the socket path. - */ - UDSRemoteStoreConfig(std::string_view scheme, std::string_view authority, const Params & params); + UDSRemoteStoreConfig(const std::filesystem::path & path, const Params & params); UDSRemoteStoreConfig(const Params & params); @@ -36,7 +31,7 @@ struct UDSRemoteStoreConfig : std::enable_shared_from_this * The default is `settings.nixDaemonSocketFile`, but we don't write * that below, instead putting in the constructor. */ - Path path; + std::filesystem::path path; static StringSet uriSchemes() { @@ -79,7 +74,7 @@ struct UDSRemoteStore : virtual IndirectRootStore, virtual RemoteStore * owned managed by the client's user account, and the server makes * the indirect symlink. */ - void addIndirectRoot(const Path & path) override; + void addIndirectRoot(const std::filesystem::path & path) override; private: diff --git a/src/libstore/include/nix/store/worker-protocol-connection.hh b/src/libstore/include/nix/store/worker-protocol-connection.hh index 591e2cf09b5f..49de3fe67396 100644 --- a/src/libstore/include/nix/store/worker-protocol-connection.hh +++ b/src/libstore/include/nix/store/worker-protocol-connection.hh @@ -23,11 +23,6 @@ struct WorkerProto::BasicConnection */ WorkerProto::Version protoVersion; - /** - * The set of features that both sides support. - */ - FeatureSet features; - /** * Coercion to `WorkerProto::ReadConn`. This makes it easy to use the * factored out serve protocol serializers with a @@ -41,7 +36,6 @@ struct WorkerProto::BasicConnection return WorkerProto::ReadConn{ .from = from, .version = protoVersion, - .provenance = features.contains(WorkerProto::featureProvenance), }; } @@ -58,7 +52,6 @@ struct WorkerProto::BasicConnection return WorkerProto::WriteConn{ .to = to, .version = protoVersion, - .provenance = features.contains(WorkerProto::featureProvenance), }; } }; @@ -89,13 +82,11 @@ struct WorkerProto::BasicClientConnection : WorkerProto::BasicConnection * @param from Taken by reference to allow for various error * handling mechanisms. * - * @param localVersion Our version which is sent over. - * - * @param supportedFeatures The protocol features that we support. + * @param localVersion Our version (number + supported features) + * which is sent over. */ // FIXME: this should probably be a constructor. - static std::tuple handshake( - BufferedSink & to, Source & from, WorkerProto::Version localVersion, const FeatureSet & supportedFeatures); + static Version handshake(BufferedSink & to, Source & from, const Version & localVersion); /** * After calling handshake, must call this to exchange some basic @@ -128,10 +119,7 @@ struct WorkerProto::BasicClientConnection : WorkerProto::BasicConnection BuildResult getBuildDerivationResponse(const StoreDirConfig & store, bool * daemonException); void narFromPath( - const StoreDirConfig & store, - bool * daemonException, - const StorePath & path, - std::function fun); + const StoreDirConfig & store, bool * daemonException, const StorePath & path, fun receiveNar); }; struct WorkerProto::BasicServerConnection : WorkerProto::BasicConnection @@ -148,13 +136,11 @@ struct WorkerProto::BasicServerConnection : WorkerProto::BasicConnection * @param from Taken by reference to allow for various error * handling mechanisms. * - * @param localVersion Our version which is sent over. - * - * @param supportedFeatures The protocol features that we support. + * @param localVersion Our version (number + supported features) + * which is sent over. */ // FIXME: this should probably be a constructor. - static std::tuple handshake( - BufferedSink & to, Source & from, WorkerProto::Version localVersion, const FeatureSet & supportedFeatures); + static Version handshake(BufferedSink & to, Source & from, const Version & localVersion); /** * After calling handshake, must call this to exchange some basic diff --git a/src/libstore/include/nix/store/worker-protocol.hh b/src/libstore/include/nix/store/worker-protocol.hh index 8ae2d261a9e9..7c205016b7e5 100644 --- a/src/libstore/include/nix/store/worker-protocol.hh +++ b/src/libstore/include/nix/store/worker-protocol.hh @@ -1,7 +1,9 @@ #pragma once ///@file +#include #include +#include #include "nix/store/common-protocol.hh" @@ -10,13 +12,6 @@ namespace nix { #define WORKER_MAGIC_1 0x6e697863 #define WORKER_MAGIC_2 0x6478696f -/* Note: you generally shouldn't change the protocol version. Define a - new `WorkerProto::Feature` instead. */ -#define PROTOCOL_VERSION (1 << 8 | 38) -#define MINIMUM_PROTOCOL_VERSION (1 << 8 | 18) -#define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) -#define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) - #define STDERR_NEXT 0x6f6c6d67 #define STDERR_READ 0x64617461 // data needed from source #define STDERR_WRITE 0x64617416 // data for sink @@ -55,9 +50,68 @@ struct WorkerProto /** * Version type for the protocol. * - * @todo Convert to struct with separate major vs minor fields. + * This bundles the protocol version number with the negotiated + * feature set. The version number has a total ordering, but the + * full `Version` (number + features) only has a partial ordering, + * so there is no `operator<=>` on `Version` itself --- callers + * must compare `.number` explicitly. + */ + struct Version + { + struct Number + { + unsigned int major; + uint8_t minor; + + constexpr auto operator<=>(const Number &) const = default; + + /** + * Convert to wire format for protocol compatibility. + * Format: (major << 8) | minor + */ + constexpr unsigned int toWire() const + { + return (major << 8) | minor; + } + + /** + * Convert from wire format. + */ + static constexpr Number fromWire(unsigned int wire) + { + return { + .major = (wire & 0xff00) >> 8, + .minor = static_cast(wire & 0x00ff), + }; + }; + } number; + + using Feature = std::string; + using FeatureSet = std::set>; + + FeatureSet features = {}; + + bool operator==(const Version &) const = default; + + /** + * Partial ordering: v1 <= v2 iff v1.number <= v2.number AND + * v1.features is a subset of v2.features. Two versions with + * incomparable feature sets are unordered. + */ + std::partial_ordering operator<=>(const Version & other) const; + }; + + /** + * @note you generally shouldn't change the protocol version number. Define a new + * `WorkerProto::Version::Feature` instead. */ - using Version = unsigned int; + static const Version latest; + + static const Version minimum; + + static constexpr std::string_view featureQueryActiveBuilds = "queryActiveBuilds"; + static constexpr std::string_view featureProvenance = "provenance"; + static constexpr std::string_view featureVersionedAddToStoreMultiple = "versionedAddToStoreMultiple"; /** * A unidirectional read connection, to be used by the read half of the @@ -66,9 +120,8 @@ struct WorkerProto struct ReadConn { Source & from; - Version version; + const Version & version; bool shortStorePaths = false; - bool provenance = false; }; /** @@ -78,9 +131,8 @@ struct WorkerProto struct WriteConn { Sink & to; - Version version; + const Version & version; bool shortStorePaths = false; - bool provenance = false; }; /** @@ -137,21 +189,13 @@ struct WorkerProto { WorkerProto::Serialise::write(store, conn, t); } - - using Feature = std::string; - using FeatureSet = std::set>; - - static constexpr std::string_view featureQueryActiveBuilds{"queryActiveBuilds"}; - static constexpr std::string_view featureProvenance{"provenance"}; - - static const FeatureSet allFeatures; }; enum struct WorkerProto::Op : uint64_t { IsValidPath = 1, - HasSubstitutes = 3, - QueryPathHash = 4, // obsolete - QueryReferences = 5, // obsolete + // HasSubstitutes = 3, // removed + // QueryPathHash = 4, // removed + // QueryReferences = 5, // removed QueryReferrers = 6, AddToStore = 7, AddTextToStore = 8, // obsolete since 1.25, Nix 3.0. Use WorkerProto::Op::AddToStore @@ -168,8 +212,8 @@ enum struct WorkerProto::Op : uint64_t { QuerySubstitutablePathInfo = 21, QueryDerivationOutputs = 22, // obsolete QueryAllValidPaths = 23, - QueryFailedPaths = 24, - ClearFailedPaths = 25, + // QueryFailedPaths = 24, // removed + // ClearFailedPaths = 25, // removed QueryPathInfo = 26, // ImportPaths = 27, // removed QueryDerivationOutputNames = 28, // obsolete diff --git a/src/libstore/include/nix/store/worker-settings.hh b/src/libstore/include/nix/store/worker-settings.hh new file mode 100644 index 000000000000..6125dfd7cc26 --- /dev/null +++ b/src/libstore/include/nix/store/worker-settings.hh @@ -0,0 +1,390 @@ +#pragma once +///@file + +#include "nix/util/configuration.hh" +#include "nix/store/global-paths.hh" +#include "nix/store/store-reference.hh" + +namespace nix { + +template<> +std::vector BaseSetting>::parse(const std::string & str) const; +template<> +std::string BaseSetting>::to_string() const; + +struct MaxBuildJobsSetting : public BaseSetting +{ + MaxBuildJobsSetting( + Config * options, + unsigned int def, + const std::string & name, + const std::string & description, + const StringSet & aliases = {}) + : BaseSetting(def, true, name, description, aliases) + { + options->addSetting(this); + } + + unsigned int parse(const std::string & str) const override; +}; + +struct WorkerSettings : public virtual Config +{ +protected: + WorkerSettings() = default; + +public: + Setting keepGoing{ + this, false, "keep-going", "Whether to keep building derivations when another build fails."}; + + Setting tryFallback{ + this, + false, + "fallback", + R"( + If set to `true`, Nix falls back to building from source if a + binary substitute fails. This is equivalent to the `--fallback` + flag. The default is `false`. + )", + {"build-fallback"}}; + + Setting logLines{ + this, + 25, + "log-lines", + "The number of lines of the tail of " + "the log to show if a build fails."}; + + MaxBuildJobsSetting maxBuildJobs{ + this, + 1, + "max-jobs", + R"( + Maximum number of jobs that Nix tries to build locally in parallel. + + The special value `auto` causes Nix to use the number of CPUs in your system. + Use `0` to disable local builds and directly use the remote machines specified in [`builders`](#conf-builders). + This doesn't affect derivations that have [`preferLocalBuild = true`](@docroot@/language/advanced-attributes.md#adv-attr-preferLocalBuild), which are always built locally. + + > **Note** + > + > The number of CPU cores to use for each build job is independently determined by the [`cores`](#conf-cores) setting. + + + The setting can be overridden using the `--max-jobs` (`-j`) command line switch. + )", + {"build-max-jobs"}}; + + Setting maxSubstitutionJobs{ + this, + 16, + "max-substitution-jobs", + R"( + This option defines the maximum number of substitution jobs that Nix + tries to run in parallel. The default is `16`. The minimum value + one can choose is `1` and lower values are interpreted as `1`. + )", + {"substitution-max-jobs"}}; + + Setting maxSilentTime{ + this, + 0, + "max-silent-time", + R"( + This option defines the maximum number of seconds that a builder can + go without producing any data on standard output or standard error. + This is useful (for instance in an automated build system) to catch + builds that are stuck in an infinite loop, or to catch remote builds + that are hanging due to network problems. It can be overridden using + the `--max-silent-time` command line switch. + + The value `0` means that there is no timeout. This is also the + default. + )", + {"build-max-silent-time"}}; + + Setting buildTimeout{ + this, + 0, + "timeout", + R"( + This option defines the maximum number of seconds that a builder can + run. This is useful (for instance in an automated build system) to + catch builds that are stuck in an infinite loop but keep writing to + their standard output or standard error. It can be overridden using + the `--timeout` command line switch. + + The value `0` means that there is no timeout. This is also the + default. + )", + {"build-timeout"}}; + + Setting buildHook{ + this, + {"nix", "__build-remote"}, + "build-hook", + R"( + The path to the helper program that executes remote builds. + + Nix communicates with the build hook over `stdio` using a custom protocol to request builds that cannot be performed directly by the Nix daemon. + The default value is the internal Nix binary that implements remote building. + + > **Important** + > + > Change this setting only if you really know what you’re doing. + )"}; + + Setting builders{ + this, + "@" + (nixConfDir() / "machines").string(), + "builders", + R"( + A semicolon- or newline-separated list of build machines. + + In addition to the [usual ways of setting configuration options](@docroot@/command-ref/conf-file.md), the value can be read from a file by prefixing its absolute path with `@`. + + > **Example** + > + > This is the default setting: + > + > ``` + > builders = @/etc/nix/machines + > ``` + + Each machine specification consists of the following elements, separated by spaces. + Only the first element is required. + To leave a field at its default, set it to `-`. + + 1. The URI of the remote store in the format `ssh://[username@]hostname[:port]`. + + > **Example** + > + > `ssh://nix@mac` + + For backward compatibility, `ssh://` may be omitted. + The hostname may be an alias defined in `~/.ssh/config`. + + 2. A comma-separated list of [Nix system types](@docroot@/development/building.md#system-type). + If omitted, this defaults to the local platform type. + + > **Example** + > + > `aarch64-darwin` + + It is possible for a machine to support multiple platform types. + + > **Example** + > + > `i686-linux,x86_64-linux` + + 3. The SSH identity file to be used to log in to the remote machine. + If omitted, SSH uses its regular identities. + + > **Example** + > + > `/home/user/.ssh/id_mac` + + 4. The maximum number of builds that Nix executes in parallel on the machine. + Typically this should be equal to the number of CPU cores. + + 5. The “speed factor”, indicating the relative speed of the machine as a positive integer. + If there are multiple machines of the right type, Nix prefers the fastest, taking load into account. + + 6. A comma-separated list of supported [system features](#conf-system-features). + + A machine is only used to build a derivation if all the features in the derivation's [`requiredSystemFeatures`](@docroot@/language/advanced-attributes.html#adv-attr-requiredSystemFeatures) attribute are supported by that machine. + + 7. A comma-separated list of required [system features](#conf-system-features). + + A machine is only used to build a derivation if all of the machine’s required features appear in the derivation’s [`requiredSystemFeatures`](@docroot@/language/advanced-attributes.html#adv-attr-requiredSystemFeatures) attribute. + + 8. The (base64-encoded) public host key of the remote machine. + If omitted, SSH uses its regular `known_hosts` file. + + The value for this field can be obtained via `base64 -w0`. + + > **Example** + > + > Multiple builders specified on the command line: + > + > ```console + > --builders 'ssh://mac x86_64-darwin ; ssh://beastie x86_64-freebsd' + > ``` + + > **Example** + > + > This specifies several machines that can perform `i686-linux` builds: + > + > ``` + > nix@scratchy.labs.cs.uu.nl i686-linux /home/nix/.ssh/id_scratchy 8 1 kvm + > nix@itchy.labs.cs.uu.nl i686-linux /home/nix/.ssh/id_scratchy 8 2 + > nix@poochie.labs.cs.uu.nl i686-linux /home/nix/.ssh/id_scratchy 1 2 kvm benchmark + > ``` + > + > However, `poochie` only builds derivations that have the attribute + > + > ```nix + > requiredSystemFeatures = [ "benchmark" ]; + > ``` + > + > or + > + > ```nix + > requiredSystemFeatures = [ "benchmark" "kvm" ]; + > ``` + > + > `itchy` cannot do builds that require `kvm`, but `scratchy` does support such builds. + > For regular builds, `itchy` is preferred over `scratchy` because it has a higher speed factor. + + For Nix to use substituters, the calling user must be in the [`trusted-users`](#conf-trusted-users) list. + + > **Note** + > + > A build machine must be accessible via SSH and have Nix installed. + > `nix` must be available in `$PATH` for the user connecting over SSH. + + > **Warning** + > + > If you are building via the Nix daemon (default), the Nix daemon user account on the local machine (that is, `root`) requires access to a user account on the remote machine (not necessarily `root`). + > + > If you can’t or don’t want to configure `root` to be able to access the remote machine, set [`store`](#conf-store) to any [local store](@docroot@/store/types/local-store.html), e.g. by passing `--store /tmp` to the command on the local machine. + + To build only on remote machines and disable local builds, set [`max-jobs`](#conf-max-jobs) to 0. + + If you want the remote machines to use substituters, set [`builders-use-substitutes`](#conf-builders-use-substitutes) to `true`. + )", + {}, + false}; + + Setting alwaysAllowSubstitutes{ + this, + false, + "always-allow-substitutes", + R"( + If set to `true`, Nix ignores the [`allowSubstitutes`](@docroot@/language/advanced-attributes.md) attribute in derivations and always attempt to use [available substituters](#conf-substituters). + )"}; + + Setting buildersUseSubstitutes{ + this, + false, + "builders-use-substitutes", + R"( + If set to `true`, Nix instructs [remote build machines](#conf-builders) to use their own [`substituters`](#conf-substituters) if available. + + It means that remote build hosts fetch as many dependencies as possible from their own substituters (e.g, from `cache.nixos.org`) instead of waiting for the local machine to upload them all. + This can drastically reduce build times if the network connection between the local machine and the remote build host is slow. + )"}; + + Setting useSubstitutes{ + this, + true, + "substitute", + R"( + If set to `true` (default), Nix uses binary substitutes if + available. This option can be disabled to force building from + source. + )", + {"build-use-substitutes"}}; + + Setting> substituters{ + this, + std::vector{StoreReference::parse("https://cache.nixos.org/")}, + "substituters", + R"( + A list of [URLs of Nix stores](@docroot@/store/types/index.md#store-url-format) to be used as substituters, separated by whitespace. + A substituter is an additional [store](@docroot@/glossary.md#gloss-store) from which Nix can obtain [store objects](@docroot@/store/store-object.md) instead of building them. + + Substituters are tried based on their priority value, which each substituter can set independently. + Lower value means higher priority. + The default is `https://cache.nixos.org`, which has a priority of 40. + + At least one of the following conditions must be met for Nix to use a substituter: + + - The substituter is in the [`trusted-substituters`](#conf-trusted-substituters) list + - The user calling Nix is in the [`trusted-users`](#conf-trusted-users) list + + In addition, each store path should be trusted as described in [`trusted-public-keys`](#conf-trusted-public-keys) + )", + {"binary-caches"}}; + + Setting maxLogSize{ + this, + 0, + "max-build-log-size", + R"( + This option defines the maximum number of bytes that a builder can + write to its stdout/stderr. If the builder exceeds this limit, it's + killed. A value of `0` (the default) means that there is no limit. + )", + {"build-max-log-size"}}; + + Setting pollInterval{this, 5, "build-poll-interval", "How often (in seconds) to poll for locks."}; + + Setting postBuildHook{ + this, + "", + "post-build-hook", + R"( + Optional. The path to a program to execute after each build. + + This option is only settable in the global `nix.conf`, or on the + command line by trusted users. + + When using the nix-daemon, the daemon executes the hook as `root`. + If the nix-daemon is not involved, the hook runs as the user + executing the nix-build. + + - The hook executes after an evaluation-time build. + + - The hook does not execute on substituted paths. + + - The hook's output always goes to the user's terminal. + + - If the hook fails, the build succeeds but no further builds + execute. + + - The hook executes synchronously, and blocks other builds from + progressing while it runs. + + The program executes with no arguments. The program's environment + contains the following environment variables: + + - `DRV_PATH` + The derivation for the built paths. + + Example: + `/nix/store/5nihn1a7pa8b25l9zafqaqibznlvvp3f-bash-4.4-p23.drv` + + - `OUT_PATHS` + Output paths of the built derivation, separated by a space + character. + + Example: + `/nix/store/l88brggg9hpy96ijds34dlq4n8fan63g-bash-4.4-p23-dev + /nix/store/vch71bhyi5akr5zs40k8h2wqxx69j80l-bash-4.4-p23-doc + /nix/store/c5cxjywi66iwn9dcx5yvwjkvl559ay6p-bash-4.4-p23-info + /nix/store/scz72lskj03ihkcn42ias5mlp4i4gr1k-bash-4.4-p23-man + /nix/store/a724znygmd1cac856j3gfsyvih3lw07j-bash-4.4-p23`. + )"}; + + Setting hostName{ + this, + "", + "host-name", + R"( + The name of this host for recording build provenance. If unset, the Unix host name is used. + )"}; + + std::optional getHostName(); + + JSONSetting buildProvenanceTags{ + this, + {}, + "build-provenance-tags", + R"( + Arbitrary name/value pairs that are recorded in the build provenance of store paths built by this machine. + This can be used to tag builds with metadata such as the CI job URL, build cluster name, etc. + )"}; +}; + +} // namespace nix diff --git a/src/libstore/indirect-root-store.cc b/src/libstore/indirect-root-store.cc index b882b2568a48..7384456e286d 100644 --- a/src/libstore/indirect-root-store.cc +++ b/src/libstore/indirect-root-store.cc @@ -2,28 +2,28 @@ namespace nix { -void IndirectRootStore::makeSymlink(const Path & link, const Path & target) +void IndirectRootStore::makeSymlink(const std::filesystem::path & link, const std::filesystem::path & target) { /* Create directories up to `gcRoot'. */ - createDirs(dirOf(link)); + createDirs(link.parent_path()); /* Create the new symlink. */ - Path tempLink = fmt("%1%.tmp-%2%-%3%", link, getpid(), rand()); + auto tempLink = std::filesystem::path(link) += fmt(".tmp-%1%-%2%", getpid(), rand()); createSymlink(target, tempLink); /* Atomically replace the old one. */ std::filesystem::rename(tempLink, link); } -Path IndirectRootStore::addPermRoot(const StorePath & storePath, const Path & _gcRoot) +std::filesystem::path IndirectRootStore::addPermRoot(const StorePath & storePath, const std::filesystem::path & _gcRoot) { - Path gcRoot(canonPath(_gcRoot)); + auto gcRoot = canonPath(_gcRoot); - if (isInStore(gcRoot)) + if (isInStore(gcRoot.string())) throw Error( "creating a garbage collector root (%1%) in the Nix store is forbidden " "(are you running nix-build inside the store?)", - gcRoot); + PathFmt(gcRoot)); /* Register this root with the garbage collector, if it's running. This should be superfluous since the caller should @@ -33,8 +33,8 @@ Path IndirectRootStore::addPermRoot(const StorePath & storePath, const Path & _g /* Don't clobber the link if it already exists and doesn't point to the Nix store. */ - if (pathExists(gcRoot) && (!std::filesystem::is_symlink(gcRoot) || !isInStore(readLink(gcRoot)))) - throw Error("cannot create symlink '%1%'; already exists", gcRoot); + if (pathExists(gcRoot) && (!std::filesystem::is_symlink(gcRoot) || !isInStore(readLink(gcRoot).string()))) + throw Error("cannot create symlink %1%; already exists", PathFmt(gcRoot)); makeSymlink(gcRoot, printStorePath(storePath)); addIndirectRoot(gcRoot); diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 3b466c9bb8b0..b1781be457ff 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -3,6 +3,7 @@ #include "nix/util/archive.hh" #include "nix/util/pool.hh" #include "nix/store/remote-store.hh" +#include "nix/store/common-protocol.hh" #include "nix/store/serve-protocol.hh" #include "nix/store/serve-protocol-connection.hh" #include "nix/store/serve-protocol-impl.hh" @@ -17,9 +18,9 @@ namespace nix { -LegacySSHStoreConfig::LegacySSHStoreConfig(std::string_view scheme, std::string_view authority, const Params & params) +LegacySSHStoreConfig::LegacySSHStoreConfig(const ParsedURL::Authority & authority, const Params & params) : StoreConfig(params) - , CommonSSHStoreConfig(scheme, ParsedURL::Authority::parse(authority), params) + , CommonSSHStoreConfig(authority, params) { } @@ -61,7 +62,7 @@ ref LegacySSHStore::openConnection() command.push_back("--store"); command.push_back(config->remoteStore.get()); } - conn->sshConn = master.startCommand(std::move(command), std::list{config->extraSshArgs}); + conn->sshConn = master.startCommand(toOsStrings(std::move(command)), toOsStrings(std::list{config->extraSshArgs})); if (config->connPipeSize) { conn->sshConn->trySetBufferSize(*config->connPipeSize); } @@ -72,7 +73,7 @@ ref LegacySSHStore::openConnection() TeeSource tee(conn->from, saved); try { conn->remoteVersion = - ServeProto::BasicClientConnection::handshake(conn->to, tee, SERVE_PROTOCOL_VERSION, config->authority.host); + ServeProto::BasicClientConnection::handshake(conn->to, tee, ServeProto::latest, config->authority.host); } catch (SerialisationError & e) { // in.close(): Don't let the remote block on us not writing. conn->sshConn->in.close(); @@ -153,7 +154,9 @@ void LegacySSHStore::addToStore(const ValidPathInfo & info, Source & source, Rep << (info.deriver ? printStorePath(*info.deriver) : "") << info.narHash.to_string(HashFormat::Base16, false); ServeProto::write(*this, *conn, info.references); - conn->to << info.registrationTime << info.narSize << info.ultimate << info.sigs << renderContentAddress(info.ca); + conn->to << info.registrationTime << info.narSize << info.ultimate; + ServeProto::write(*this, *conn, info.sigs); + conn->to << renderContentAddress(info.ca); try { copyNAR(source, conn->to); } catch (...) { @@ -171,18 +174,18 @@ void LegacySSHStore::narFromPath(const StorePath & path, Sink & sink) narFromPath(path, [&](auto & source) { copyNAR(source, sink); }); } -void LegacySSHStore::narFromPath(const StorePath & path, std::function fun) +void LegacySSHStore::narFromPath(const StorePath & path, fun receiveNar) { auto conn(connections->get()); - conn->narFromPath(*this, path, fun); + conn->narFromPath(*this, path, receiveNar); } static ServeProto::BuildOptions buildSettings() { return { - .maxSilentTime = settings.maxSilentTime, - .buildTimeout = settings.buildTimeout, - .maxLogSize = settings.maxLogSize, + .maxSilentTime = settings.getWorkerSettings().maxSilentTime, + .buildTimeout = settings.getWorkerSettings().buildTimeout, + .maxLogSize = settings.getWorkerSettings().maxLogSize, .nrRepeats = 0, // buildRepeat hasn't worked for ages anyway .enforceDeterminism = 0, .keepFailed = settings.keepFailed, @@ -198,7 +201,7 @@ BuildResult LegacySSHStore::buildDerivation(const StorePath & drvPath, const Bas return conn->getBuildDerivationResponse(*this); } -std::function LegacySSHStore::buildDerivationAsync( +fun LegacySSHStore::buildDerivationAsync( const StorePath & drvPath, const BasicDerivation & drv, const ServeProto::BuildOptions & options) { // Until we have C++23 std::move_only_function @@ -241,13 +244,11 @@ void LegacySSHStore::buildPaths( conn->to.flush(); - auto status = readInt(conn->from); - if (!BuildResult::Success::statusIs(status)) { - BuildResult::Failure failure{ - .status = (BuildResult::Failure::Status) status, - }; - conn->from >> failure.errorMsg; - throw Error(failure.status, std::move(failure.errorMsg)); + auto status = CommonProto::Serialise::read(*this, {conn->from}); + if (auto * failure = std::get_if(&status)) { + std::string errorMsg; + conn->from >> errorMsg; + throw BuildError(*failure, std::move(errorMsg)); } } @@ -289,7 +290,7 @@ void LegacySSHStore::connect() unsigned int LegacySSHStore::getProtocol() { auto conn(connections->get()); - return conn->remoteVersion; + return conn->remoteVersion.toWire(); } pid_t LegacySSHStore::getConnectionPid() diff --git a/src/libstore/linux/include/nix/store/personality.hh b/src/libstore/linux/include/nix/store/personality.hh index 01bf2bf331ef..dd9c4c2dc5bb 100644 --- a/src/libstore/linux/include/nix/store/personality.hh +++ b/src/libstore/linux/include/nix/store/personality.hh @@ -5,6 +5,12 @@ namespace nix::linux { -void setPersonality(std::string_view system); +struct PersonalityArgs +{ + std::string_view system; + bool impersonateLinux26; +}; -} +void setPersonality(PersonalityArgs args); + +} // namespace nix::linux diff --git a/src/libstore/linux/personality.cc b/src/libstore/linux/personality.cc index d268706b2386..f669c9fa127a 100644 --- a/src/libstore/linux/personality.cc +++ b/src/libstore/linux/personality.cc @@ -1,5 +1,6 @@ #include "nix/store/personality.hh" -#include "nix/store/globals.hh" +#include "nix/store/config.hh" +#include "nix/util/error.hh" #include #include @@ -8,23 +9,23 @@ namespace nix::linux { -void setPersonality(std::string_view system) +void setPersonality(PersonalityArgs args) { /* Change the personality to 32-bit if we're doing an i686-linux build on an x86_64-linux machine. */ struct utsname utsbuf; uname(&utsbuf); - if ((system == "i686-linux" + if ((args.system == "i686-linux" && (std::string_view(NIX_LOCAL_SYSTEM) == "x86_64-linux" || (!strcmp(utsbuf.sysname, "Linux") && !strcmp(utsbuf.machine, "x86_64")))) - || system == "armv7l-linux" || system == "armv6l-linux" || system == "armv5tel-linux") { + || args.system == "armv7l-linux" || args.system == "armv6l-linux" || args.system == "armv5tel-linux") { if (personality(PER_LINUX32) == -1) throw SysError("cannot set 32-bit personality"); } /* Impersonate a Linux 2.6 machine to get some determinism in builds that depend on the kernel version. */ - if ((system == "i686-linux" || system == "x86_64-linux") && settings.impersonateLinux26) { + if ((args.system == "i686-linux" || args.system == "x86_64-linux") && args.impersonateLinux26) { int cur = personality(0xffffffff); if (cur != -1) personality(cur | 0x0020000 /* == UNAME26 */); diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc index 63730a01bd79..8dba477315f3 100644 --- a/src/libstore/local-binary-cache-store.cc +++ b/src/libstore/local-binary-cache-store.cc @@ -8,8 +8,25 @@ namespace nix { +static std::filesystem::path checkBinaryCachePath(const std::filesystem::path & root, const std::string & path) +{ + auto p = std::filesystem::path(requireCString(path)); + if (p.empty()) + throw Error("local binary cache path must not be empty"); + + if (p.is_absolute()) + throw Error("local binary cache path '%s' must not be absolute", path); + + for (const auto & segment : p) { + if (segment.native() == OS_STR("..") || segment.native() == OS_STR(".")) + throw Error("local binary cache path '%s' must not contain '..' or '.' segments", path); + } + + return root / p.relative_path(); +} + LocalBinaryCacheStoreConfig::LocalBinaryCacheStoreConfig( - std::string_view scheme, PathView binaryCacheDir, const StoreReference::Params & params) + const std::filesystem::path & binaryCacheDir, const StoreReference::Params & params) : Store::Config{params} , BinaryCacheStoreConfig{params} , binaryCacheDir(binaryCacheDir) @@ -29,7 +46,7 @@ StoreReference LocalBinaryCacheStoreConfig::getReference() const .variant = StoreReference::Specified{ .scheme = "file", - .authority = binaryCacheDir, + .authority = encodeUrlPath(pathToUrlPath(binaryCacheDir)), }, }; } @@ -56,11 +73,13 @@ struct LocalBinaryCacheStore : virtual BinaryCacheStore void upsertFile( const std::string & path, RestartableSource & source, const std::string & mimeType, uint64_t sizeHint) override { - auto path2 = config->binaryCacheDir + "/" + path; + auto path2 = checkBinaryCachePath(config->binaryCacheDir, path); static std::atomic counter{0}; - Path tmp = fmt("%s.tmp.%d.%d", path2, getpid(), ++counter); + createDirs(path2.parent_path()); + auto tmp = path2; + tmp += fmt(".tmp.%d.%d", getpid(), ++counter); AutoDelete del(tmp, false); - writeFile(tmp, source); + writeFile(tmp, source); /* TODO: Don't follow symlinks? */ std::filesystem::rename(tmp, path2); del.cancel(); } @@ -68,9 +87,10 @@ struct LocalBinaryCacheStore : virtual BinaryCacheStore void getFile(const std::string & path, Sink & sink) override { try { - readFile(config->binaryCacheDir + "/" + path, sink); - } catch (SysError & e) { - if (e.errNo == ENOENT) + /* TODO: Don't follow symlinks? */ + readFile(checkBinaryCachePath(config->binaryCacheDir, path), sink); + } catch (SystemError & e) { + if (e.is(std::errc::no_such_file_or_directory)) throw NoSuchBinaryCacheFile("file '%s' does not exist in binary cache", path); throw; } @@ -99,17 +119,17 @@ struct LocalBinaryCacheStore : virtual BinaryCacheStore void LocalBinaryCacheStore::init() { - createDirs(config->binaryCacheDir + "/nar"); - createDirs(config->binaryCacheDir + "/" + realisationsPrefix); + createDirs(config->binaryCacheDir / "nar"); + createDirs(config->binaryCacheDir / realisationsPrefix); if (config->writeDebugInfo) - createDirs(config->binaryCacheDir + "/debuginfo"); - createDirs(config->binaryCacheDir + "/log"); + createDirs(config->binaryCacheDir / "debuginfo"); + createDirs(config->binaryCacheDir / "log"); BinaryCacheStore::init(); } bool LocalBinaryCacheStore::fileExists(const std::string & path) { - return pathExists(config->binaryCacheDir + "/" + path); + return pathExists(checkBinaryCachePath(config->binaryCacheDir, path)); } StringSet LocalBinaryCacheStoreConfig::uriSchemes() diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc index b8f8c6dbdea6..d3563329b0e3 100644 --- a/src/libstore/local-fs-store.cc +++ b/src/libstore/local-fs-store.cc @@ -8,17 +8,17 @@ namespace nix { -Path LocalFSStoreConfig::getDefaultStateDir() +std::filesystem::path LocalFSStoreConfig::getDefaultStateDir() { return settings.nixStateDir; } -Path LocalFSStoreConfig::getDefaultLogDir() +std::filesystem::path LocalFSStoreConfig::getDefaultLogDir() { - return settings.nixLogDir; + return settings.getLogFileSettings().nixLogDir; } -LocalFSStoreConfig::LocalFSStoreConfig(PathView rootDir, const Params & params) +LocalFSStoreConfig::LocalFSStoreConfig(const std::filesystem::path & rootDir, const Params & params) : StoreConfig(params) /* Default `?root` from `rootDir` if non set * NOTE: We would like to just do rootDir.set(...), which would take care of @@ -29,8 +29,7 @@ LocalFSStoreConfig::LocalFSStoreConfig(PathView rootDir, const Params & params) * manually repeat the same normalization logic. */ , rootDir{makeRootDirSetting( - *this, - !rootDir.empty() && params.count("root") == 0 ? std::optional{canonPath(rootDir)} : std::nullopt)} + *this, !rootDir.empty() && params.count("root") == 0 ? std::optional{canonPath(rootDir)} : std::nullopt)} { } @@ -76,7 +75,7 @@ struct LocalStoreAccessor : PosixSourceAccessor return PosixSourceAccessor::readDirectory(path); } - void readFile(const CanonPath & path, Sink & sink, std::function sizeCallback) override + void readFile(const CanonPath & path, Sink & sink, fun sizeCallback) override { requireStoreObject(path); return PosixSourceAccessor::readFile(path, sink, sizeCallback); @@ -112,7 +111,7 @@ std::shared_ptr LocalFSStore::getFSAccessor(const StorePath & pa return std::make_shared(std::move(absPath)); } -const std::string LocalFSStore::drvsLogDir = "drvs"; +const std::filesystem::path LocalFSStore::drvsLogDir = "drvs"; std::optional LocalFSStore::getBuildLogExact(const StorePath & path) { @@ -120,10 +119,10 @@ std::optional LocalFSStore::getBuildLogExact(const StorePath & path for (int j = 0; j < 2; j++) { - Path logPath = - j == 0 ? fmt("%s/%s/%s/%s", config.logDir.get(), drvsLogDir, baseName.substr(0, 2), baseName.substr(2)) - : fmt("%s/%s/%s", config.logDir.get(), drvsLogDir, baseName); - Path logBz2Path = logPath + ".bz2"; + auto logPath = config.logDir.get() + / (j == 0 ? drvsLogDir / baseName.substr(0, 2) / baseName.substr(2) : drvsLogDir / baseName); + auto logBz2Path = logPath; + logBz2Path += ".bz2"; if (pathExists(logPath)) return readFile(logPath); diff --git a/src/libstore/local-gc.cc b/src/libstore/local-gc.cc new file mode 100644 index 000000000000..e3c74f35752b --- /dev/null +++ b/src/libstore/local-gc.cc @@ -0,0 +1,165 @@ +#include "nix/store/gc-store.hh" +#include "nix/store/store-dir-config.hh" +#include "nix/util/file-system.hh" +#include "nix/util/signals.hh" +#include "nix/util/types.hh" +#include "nix/store/local-gc.hh" +#include +#include + +#if !defined(__linux__) +// For shelling out to lsof +# include "store-config-private.hh" +# include "nix/util/environment-variables.hh" +# include "nix/util/processes.hh" +#endif + +namespace nix { + +/** + * Key is a mere string because cannot has path with macOS's libc++ + */ +typedef boost::unordered_flat_map< + std::string, + boost::unordered_flat_set>, + StringViewHash, + std::equal_to<>> + UncheckedRoots; + +static void readProcLink(const std::filesystem::path & file, UncheckedRoots & roots) +{ + std::filesystem::path buf; + try { + buf = std::filesystem::read_symlink(file); + } catch (std::filesystem::filesystem_error & e) { + if (e.code() == std::errc::no_such_file_or_directory || e.code() == std::errc::permission_denied + || e.code() == std::errc::no_such_process) + return; + throw SystemError(e.code(), "reading symlink '%s'", PathFmt(file)); + } + if (buf.is_absolute()) + roots[buf.string()].emplace(file.string()); +} + +static std::string quoteRegexChars(const std::string & raw) +{ + static auto specialRegex = boost::regex(R"([.^$\\*+?()\[\]{}|])"); + return boost::regex_replace(raw, specialRegex, R"(\\$&)"); +} + +#ifdef __linux__ +static void readFileRoots(const std::filesystem::path & path, UncheckedRoots & roots) +{ + try { + roots[readFile(path)].emplace(path.string()); + } catch (SystemError & e) { + if (!e.is(std::errc::no_such_file_or_directory) && !e.is(std::errc::permission_denied)) + throw; + } +} +#endif + +Roots findRuntimeRootsUnchecked(const StoreDirConfig & config) +{ + UncheckedRoots unchecked; + + auto procDir = AutoCloseDir{opendir("/proc")}; + if (procDir) { + struct dirent * ent; + static const auto digitsRegex = boost::regex(R"(^\d+$)"); + static const auto mapRegex = boost::regex(R"(^\s*\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+(/\S+)\s*$)"); + auto storePathRegex = boost::regex(quoteRegexChars(config.storeDir) + R"(/[0-9a-z]+[0-9a-zA-Z\+\-\._\?=]*)"); + while (errno = 0, ent = readdir(procDir.get())) { + checkInterrupt(); + if (boost::regex_match(ent->d_name, digitsRegex)) { + try { + readProcLink(fmt("/proc/%s/exe", ent->d_name), unchecked); + readProcLink(fmt("/proc/%s/cwd", ent->d_name), unchecked); + + auto fdStr = fmt("/proc/%s/fd", ent->d_name); + auto fdDir = AutoCloseDir(opendir(fdStr.c_str())); + if (!fdDir) { + if (errno == ENOENT || errno == EACCES) + continue; + throw SysError("opening %1%", fdStr); + } + struct dirent * fd_ent; + while (errno = 0, fd_ent = readdir(fdDir.get())) { + if (fd_ent->d_name[0] != '.') + readProcLink(fmt("%s/%s", fdStr, fd_ent->d_name), unchecked); + } + if (errno) { + if (errno == ESRCH) + continue; + throw SysError("iterating /proc/%1%/fd", ent->d_name); + } + fdDir.reset(); + + std::filesystem::path mapFile = fmt("/proc/%s/maps", ent->d_name); + auto mapLines = tokenizeString>(readFile(mapFile.string()), "\n"); + for (const auto & line : mapLines) { + auto match = boost::smatch{}; + if (boost::regex_match(line, match, mapRegex)) + unchecked[match[1]].emplace(mapFile.string()); + } + + auto envFile = fmt("/proc/%s/environ", ent->d_name); + auto envString = readFile(envFile); + auto env_end = boost::sregex_iterator{}; + for (auto i = boost::sregex_iterator{envString.begin(), envString.end(), storePathRegex}; + i != env_end; + ++i) + unchecked[i->str()].emplace(envFile); + } catch (SystemError & e) { + if (errno == ENOENT || errno == EACCES || errno == ESRCH) + continue; + throw; + } + } + } + if (errno) + throw SysError("iterating /proc"); + } + +#if !defined(__linux__) + // lsof is really slow on OS X. This actually causes the gc-concurrent.sh test to fail. + // See: https://github.com/NixOS/nix/issues/3011 + // Because of this we disable lsof when running the tests. + if (getEnv("_NIX_TEST_NO_LSOF") != "1") { + try { + boost::regex lsofRegex(R"(^n(/.*)$)"); + auto lsofLines = tokenizeString>( + runProgram(LSOF, true, {OS_STR("-n"), OS_STR("-w"), OS_STR("-F"), OS_STR("n")}), "\n"); + for (const auto & line : lsofLines) { + boost::smatch match; + if (boost::regex_match(line, match, lsofRegex)) + unchecked[match[1].str()].emplace("{lsof}"); + } + } catch (ExecError & e) { + /* lsof not installed, lsof failed */ + } + } +#endif + +#ifdef __linux__ + readFileRoots("/proc/sys/kernel/modprobe", unchecked); + readFileRoots("/proc/sys/kernel/fbsplash", unchecked); + readFileRoots("/proc/sys/kernel/poweroff_cmd", unchecked); +#endif + + Roots roots; + + for (auto & [target, links] : unchecked) { + if (!config.isInStore(target)) + continue; + try { + auto path = config.toStorePath(target).first; + roots[path].insert(links.begin(), links.end()); + } catch (BadStorePath &) { + } + } + + return roots; +} + +} // namespace nix diff --git a/src/libstore/local-overlay-store.cc b/src/libstore/local-overlay-store.cc index c8aa1d1a2b62..705e9ddf71db 100644 --- a/src/libstore/local-overlay-store.cc +++ b/src/libstore/local-overlay-store.cc @@ -2,6 +2,7 @@ #include "nix/store/local-overlay-store.hh" #include "nix/util/callback.hh" +#include "nix/util/os-string.hh" #include "nix/store/realisation.hh" #include "nix/util/processes.hh" #include "nix/util/url.hh" @@ -33,9 +34,9 @@ StoreReference LocalOverlayStoreConfig::getReference() const }; } -Path LocalOverlayStoreConfig::toUpperPath(const StorePath & path) const +std::filesystem::path LocalOverlayStoreConfig::toUpperPath(const StorePath & path) const { - return upperLayer + "/" + path.to_string(); + return upperLayer.get() / path.to_string(); } LocalOverlayStore::LocalOverlayStore(ref config) @@ -43,13 +44,13 @@ LocalOverlayStore::LocalOverlayStore(ref config) , LocalFSStore{*config} , LocalStore{static_cast>(config)} , config{config} - , lowerStore(openStore(percentDecode(config->lowerStoreUri.get())).dynamic_pointer_cast()) + , lowerStore(openStore(config->lowerStoreUri.get()).dynamic_pointer_cast()) { if (config->checkMount.get()) { std::smatch match; std::string mountInfo; auto mounts = readFile(std::filesystem::path{"/proc/self/mounts"}); - auto regex = std::regex(R"((^|\n)overlay )" + config->realStoreDir.get() + R"( .*(\n|$))"); + auto regex = std::regex(R"((^|\n)overlay )" + config->realStoreDir.get().string() + R"( .*(\n|$))"); // Mount points can be stacked, so there might be multiple matching entries. // Loop until the last match, which will be the current state of the mount point. @@ -58,16 +59,16 @@ LocalOverlayStore::LocalOverlayStore(ref config) mounts = match.suffix(); } - auto checkOption = [&](std::string option, std::string value) { - return std::regex_search(mountInfo, std::regex("\\b" + option + "=" + value + "( |,)")); + auto checkOption = [&](std::string_view option, const std::filesystem::path & value) { + return std::regex_search(mountInfo, std::regex("\\b" + option + "=" + value.string() + "( |,)")); }; auto expectedLowerDir = lowerStore->config.realStoreDir.get(); - if (!checkOption("lowerdir", expectedLowerDir) || !checkOption("upperdir", config->upperLayer)) { - debug("expected lowerdir: %s", expectedLowerDir); - debug("expected upperdir: %s", config->upperLayer); + if (!checkOption("lowerdir", expectedLowerDir) || !checkOption("upperdir", config->upperLayer.get())) { + debug("expected lowerdir: %s", PathFmt(lowerStore->config.realStoreDir.get())); + debug("expected upperdir: %s", PathFmt(config->upperLayer.get())); debug("actual mount: %s", mountInfo); - throw Error("overlay filesystem '%s' mounted incorrectly", config->realStoreDir.get()); + throw Error("overlay filesystem %s mounted incorrectly", PathFmt(config->realStoreDir.get())); } } } @@ -204,19 +205,18 @@ void LocalOverlayStore::collectGarbage(const GCOptions & options, GCResults & re remountIfNecessary(); } -void LocalOverlayStore::deleteStorePath(const Path & path, uint64_t & bytesFreed) +void LocalOverlayStore::deleteStorePath(const std::filesystem::path & path, uint64_t & bytesFreed, bool isKnownPath) { - auto mergedDir = config->realStoreDir.get() + "/"; - if (path.substr(0, mergedDir.length()) != mergedDir) { - warn("local-overlay: unexpected gc path '%s' ", path); + if (path.parent_path() != config->realStoreDir.get()) { + warn("local-overlay: unexpected gc path %s", PathFmt(path)); return; } - StorePath storePath = {path.substr(mergedDir.length())}; + StorePath storePath = {path.filename().string()}; auto upperPath = config->toUpperPath(storePath); if (pathExists(upperPath)) { - debug("upper exists: %s", path); + debug("upper exists: %s", PathFmt(path)); if (lowerStore->isValidPath(storePath)) { debug("lower exists: %s", storePath.to_string()); // Path also exists in lower store. @@ -226,7 +226,7 @@ void LocalOverlayStore::deleteStorePath(const Path & path, uint64_t & bytesFreed } else { // Path does not exist in lower store. // So we can delete via overlayfs and not need to remount. - LocalStore::deleteStorePath(path, bytesFreed); + LocalStore::deleteStorePath(path, bytesFreed, isKnownPath); } } } @@ -246,7 +246,7 @@ void LocalOverlayStore::optimiseStore() if (lowerStore->isValidPath(path)) { uint64_t bytesFreed = 0; // Deduplicate store path - deleteStorePath(toRealPath(path), bytesFreed); + deleteStorePath(toRealPath(path), bytesFreed, true); } done++; act.progress(done, paths.size()); @@ -260,7 +260,7 @@ LocalStore::VerificationResult LocalOverlayStore::verifyAllValidPaths(RepairFlag StorePathSet done; auto existsInStoreDir = [&](const StorePath & storePath) { - return pathExists(config->realStoreDir.get() + "/" + storePath.to_string()); + return pathExists((config->realStoreDir.get() / storePath.to_string()).string()); }; bool errors = false; @@ -281,9 +281,9 @@ void LocalOverlayStore::remountIfNecessary() return; if (config->remountHook.get().empty()) { - warn("'%s' needs remounting, set remount-hook to do this automatically", config->realStoreDir.get()); + warn("%s needs remounting, set remount-hook to do this automatically", PathFmt(config->realStoreDir.get())); } else { - runProgram(config->remountHook, false, {config->realStoreDir}); + runProgram(config->remountHook.get(), false, {config->realStoreDir.get().native()}); } _remountRequired = false; diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index b1cf10185dbe..0f9c7cb87a28 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -6,7 +6,6 @@ #include "nix/store/worker-protocol.hh" #include "nix/store/derivations.hh" #include "nix/store/realisation.hh" -#include "nix/store/nar-info.hh" #include "nix/store/references.hh" #include "nix/util/callback.hh" #include "nix/util/topo-sort.hh" @@ -16,13 +15,10 @@ #include "nix/store/posix-fs-canonicalise.hh" #include "nix/util/posix-source-accessor.hh" #include "nix/store/keys.hh" -#include "nix/util/url.hh" #include "nix/util/users.hh" -#include "nix/store/store-open.hh" #include "nix/store/store-registration.hh" #include "nix/util/provenance.hh" -#include #include #include @@ -34,7 +30,6 @@ #include #include #include -#include #include #include @@ -62,9 +57,9 @@ namespace nix { -LocalStoreConfig::LocalStoreConfig(std::string_view scheme, std::string_view authority, const Params & params) +LocalStoreConfig::LocalStoreConfig(const std::filesystem::path & path, const Params & params) : StoreConfig(params) - , LocalFSStoreConfig(authority, params) + , LocalFSStoreConfig(path, params) { } @@ -75,11 +70,18 @@ std::string LocalStoreConfig::doc() ; } -Path LocalBuildStoreConfig::getBuildDir() const +const LocalSettings & LocalBuildStoreConfig::getLocalSettings() const + +{ + return settings.getLocalSettings(); +} + +std::filesystem::path LocalBuildStoreConfig::getBuildDir() const { - return settings.buildDir.get().has_value() ? *settings.buildDir.get() - : buildDir.get().has_value() ? *buildDir.get() - : stateDir.get() + "/builds"; + auto & bd = getLocalSettings().buildDir.get(); + return bd.has_value() ? *bd + : buildDir.get().has_value() ? *buildDir.get() + : AbsolutePath{stateDir.get() / "builds"}; } ref LocalStore::Config::openStore() const @@ -111,8 +113,6 @@ struct LocalStore::State::Stmts SQLiteStmt QueryAllRealisedOutputs; SQLiteStmt QueryPathFromHashPart; SQLiteStmt QueryValidPaths; - SQLiteStmt QueryRealisationReferences; - SQLiteStmt AddRealisationReference; }; LocalStore::LocalStore(ref config) @@ -120,13 +120,13 @@ LocalStore::LocalStore(ref config) , LocalFSStore{*config} , config{config} , _state(make_ref>()) - , dbDir(config->stateDir + "/db") - , linksDir(config->realStoreDir + "/.links") - , reservedPath(dbDir + "/reserved") - , schemaPath(dbDir + "/schema") - , tempRootsDir(config->stateDir + "/temproots") - , fnTempRoots(fmt("%s/%d", tempRootsDir, getpid())) - , activeBuildsDir(config->stateDir + "/active-builds") + , dbDir(config->stateDir.get() / "db") + , linksDir(config->realStoreDir.get() / ".links") + , reservedPath(dbDir / "reserved") + , schemaPath(dbDir / "schema") + , tempRootsDir(config->stateDir.get() / "temproots") + , fnTempRoots(tempRootsDir / std::to_string(getpid())) + , activeBuildsDir(config->stateDir.get() / "active-builds") { auto state(_state->lock()); state->stmts = std::make_unique(); @@ -139,18 +139,17 @@ LocalStore::LocalStore(ref config) makeStoreWritable(); } createDirs(linksDir); - Path profilesDir = config->stateDir + "/profiles"; + auto profilesDir = config->stateDir.get() / "profiles"; createDirs(profilesDir); createDirs(tempRootsDir); createDirs(dbDir); - Path gcRootsDir = config->stateDir + "/gcroots"; - if (!pathExists(gcRootsDir)) { - createDirs(gcRootsDir); - replaceSymlink(profilesDir, gcRootsDir + "/profiles"); - } + auto gcRootsDir = config->stateDir.get() / "gcroots"; + const auto & localSettings = config->getLocalSettings(); + const auto & gcSettings = localSettings.getGCSettings(); + createDirs(gcRootsDir); createDirs(activeBuildsDir); - for (auto & perUserDir : {profilesDir + "/per-user", gcRootsDir + "/per-user"}) { + for (auto & perUserDir : {profilesDir / "per-user", gcRootsDir / "per-user"}) { createDirs(perUserDir); if (!config->readOnly) { // Skip chmod call if the directory already has the correct permissions (0755). @@ -163,38 +162,36 @@ LocalStore::LocalStore(ref config) #ifndef _WIN32 /* Optionally, create directories and set permissions for a multi-user install. */ - if (isRootUser() && settings.buildUsersGroup != "") { + if (isRootUser() && localSettings.buildUsersGroup != "") { mode_t perm = 01775; - struct group * gr = getgrnam(settings.buildUsersGroup.get().c_str()); + struct group * gr = getgrnam(localSettings.buildUsersGroup.get().c_str()); if (!gr) printError( - "warning: the group '%1%' specified in 'build-users-group' does not exist", settings.buildUsersGroup); + "warning: the group '%1%' specified in 'build-users-group' does not exist", + localSettings.buildUsersGroup); else if (!config->readOnly) { - struct stat st; - if (stat(config->realStoreDir.get().c_str(), &st)) - throw SysError("getting attributes of path '%1%'", config->realStoreDir); + auto st = stat(config->realStoreDir.get()); if (st.st_uid != 0 || st.st_gid != gr->gr_gid || (st.st_mode & ~S_IFMT) != perm) { if (chown(config->realStoreDir.get().c_str(), 0, gr->gr_gid) == -1) - throw SysError("changing ownership of path '%1%'", config->realStoreDir); - if (chmod(config->realStoreDir.get().c_str(), perm) == -1) - throw SysError("changing permissions on path '%1%'", config->realStoreDir); + throw SysError("changing ownership of path %s", PathFmt(config->realStoreDir.get())); + chmod(config->realStoreDir.get(), perm); } } } #endif /* Ensure that the store and its parents are not symlinks. */ - if (!settings.allowSymlinkedStore) { + if (!localSettings.allowSymlinkedStore) { std::filesystem::path path = config->realStoreDir.get(); std::filesystem::path root = path.root_path(); while (path != root) { if (std::filesystem::is_symlink(path)) throw Error( - "the path '%1%' is a symlink; " + "the path %1% is a symlink; " "this is not allowed for the Nix store and its parent directories", - path); + PathFmt(path)); path = path.parent_path(); } } @@ -204,10 +201,10 @@ LocalStore::LocalStore(ref config) needed, we reserve some dummy space that we can free just before doing a garbage collection. */ try { - struct stat st; - if (stat(reservedPath.c_str(), &st) == -1 || st.st_size != settings.reservedSize) { + auto st = maybeStat(reservedPath); + if (!st || st->st_size != gcSettings.reservedSize) { AutoCloseFD fd = toDescriptor(open( - reservedPath.c_str(), + reservedPath.string().c_str(), O_WRONLY | O_CREAT #ifndef _WIN32 | O_CLOEXEC @@ -216,16 +213,16 @@ LocalStore::LocalStore(ref config) 0600)); int res = -1; #if HAVE_POSIX_FALLOCATE - res = posix_fallocate(fd.get(), 0, settings.reservedSize); + res = posix_fallocate(fd.get(), 0, gcSettings.reservedSize); #endif if (res == -1) { - writeFull(fd.get(), std::string(settings.reservedSize, 'X')); + writeFull(fd.get(), std::string(gcSettings.reservedSize, 'X')); [[gnu::unused]] auto res2 = #ifdef _WIN32 SetEndOfFile(fd.get()) #else - ftruncate(fd.get(), settings.reservedSize) + ftruncate(fd.get(), gcSettings.reservedSize) #endif ; } @@ -236,11 +233,11 @@ LocalStore::LocalStore(ref config) /* Acquire the big fat lock in shared mode to make sure that no schema upgrade is in progress. */ if (!config->readOnly) { - Path globalLockPath = dbDir + "/big-lock"; + auto globalLockPath = dbDir / "big-lock"; try { - globalLock = openLockFile(globalLockPath.c_str(), true); - } catch (SysError & e) { - if (e.errNo == EACCES || e.errNo == EPERM) { + globalLock = openLockFile(globalLockPath, true); + } catch (SystemError & e) { + if (e.is(std::errc::permission_denied) || e.is(std::errc::operation_not_permitted)) { e.addTrace( {}, "This command may have been run as non-root in a single-user Nix installation,\n" @@ -396,29 +393,14 @@ LocalStore::LocalStore(ref config) where drvPath = ? ; )"); - state->stmts->QueryRealisationReferences.create( - state->db, - R"( - select drvPath, outputName from Realisations - join RealisationsRefs on realisationReference = Realisations.id - where referrer = ?; - )"); - state->stmts->AddRealisationReference.create( - state->db, - R"( - insert or replace into RealisationsRefs (referrer, realisationReference) - values ( - (select id from Realisations where drvPath = ? and outputName = ?), - (select id from Realisations where drvPath = ? and outputName = ?)); - )"); } } AutoCloseFD LocalStore::openGCLock() { - Path fnGCLock = config->stateDir + "/gc.lock"; + auto fnGCLock = config->stateDir.get() / "gc.lock"; auto fdGCLock = open( - fnGCLock.c_str(), + fnGCLock.string().c_str(), O_RDWR | O_CREAT #ifndef _WIN32 | O_CLOEXEC @@ -426,13 +408,30 @@ AutoCloseFD LocalStore::openGCLock() , 0600); if (!fdGCLock) - throw SysError("opening global GC lock '%1%'", fnGCLock); + throw SysError("opening global GC lock %1%", PathFmt(fnGCLock)); return toDescriptor(fdGCLock); } -void LocalStore::deleteStorePath(const Path & path, uint64_t & bytesFreed) +void LocalStore::deleteStorePath(const std::filesystem::path & path, uint64_t & bytesFreed, bool isKnownPath) { - deletePath(path, bytesFreed); + try { + deletePath(path, bytesFreed); + } catch (SystemError & e) { + if (config->ignoreGcDeleteFailure) { + logWarning( + {.msg = HintFmt( + isKnownPath ? "ignoring failure to remove store path %1%: %2%" + : "ignoring failure to remove garbage in store directory %1%: %2%", + PathFmt(path), + e.info().msg)}); + } else { + e.addTrace( + {}, + isKnownPath ? "While deleting store path %1%" : "While deleting garbage in store directory %1%", + PathFmt(path)); + throw; + } + } } LocalStore::~LocalStore() @@ -454,29 +453,41 @@ LocalStore::~LocalStore() auto fdTempRoots(_fdTempRoots.lock()); if (*fdTempRoots) { fdTempRoots->close(); - unlink(fnTempRoots.c_str()); + tryUnlink(fnTempRoots); } } catch (...) { ignoreExceptionInDestructor(); } } +std::filesystem::path LocalStoreConfig::getRootsSocketPath() const +{ + return std::filesystem::path(stateDir.get()) / "gc-roots-socket" / "socket"; +} + StoreReference LocalStoreConfig::getReference() const { auto params = getQueryParams(); /* Back-compatibility kludge. Tools like nix-output-monitor expect 'local' and can't parse 'local://'. */ if (params.empty()) + /* TODO: Add the rootDir here as the authority? */ return {.variant = StoreReference::Local{}}; return { .variant = StoreReference::Specified{ .scheme = *uriSchemes().begin(), + /* TODO: Add the rootDir here as the authority? */ }, .params = std::move(params), }; } +bool LocalStoreConfig::getReadOnly() const +{ + return readOnly.get() || StoreConfig::getReadOnly(); +} + int LocalStore::getSchema() { int curSchema = 0; @@ -484,7 +495,7 @@ int LocalStore::getSchema() auto s = readFile(schemaPath); auto n = string2Int(s); if (!n) - throw Error("'%1%' is corrupt", schemaPath); + throw Error("%1% is corrupt", PathFmt(schemaPath)); curSchema = *n; } return curSchema; @@ -496,15 +507,15 @@ void LocalStore::openDB(State & state, bool create) throw Error("cannot create database while in read-only mode"); } - if (access(dbDir.c_str(), R_OK | (config->readOnly ? 0 : W_OK))) - throw SysError("Nix database directory '%1%' is not writable", dbDir); + if (access(dbDir.string().c_str(), R_OK | (config->readOnly ? 0 : W_OK))) + throw SysError("Nix database directory %1% is not writable", PathFmt(dbDir)); /* Open the Nix database. */ auto & db(state.db); auto openMode = config->readOnly ? SQLiteOpenMode::Immutable : create ? SQLiteOpenMode::Normal : SQLiteOpenMode::NoCreate; - state.db = SQLite(std::filesystem::path(dbDir) / "db.sqlite", openMode); + state.db = SQLite(dbDir / "db.sqlite", {.mode = openMode, .useWAL = settings.useSQLiteWAL}); #ifdef __CYGWIN__ /* The cygwin version of sqlite3 has a patch which calls @@ -523,7 +534,7 @@ void LocalStore::openDB(State & state, bool create) should be safe enough. If the user asks for it, don't sync at all. This can cause database corruption if the system crashes. */ - std::string syncMode = settings.fsyncMetadata ? "normal" : "off"; + std::string syncMode = config->getLocalSettings().fsyncMetadata ? "normal" : "off"; db.exec("pragma synchronous = " + syncMode); /* Set the SQLite journal mode. WAL mode is fastest, so it's the @@ -620,7 +631,7 @@ void LocalStore::makeStoreWritable() if (stat.f_flag & ST_RDONLY) { if (mount(0, config->realStoreDir.get().c_str(), "none", MS_REMOUNT | MS_BIND, 0) == -1) - throw SysError("remounting %1% writable", config->realStoreDir); + throw SysError("remounting %s writable", PathFmt(config->realStoreDir.get())); } #endif } @@ -645,7 +656,8 @@ void LocalStore::registerDrvOutput(const Realisation & info) auto combinedSignatures = oldR->signatures; combinedSignatures.insert(info.signatures.begin(), info.signatures.end()); state->stmts->UpdateRealisedOutput - .use()(concatStringsSep(" ", combinedSignatures))(info.id.strHash())(info.id.outputName) + .use()(concatStringsSep(" ", Signature::toStrings(combinedSignatures)))(info.id.strHash())( + info.id.outputName) .exec(); } else { throw Error( @@ -660,26 +672,7 @@ void LocalStore::registerDrvOutput(const Realisation & info) } else { state->stmts->RegisterRealisedOutput .use()(info.id.strHash())(info.id.outputName)(printStorePath(info.outPath))( - concatStringsSep(" ", info.signatures)) - .exec(); - } - for (auto & [outputId, depPath] : info.dependentRealisations) { - auto localRealisation = queryRealisationCore_(*state, outputId); - if (!localRealisation) - throw Error( - "unable to register the derivation '%s' as it " - "depends on the non existent '%s'", - info.id.to_string(), - outputId.to_string()); - if (localRealisation->second.outPath != depPath) - throw Error( - "unable to register the derivation '%s' as it " - "depends on a realisation of '%s' that doesn’t" - "match what we have locally", - info.id.to_string(), - outputId.to_string()); - state->stmts->AddRealisationReference - .use()(info.id.strHash())(info.id.outputName)(outputId.strHash())(outputId.outputName) + concatStringsSep(" ", Signature::toStrings(info.signatures))) .exec(); } }); @@ -692,18 +685,19 @@ void LocalStore::cacheDrvOutputMapping( [&]() { state.stmts->AddDerivationOutput.use()(deriver)(outputName) (printStorePath(output)).exec(); }); } -uint64_t LocalStore::addValidPath(State & state, const ValidPathInfo & info, bool checkOutputs) +uint64_t LocalStore::addValidPath(State & state, const ValidPathInfo & info) { if (info.ca.has_value() && !info.isContentAddressed(*this)) throw Error( "cannot add path '%s' to the Nix store because it claims to be content-addressed but isn't", printStorePath(info.path)); - auto query = state.stmts->RegisterValidPath.use()(printStorePath(info.path))( - info.narHash.to_string(HashFormat::Base16, true))(info.registrationTime == 0 ? time(0) : info.registrationTime)( + auto query = state.stmts->RegisterValidPath.use()(printStorePath(info.path))(info.narHash.to_string( + HashFormat::Base16, true))(info.registrationTime == 0 ? time(nullptr) : info.registrationTime)( info.deriver ? printStorePath(*info.deriver) : "", (bool) info.deriver)(info.narSize, info.narSize != 0)(info.ultimate ? 1 : 0, info.ultimate)( - concatStringsSep(" ", info.sigs), !info.sigs.empty())(renderContentAddress(info.ca), (bool) info.ca); + concatStringsSep(" ", Signature::toStrings(info.sigs)), + !info.sigs.empty())(renderContentAddress(info.ca), (bool) info.ca); if (experimentalFeatureSettings.isEnabled(Xp::Provenance)) query(info.provenance ? info.provenance->to_json_str() : "", (bool) info.provenance); query.exec(); @@ -714,17 +708,16 @@ uint64_t LocalStore::addValidPath(State & state, const ValidPathInfo & info, boo efficiently query whether a path is an output of some derivation. */ if (info.path.isDerivation()) { - auto drv = readInvalidDerivation(info.path); + auto parsedDrv = readInvalidDerivation(info.path); /* Verify that the output paths in the derivation are correct (i.e., follow the scheme for computing output paths from derivations). Note that if this throws an error, then the DB transaction is rolled back, so the path validity registration above is undone. */ - if (checkOutputs) - drv.checkInvariants(*this, info.path); + parsedDrv.checkInvariants(*this, info.path); - for (auto & i : drv.outputsAndOptPaths(*this)) { + for (auto & i : parsedDrv.outputsAndOptPaths(*this)) { /* Floating CA derivations have indeterminate output paths until they are built, so don't register anything in that case */ if (i.second.second) @@ -784,7 +777,7 @@ std::shared_ptr LocalStore::queryPathInfoInternal(State & s s = (const char *) sqlite3_column_text(state.stmts->QueryPathInfo, 6); if (s) - info->sigs = tokenizeString(s, " "); + info->sigs = Signature::parseMany(tokenizeString(s, " ")); s = (const char *) sqlite3_column_text(state.stmts->QueryPathInfo, 7); if (s) @@ -810,7 +803,8 @@ void LocalStore::updatePathInfo(State & state, const ValidPathInfo & info) { state.stmts->UpdatePathInfo .use()(info.narSize, info.narSize != 0)(info.narHash.to_string(HashFormat::Base16, true))( - info.ultimate ? 1 : 0, info.ultimate)(concatStringsSep(" ", info.sigs), !info.sigs.empty())( + info.ultimate ? 1 : 0, + info.ultimate)(concatStringsSep(" ", Signature::toStrings(info.sigs)), !info.sigs.empty())( renderContentAddress(info.ca), (bool) info.ca)(printStorePath(info.path)) .exec(); } @@ -903,7 +897,7 @@ std::optional LocalStore::queryPathFromHashPart(const std::string & h if (hashPart.size() != StorePath::HashLen) throw Error("invalid hash part"); - Path prefix = storeDir + "/" + hashPart; + std::string prefix = storeDir + "/" + hashPart; return retrySQLite>([&]() -> std::optional { auto state(_state->lock()); @@ -920,40 +914,6 @@ std::optional LocalStore::queryPathFromHashPart(const std::string & h }); } -StorePathSet LocalStore::querySubstitutablePaths(const StorePathSet & paths) -{ - if (!settings.useSubstitutes) - return StorePathSet(); - - StorePathSet remaining; - for (auto & i : paths) - remaining.insert(i); - - StorePathSet res; - - for (auto & sub : getDefaultSubstituters()) { - if (remaining.empty()) - break; - if (sub->storeDir != storeDir) - continue; - if (!sub->config.wantMassQuery) - continue; - - auto valid = sub->queryValidPaths(remaining); - - StorePathSet remaining2; - for (auto & path : remaining) - if (valid.count(path)) - res.insert(path); - else - remaining2.insert(path); - - std::swap(remaining, remaining2); - } - - return res; -} - void LocalStore::registerValidPath(const ValidPathInfo & info) { registerValidPaths({{info.path, info}}); @@ -966,7 +926,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) be fsync-ed. So some may want to fsync them before registering the validity, at the expense of some speed of the path registering operation. */ - if (settings.syncBeforeRegistering) + if (config->getLocalSettings().syncBeforeRegistering) sync(); #endif @@ -981,7 +941,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) if (isValidPath_(*state, i.path)) updatePathInfo(*state, i); else - addValidPath(*state, i, false); + addValidPath(*state, i); paths.insert(i.path); } @@ -991,15 +951,6 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) state->stmts->AddReference.use()(referrer)(queryValidPathId(*state, j)).exec(); } - /* Check that the derivation outputs are correct. We can't do - this in addValidPath() above, because the references might - not be valid yet. */ - for (auto & [_, i] : infos) - if (i.path.isDerivation()) { - // FIXME: inefficient; we already loaded the derivation in addValidPath(). - readInvalidDerivation(i.path).checkInvariants(*this, i.path); - } - /* Do a topological sort of the paths. This will throw an error if a cycle is detected and roll back the transaction. Cycles can only occur when a derivation @@ -1087,7 +1038,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, RepairF TeeSource wrapperSource{source, hashSink}; - restorePath(realPath, wrapperSource, settings.fsyncStorePaths); + restorePath(realPath, wrapperSource, config->getLocalSettings().fsyncStorePaths); auto hashResult = hashSink.finish(); @@ -1143,11 +1094,11 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, RepairF autoGC(); - canonicalisePathMetaData(realPath); + canonicalisePathMetaData(realPath, {NIX_WHEN_SUPPORT_ACLS(config->getLocalSettings().ignoredAcls)}); optimisePath(realPath, repair); // FIXME: combine with hashPath() - if (settings.fsyncStorePaths) { + if (config->getLocalSettings().fsyncStorePaths) { recursiveSync(realPath); syncParent(realPath); } @@ -1176,6 +1127,7 @@ StorePath LocalStore::addToStoreFromDump( /* For computing the store path. */ auto hashSink = std::make_unique(hashAlgo); TeeSource source{source0, *hashSink}; + const LocalSettings & localSettings = config->getLocalSettings(); /* Read the source path into memory, but only if it's up to narBufferSize bytes. If it's larger, write it to a temporary @@ -1199,10 +1151,10 @@ StorePath LocalStore::addToStoreFromDump( /* Fill out buffer, and decide whether we are working strictly in memory based on whether we break out because the buffer is full or the original source is empty */ - while (dump.size() < settings.narBufferSize) { + while (dump.size() < localSettings.narBufferSize) { auto oldSize = dump.size(); constexpr size_t chunkSize = 65536; - auto want = std::min(chunkSize, settings.narBufferSize - oldSize); + auto want = std::min(chunkSize, localSettings.narBufferSize - oldSize); if (auto tmp = realloc(dumpBuffer.get(), oldSize + want)) { dumpBuffer.release(); dumpBuffer.reset((char *) tmp); @@ -1239,7 +1191,7 @@ StorePath LocalStore::addToStoreFromDump( delTempDir = std::make_unique(tempDir); tempPath = tempDir / "x"; - restorePath(tempPath.string(), bothSource, dumpMethod, settings.fsyncStorePaths); + restorePath(tempPath.string(), bothSource, dumpMethod, localSettings.fsyncStorePaths); dumpBuffer.reset(); dump = {}; @@ -1283,7 +1235,7 @@ StorePath LocalStore::addToStoreFromDump( switch (fim) { case FileIngestionMethod::Flat: case FileIngestionMethod::NixArchive: - restorePath(realPath, dumpSource, (FileSerialisationMethod) fim, settings.fsyncStorePaths); + restorePath(realPath, dumpSource, (FileSerialisationMethod) fim, localSettings.fsyncStorePaths); break; case FileIngestionMethod::Git: // doesn't correspond to serialization method, so @@ -1304,11 +1256,12 @@ StorePath LocalStore::addToStoreFromDump( narHash = narSink.finish(); } - canonicalisePathMetaData(realPath); // FIXME: merge into restorePath + canonicalisePathMetaData( + realPath, {NIX_WHEN_SUPPORT_ACLS(localSettings.ignoredAcls)}); // FIXME: merge into restorePath optimisePath(realPath, repair); - if (settings.fsyncStorePaths) { + if (localSettings.fsyncStorePaths) { recursiveSync(realPath); syncParent(realPath); } @@ -1359,7 +1312,9 @@ void LocalStore::invalidatePathChecked(const StorePath & path) referrers.erase(path); /* ignore self-references */ if (!referrers.empty()) throw PathInUse( - "cannot delete path '%s' because it is in use by %s", printStorePath(path), showPaths(referrers)); + "cannot delete path '%s' because it is in use by %s", + printStorePath(path), + concatMapStringsSep(", ", referrers, [&](auto & p) { return "'" + printStorePath(p) + "'"; })); invalidatePath(*state, path); } @@ -1386,15 +1341,16 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) for (auto & link : DirectoryIterator{linksDir}) { checkInterrupt(); auto name = link.path().filename(); - printMsg(lvlTalkative, "checking contents of %s", name); + printMsg(lvlTalkative, "checking contents of %s", PathFmt(name)); std::string hash = hashPath(makeFSSourceAccessor(link.path()), FileIngestionMethod::NixArchive, HashAlgorithm::SHA256) .first.to_string(HashFormat::Nix32, false); if (hash != name.string()) { - printError("link %s was modified! expected hash %s, got '%s'", link.path(), name, hash); + printError( + "link %s was modified! expected hash %s, got '%s'", PathFmt(link.path()), name.string(), hash); if (repair) { std::filesystem::remove(link.path()); - printInfo("removed link %s", link.path()); + printInfo("removed link %s", PathFmt(link.path())); } else { errors = true; } @@ -1456,7 +1412,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) if (isValidPath(i)) logError(e.info()); else - warn(e.msg()); + logWarning(e.info()); errors = true; } } @@ -1506,7 +1462,7 @@ LocalStore::VerificationResult LocalStore::verifyAllValidPaths(RepairFlag repair void LocalStore::verifyPath( const StorePath & path, - std::function existsInStoreDir, + fun existsInStoreDir, StorePathSet & done, StorePathSet & validPaths, RepairFlag repair, @@ -1556,7 +1512,7 @@ void LocalStore::verifyPath( unsigned int LocalStore::getProtocol() { - return PROTOCOL_VERSION; + return WorkerProto::latest.number.toWire(); } std::optional LocalStore::isTrustedClient() @@ -1569,7 +1525,7 @@ void LocalStore::vacuumDB() _state->lock()->db.exec("vacuum"); } -void LocalStore::addSignatures(const StorePath & storePath, const StringSet & sigs) +void LocalStore::addSignatures(const StorePath & storePath, const std::set & sigs) { retrySQLite([&]() { auto state(_state->lock()); @@ -1600,7 +1556,7 @@ LocalStore::queryRealisationCore_(LocalStore::State & state, const DrvOutput & i {realisationDbId, UnkeyedRealisation{ .outPath = outputPath, - .signatures = signatures, + .signatures = Signature::parseMany(signatures), }}}; } @@ -1611,21 +1567,6 @@ std::optional LocalStore::queryRealisation_(LocalStore return std::nullopt; auto [realisationDbId, res] = *maybeCore; - std::map dependentRealisations; - auto useRealisationRefs(state.stmts->QueryRealisationReferences.use()(realisationDbId)); - while (useRealisationRefs.next()) { - auto depId = DrvOutput{ - Hash::parseAnyPrefixed(useRealisationRefs.getStr(0)), - useRealisationRefs.getStr(1), - }; - auto dependentRealisation = queryRealisationCore_(state, depId); - assert(dependentRealisation); // Enforced by the db schema - auto outputPath = dependentRealisation->second.outPath; - dependentRealisations.insert({depId, outputPath}); - } - - res.dependentRealisations = dependentRealisations; - return {res}; } @@ -1655,16 +1596,18 @@ void LocalStore::addBuildLog(const StorePath & drvPath, std::string_view log) auto baseName = drvPath.to_string(); - auto logPath = fmt("%s/%s/%s/%s.bz2", config->logDir, drvsLogDir, baseName.substr(0, 2), baseName.substr(2)); + auto logPath = + config->logDir.get() / drvsLogDir / baseName.substr(0, 2) / (std::string(baseName.substr(2)) + ".bz2"); if (pathExists(logPath)) return; - createDirs(dirOf(logPath)); + createDirs(logPath.parent_path()); - auto tmpFile = fmt("%s.tmp.%d", logPath, getpid()); + auto tmpFile = logPath; + tmpFile += ".tmp." + std::to_string(getpid()); - writeFile(tmpFile, compress("bzip2", log)); + writeFile(tmpFile, compress(CompressionAlgo::bzip2, log)); std::filesystem::rename(tmpFile, logPath); } diff --git a/src/libstore/machines.cc b/src/libstore/machines.cc index d614676668bf..3b11ec902525 100644 --- a/src/libstore/machines.cc +++ b/src/libstore/machines.cc @@ -69,8 +69,8 @@ StoreReference Machine::completeStoreReference() const } if (generic && (generic->scheme == "ssh" || generic->scheme == "ssh-ng")) { - if (sshKey != "") - storeUri.params["ssh-key"] = sshKey; + if (!sshKey.empty()) + storeUri.params["ssh-key"] = sshKey.string(); if (sshPublicHostKey != "") storeUri.params["base64-ssh-public-host-key"] = sshPublicHostKey; } @@ -111,8 +111,8 @@ static std::vector expandBuilderLines(const std::string & builders) std::string text; try { text = readFile(path); - } catch (const SysError & e) { - if (e.errNo != ENOENT) + } catch (const SystemError & e) { + if (!e.is(std::errc::no_such_file_or_directory)) throw; debug("cannot find machines file '%s'", path); continue; @@ -207,9 +207,4 @@ Machines Machine::parseConfig(const StringSet & defaultSystems, const std::strin return parseBuilderLines(defaultSystems, builderLines); } -Machines getMachines() -{ - return Machine::parseConfig({settings.thisSystem}, settings.builders); -} - } // namespace nix diff --git a/src/libstore/meson.build b/src/libstore/meson.build index 7a53fd65d86a..f6de0e35fd52 100644 --- a/src/libstore/meson.build +++ b/src/libstore/meson.build @@ -47,35 +47,37 @@ deps_public_maybe_subproject = [ ] subdir('nix-meson-build-support/subprojects') -run_command( - 'ln', - '-s', - meson.project_build_root() / '__nothing_link_target', - meson.project_build_root() / '__nothing_symlink', - # native doesn't allow dangling symlinks, which the tests require - env : {'MSYS' : 'winsymlinks:lnk'}, - check : true, -) -can_link_symlink = run_command( - 'ln', - meson.project_build_root() / '__nothing_symlink', - meson.project_build_root() / '__nothing_hardlink', - check : false, -).returncode() == 0 -run_command( - 'rm', - '-f', - meson.project_build_root() / '__nothing_symlink', - meson.project_build_root() / '__nothing_hardlink', - check : true, -) +can_link_symlink = false +native_ln = find_program('ln', required : false, native : true) +if native_ln.found() + run_command( + native_ln, + '-s', + meson.project_build_root() / '__nothing_link_target', + meson.project_build_root() / '__nothing_symlink', + # native doesn't allow dangling symlinks, which the tests require + env : {'MSYS' : 'winsymlinks:lnk'}, + check : true, + ) + can_link_symlink = run_command( + native_ln, + meson.project_build_root() / '__nothing_symlink', + meson.project_build_root() / '__nothing_hardlink', + check : false, + ).returncode() == 0 + run_command( + 'rm', + '-f', + meson.project_build_root() / '__nothing_symlink', + meson.project_build_root() / '__nothing_hardlink', + check : true, + ) +endif summary('can hardlink to symlink', can_link_symlink, bool_yn : true) configdata_priv.set('CAN_LINK_SYMLINK', can_link_symlink.to_int()) -# Check for each of these functions, and create a define like `#define HAVE_LCHOWN 1`. +# Check for each of these functions, and create a define like `#define HAVE_POSIX_FALLOCATE 1`. check_funcs = [ - # Optionally used for canonicalising files from the build - 'lchown', 'posix_fallocate', 'statvfs', ] @@ -121,17 +123,10 @@ boost = dependency( # put in `deps_other`. deps_other += boost -curl = dependency('libcurl', 'curl', version : '>= 7.75.0') -if curl.version().version_compare('>=8.16.0') and curl.version().version_compare( - '<8.17.0', -) - # Out of precaution, avoid building with libcurl version that suffer from https://github.com/curl/curl/issues/19334. - error( - 'curl @0@ has issues with write pausing, please use libcurl < 8.16 or >= 8.17, see https://github.com/curl/curl/issues/19334'.format( - curl.version(), - ), - ) -endif +# This is quite new, but curl has a bunch of known issues with write pausing and decompression. +# Please use libcurl >= 8.17. See https://github.com/curl/curl/issues/19334, https://github.com/curl/curl/issues/16280, https://github.com/curl/curl/issues/16955 if in doubt. +# Patch out this check at your own risk. +curl = dependency('libcurl', 'curl', version : '>= 8.17.0') deps_private += curl @@ -162,14 +157,15 @@ sqlite = dependency('sqlite3', 'sqlite', version : '>=3.6.19') deps_private += sqlite s3_aws_auth = get_option('s3-aws-auth') -aws_crt_cpp = cxx.find_library('aws-crt-cpp', required : s3_aws_auth) +aws_crt_cpp = dependency( + 'aws-crt-cpp', + required : s3_aws_auth, + method : 'cmake', + modules : [ 'AWS::aws-crt-cpp' ], +) if s3_aws_auth.enabled() deps_other += aws_crt_cpp - aws_c_common = cxx.find_library('aws-c-common', required : true) - deps_other += aws_c_common - aws_c_auth = cxx.find_library('aws-c-auth', required : true) - deps_other += aws_c_auth endif configdata_pub.set('NIX_WITH_AWS_AUTH', s3_aws_auth.enabled().to_int()) @@ -215,7 +211,6 @@ prefix = get_option('prefix') # it is already an absolute path (which is the default for store-dir, localstatedir, and log-dir). path_opts = [ # Meson built-ins. - 'datadir', 'mandir', 'libdir', 'includedir', @@ -243,20 +238,24 @@ endforeach # sysconfdir doesn't get anything installed to directly, and is only used to # tell Nix where to look for nix.conf, so it doesn't get appended to prefix. -sysconfdir = get_option('sysconfdir') -if not fs.is_absolute(sysconfdir) - sysconfdir = '/' / sysconfdir +if host_machine.system() != 'windows' + sysconfdir = get_option('sysconfdir') + if not fs.is_absolute(sysconfdir) + sysconfdir = '/' / sysconfdir + endif endif # Aside from prefix itself, each of these was made into an absolute path # by joining it with prefix, unless it was already an absolute path # (which is the default for store-dir, localstatedir, and log-dir). -configdata_priv.set_quoted('NIX_PREFIX', prefix) configdata_priv.set_quoted('NIX_STORE_DIR', store_dir) -configdata_priv.set_quoted('NIX_DATA_DIR', datadir) configdata_priv.set_quoted('NIX_STATE_DIR', localstatedir / 'nix') configdata_priv.set_quoted('NIX_LOG_DIR', log_dir) -configdata_priv.set_quoted('NIX_CONF_DIR', sysconfdir / 'nix') +# On Windows, NIX_CONF_DIR is determined at runtime using the Windows known +# folders API (FOLDERID_ProgramData), so we don't define it at compile time. +if host_machine.system() != 'windows' + configdata_priv.set_quoted('NIX_CONF_DIR', sysconfdir / 'nix') +endif configdata_priv.set_quoted('NIX_MAN_DIR', mandir) lsof = find_program('lsof', required : false) @@ -292,6 +291,7 @@ sources = files( 'async-path-writer.cc', 'binary-cache-store.cc', 'build-result.cc', + 'build/build-log.cc', 'build/derivation-builder.cc', 'build/derivation-building-goal.cc', 'build/derivation-check.cc', @@ -327,6 +327,7 @@ sources = files( 'legacy-ssh-store.cc', 'local-binary-cache-store.cc', 'local-fs-store.cc', + 'local-gc.cc', 'local-overlay-store.cc', 'local-store-active-builds.cc', 'local-store.cc', diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index f9a339a00574..e55dc424f0c5 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -1,4 +1,5 @@ #include "nix/store/derivations.hh" +#include "nix/util/fun.hh" #include "nix/store/parsed-derivations.hh" #include "nix/store/derivation-options.hh" #include "nix/store/globals.hh" @@ -66,7 +67,7 @@ void Store::computeFSClosure( paths_, [&](const StorePath & path, std::function> &)> processEdges) { std::promise> promise; - std::function>)> getDependencies = + fun>)> getDependencies = [&](std::future> fut) { try { promise.set_value(queryDeps(path, fut)); @@ -125,7 +126,7 @@ MissingPaths Store::queryMissing(const std::vector & targets) Sync state_; - std::function doPath; + fun doPath = [&](const DerivedPath &) { unreachable(); }; auto enqueueDerivedPaths = [&](this auto self, ref inputDrv, @@ -236,7 +237,8 @@ MissingPaths Store::queryMissing(const std::vector & targets) throw; } - if (!knownOutputPaths && settings.useSubstitutes && drvOptions.substitutesAllowed()) { + if (!knownOutputPaths && settings.getWorkerSettings().useSubstitutes + && drvOptions.substitutesAllowed(settings.getWorkerSettings())) { experimentalFeatureSettings.require(Xp::CaDerivations); // If there are unknown output paths, attempt to find if the @@ -266,7 +268,8 @@ MissingPaths Store::queryMissing(const std::vector & targets) } } - if (knownOutputPaths && settings.useSubstitutes && drvOptions.substitutesAllowed()) { + if (knownOutputPaths && settings.getWorkerSettings().useSubstitutes + && drvOptions.substitutesAllowed(settings.getWorkerSettings())) { auto drvState = make_ref>(DrvState(invalid.size())); for (auto & output : invalid) pool.enqueue(std::bind(checkOutput, drvPath, drv, output, drvState)); @@ -334,65 +337,6 @@ StorePaths Store::topoSortPaths(const StorePathSet & paths) result); } -std::map -drvOutputReferences(const std::set & inputRealisations, const StorePathSet & pathReferences) -{ - std::map res; - - for (const auto & input : inputRealisations) { - if (pathReferences.count(input.outPath)) { - res.insert({input.id, input.outPath}); - } - } - - return res; -} - -std::map -drvOutputReferences(Store & store, const Derivation & drv, const StorePath & outputPath, Store * evalStore_) -{ - auto & evalStore = evalStore_ ? *evalStore_ : store; - - std::set inputRealisations; - - auto accumRealisations = [&](this auto & self, - const StorePath & inputDrv, - const DerivedPathMap::ChildNode & inputNode) -> void { - if (!inputNode.value.empty()) { - auto outputHashes = staticOutputHashes(evalStore, evalStore.readDerivation(inputDrv)); - for (const auto & outputName : inputNode.value) { - auto outputHash = get(outputHashes, outputName); - if (!outputHash) - throw Error( - "output '%s' of derivation '%s' isn't realised", outputName, store.printStorePath(inputDrv)); - DrvOutput key{*outputHash, outputName}; - auto thisRealisation = store.queryRealisation(key); - if (!thisRealisation) - throw Error( - "output '%s' of derivation '%s' isn’t built", outputName, store.printStorePath(inputDrv)); - inputRealisations.insert({*thisRealisation, std::move(key)}); - } - } - if (!inputNode.value.empty()) { - auto d = makeConstantStorePathRef(inputDrv); - for (const auto & [outputName, childNode] : inputNode.childMap) { - SingleDerivedPath next = SingleDerivedPath::Built{d, outputName}; - self( - // TODO deep resolutions for dynamic derivations, issue #8947, would go here. - resolveDerivedPath(store, next, evalStore_), - childNode); - } - } - }; - - for (const auto & [inputDrv, inputNode] : drv.inputDrvs.map) - accumRealisations(inputDrv, inputNode); - - auto info = store.queryPathInfo(outputPath); - - return drvOutputReferences(Realisation::closure(store, inputRealisations), info->references); -} - OutputPathMap resolveDerivedPath(Store & store, const DerivedPath::Built & bfd, Store * evalStore_) { auto drvPath = resolveDerivedPath(store, *bfd.drvPath, evalStore_); diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index ff1481b912b3..2ab411d80814 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -60,17 +60,15 @@ create table if not exists LastPurge ( )sql"; -class NarInfoDiskCacheImpl : public NarInfoDiskCache +struct NarInfoDiskCacheImpl : NarInfoDiskCache { -public: - /* How often to purge expired entries from the cache. */ const int purgeInterval = 24 * 3600; struct Cache { int id; - Path storeDir; + std::string storeDir; bool wantMassQuery; int priority; }; @@ -85,13 +83,17 @@ class NarInfoDiskCacheImpl : public NarInfoDiskCache Sync _state; - NarInfoDiskCacheImpl(Path dbPath = (getCacheDir() / "binary-cache-detsys-v1.sqlite").string()) + NarInfoDiskCacheImpl( + const Settings & settings, + SQLiteSettings sqliteSettings, + std::filesystem::path dbPath = getCacheDir() / "binary-cache-detsys-v1.sqlite") + : NarInfoDiskCache{settings} { auto state(_state.lock()); - createDirs(dirOf(dbPath)); + createDirs(dbPath.parent_path()); - state->db = SQLite(dbPath); + state->db = SQLite(dbPath, SQLite::Settings{sqliteSettings}); state->db.isCache(); @@ -142,7 +144,7 @@ class NarInfoDiskCacheImpl : public NarInfoDiskCache /* Periodically purge expired entries from the database. */ retrySQLite([&]() { - auto now = time(0); + auto now = time(nullptr); SQLiteStmt queryLastPurge(state->db, "select value from LastPurge"); auto queryLastPurge_(queryLastPurge.use()); @@ -154,8 +156,8 @@ class NarInfoDiskCacheImpl : public NarInfoDiskCache .use() // Use a minimum TTL to prevent --refresh from // nuking the entire disk cache. - (now - std::max(settings.ttlNegativeNarInfoCache.get(), 3600U))( - now - std::max(settings.ttlPositiveNarInfoCache.get(), 30 * 24 * 3600U)) + (now - std::max(settings.ttlNegative.get(), 3600U))( + now - std::max(settings.ttlPositive.get(), 30 * 24 * 3600U)) .exec(); debug("deleted %d entries from the NAR info disk cache", sqlite3_changes(state->db)); @@ -181,7 +183,11 @@ class NarInfoDiskCacheImpl : public NarInfoDiskCache { auto i = state.caches.find(uri); if (i == state.caches.end()) { - auto queryCache(state.queryCache.use()(uri)(time(0) - settings.ttlNarInfoCacheMeta)); + /* Important: always use int64_t even on 32 bit systems. Otherwise + the the subtraction would promote time_t to unsigned if time_t is + 32 bit. */ + auto timestamp = static_cast(time(nullptr)) - static_cast(settings.ttlMeta.get()); + auto queryCache(state.queryCache.use()(uri)(timestamp)); if (!queryCache.next()) return std::nullopt; auto cache = Cache{ @@ -196,7 +202,7 @@ class NarInfoDiskCacheImpl : public NarInfoDiskCache } public: - int createCache(const std::string & uri, const Path & storeDir, bool wantMassQuery, int priority) override + int createCache(const std::string & uri, const std::string & storeDir, bool wantMassQuery, int priority) override { return retrySQLite([&]() { auto state(_state.lock()); @@ -217,7 +223,7 @@ class NarInfoDiskCacheImpl : public NarInfoDiskCache }; { - auto r(state->insertCache.use()(uri)(time(0))(storeDir) (wantMassQuery) (priority)); + auto r(state->insertCache.use()(uri)(time(nullptr))(storeDir) (wantMassQuery) (priority)); if (!r.next()) { unreachable(); } @@ -251,10 +257,10 @@ class NarInfoDiskCacheImpl : public NarInfoDiskCache auto & cache(getCache(*state, uri)); - auto now = time(0); + auto now = time(nullptr); - auto queryNAR(state->queryNAR.use()(cache.id)(hashPart) (now - settings.ttlNegativeNarInfoCache)( - now - settings.ttlPositiveNarInfoCache)); + auto queryNAR( + state->queryNAR.use()(cache.id)(hashPart) (now - settings.ttlNegative)(now - settings.ttlPositive)); if (!queryNAR.next()) return {oUnknown, 0}; @@ -276,7 +282,7 @@ class NarInfoDiskCacheImpl : public NarInfoDiskCache if (!queryNAR.isNull(9)) narInfo->deriver = StorePath(queryNAR.getStr(9)); for (auto & sig : tokenizeString(queryNAR.getStr(10), " ")) - narInfo->sigs.insert(sig); + narInfo->sigs.insert(Signature::parse(sig)); narInfo->ca = ContentAddress::parseOpt(queryNAR.getStr(11)); if (experimentalFeatureSettings.isEnabled(Xp::Provenance) && !queryNAR.isNull(12)) narInfo->provenance = Provenance::from_json_str_optional(queryNAR.getStr(12)); @@ -294,10 +300,10 @@ class NarInfoDiskCacheImpl : public NarInfoDiskCache auto & cache(getCache(*state, uri)); - auto now = time(0); + auto now = time(nullptr); auto queryRealisation(state->queryRealisation.use()(cache.id)(id.to_string())( - now - settings.ttlNegativeNarInfoCache)(now - settings.ttlPositiveNarInfoCache)); + now - settings.ttlNegative)(now - settings.ttlPositive)); if (!queryRealisation.next()) return {oUnknown, 0}; @@ -338,14 +344,14 @@ class NarInfoDiskCacheImpl : public NarInfoDiskCache narInfo && narInfo->fileHash)( narInfo ? narInfo->fileSize : 0, narInfo != 0 && narInfo->fileSize)(info->narHash.to_string( HashFormat::Nix32, true))(info->narSize)(concatStringsSep(" ", info->shortRefs()))( - info->deriver ? std::string(info->deriver->to_string()) : "", - (bool) info->deriver)(concatStringsSep(" ", info->sigs))(renderContentAddress(info->ca))( + info->deriver ? std::string(info->deriver->to_string()) : "", (bool) info->deriver)( + concatStringsSep(" ", Signature::toStrings(info->sigs)))(renderContentAddress(info->ca))( info->provenance ? info->provenance->to_json_str() : "", - experimentalFeatureSettings.isEnabled(Xp::Provenance) && info->provenance)(time(0)) + experimentalFeatureSettings.isEnabled(Xp::Provenance) && info->provenance)(time(nullptr)) .exec(); } else { - state->insertMissingNAR.use()(cache.id)(hashPart) (time(0)).exec(); + state->insertMissingNAR.use()(cache.id)(hashPart) (time(nullptr)).exec(); } }); } @@ -358,7 +364,8 @@ class NarInfoDiskCacheImpl : public NarInfoDiskCache auto & cache(getCache(*state, uri)); state->insertRealisation - .use()(cache.id)(realisation.id.to_string())(static_cast(realisation).dump())(time(0)) + .use()(cache.id)(realisation.id.to_string())(static_cast(realisation).dump())( + time(nullptr)) .exec(); }); } @@ -369,20 +376,21 @@ class NarInfoDiskCacheImpl : public NarInfoDiskCache auto state(_state.lock()); auto & cache(getCache(*state, uri)); - state->insertMissingRealisation.use()(cache.id)(id.to_string())(time(0)).exec(); + state->insertMissingRealisation.use()(cache.id)(id.to_string())(time(nullptr)).exec(); }); } }; -ref getNarInfoDiskCache() +ref NarInfoDiskCache::get(const Settings & settings, SQLiteSettings sqliteSettings) { - static ref cache = make_ref(); + static ref cache = make_ref(settings, sqliteSettings); return cache; } -ref getTestNarInfoDiskCache(Path dbPath) +ref +NarInfoDiskCache::getTest(const Settings & settings, SQLiteSettings sqliteSettings, std::filesystem::path dbPath) { - return make_ref(dbPath); + return make_ref(settings, sqliteSettings, dbPath); } } // namespace nix diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index ef72987699d9..0f225f22c7d3 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -79,7 +79,7 @@ NarInfo::NarInfo(const StoreDirConfig & store, const std::string & s, const std: if (value != "unknown-deriver") deriver = StorePath(value); } else if (name == "Sig") - sigs.insert(value); + sigs.insert(Signature::parse(value)); else if (name == "CA") { if (ca) throw corrupt("extra CA"); @@ -126,7 +126,7 @@ std::string NarInfo::to_string(const StoreDirConfig & store) const res += "Deriver: " + std::string(deriver->to_string()) + "\n"; for (const auto & sig : sigs) - res += "Sig: " + sig + "\n"; + res += "Sig: " + sig.to_string() + "\n"; if (ca) res += "CA: " + renderContentAddress(*ca) + "\n"; diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index d1e858300277..7d37dbf9d4a2 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -3,6 +3,7 @@ #include "nix/util/signals.hh" #include "nix/store/posix-fs-canonicalise.hh" #include "nix/util/posix-source-accessor.hh" +#include "nix/util/file-system.hh" #include #include @@ -17,19 +18,18 @@ namespace nix { -static void makeWritable(const Path & path) +static void makeWritable(const std::filesystem::path & path) { auto st = lstat(path); - if (chmod(path.c_str(), st.st_mode | S_IWUSR) == -1) - throw SysError("changing writability of '%1%'", path); + chmod(path, st.st_mode | S_IWUSR); } struct MakeReadOnly { - Path path; + std::filesystem::path path; - MakeReadOnly(const PathView path) - : path(path) + MakeReadOnly(std::filesystem::path path) + : path(std::move(path)) { } @@ -37,8 +37,8 @@ struct MakeReadOnly { try { /* This will make the path read-only. */ - if (path != "") - canonicaliseTimestampAndPermissions(path); + if (!path.empty()) + canonicaliseTimestampAndPermissions(path.string()); } catch (...) { ignoreExceptionInDestructor(); } @@ -50,9 +50,9 @@ LocalStore::InodeHash LocalStore::loadInodeHash() debug("loading hash inodes in memory"); InodeHash inodeHash; - AutoCloseDir dir(opendir(linksDir.c_str())); + AutoCloseDir dir(opendir(linksDir.string().c_str())); if (!dir) - throw SysError("opening directory '%1%'", linksDir); + throw SysError("opening directory %1%", PathFmt(linksDir)); struct dirent * dirent; while (errno = 0, dirent = readdir(dir.get())) { /* sic */ @@ -61,20 +61,20 @@ LocalStore::InodeHash LocalStore::loadInodeHash() inodeHash.insert(dirent->d_ino); } if (errno) - throw SysError("reading directory '%1%'", linksDir); + throw SysError("reading directory %1%", PathFmt(linksDir)); printMsg(lvlTalkative, "loaded %1% hash inodes", inodeHash.size()); return inodeHash; } -Strings LocalStore::readDirectoryIgnoringInodes(const Path & path, const InodeHash & inodeHash) +Strings LocalStore::readDirectoryIgnoringInodes(const std::filesystem::path & path, const InodeHash & inodeHash) { Strings names; - AutoCloseDir dir(opendir(path.c_str())); + AutoCloseDir dir(opendir(path.string().c_str())); if (!dir) - throw SysError("opening directory '%1%'", path); + throw SysError("opening directory %s", PathFmt(path)); struct dirent * dirent; while (errno = 0, dirent = readdir(dir.get())) { /* sic */ @@ -91,13 +91,13 @@ Strings LocalStore::readDirectoryIgnoringInodes(const Path & path, const InodeHa names.push_back(name); } if (errno) - throw SysError("reading directory '%1%'", path); + throw SysError("reading directory %s", PathFmt(path)); return names; } void LocalStore::optimisePath_( - Activity * act, OptimiseStats & stats, const Path & path, InodeHash & inodeHash, RepairFlag repair) + Activity * act, OptimiseStats & stats, const std::filesystem::path & path, InodeHash & inodeHash, RepairFlag repair) { checkInterrupt(); @@ -110,8 +110,8 @@ void LocalStore::optimisePath_( See https://github.com/NixOS/nix/issues/1443 and https://github.com/NixOS/nix/pull/2230 for more discussion. */ - if (std::regex_search(path, std::regex("\\.app/Contents/.+$"))) { - debug("'%1%' is not allowed to be linked in macOS", path); + if (std::regex_search(path.string(), std::regex("\\.app/Contents/.+$"))) { + debug("%s is not allowed to be linked in macOS", PathFmt(path)); return; } #endif @@ -119,7 +119,7 @@ void LocalStore::optimisePath_( if (S_ISDIR(st.st_mode)) { Strings names = readDirectoryIgnoringInodes(path, inodeHash); for (auto & i : names) - optimisePath_(act, stats, path + "/" + i, inodeHash, repair); + optimisePath_(act, stats, path / i, inodeHash, repair); return; } @@ -136,13 +136,13 @@ void LocalStore::optimisePath_( NixOS (example: $fontconfig/var/cache being modified). Skip those files. FIXME: check the modification time. */ if (S_ISREG(st.st_mode) && (st.st_mode & S_IWUSR)) { - warn("skipping suspicious writable file '%1%'", path); + warn("skipping suspicious writable file '%s'", PathFmt(path)); return; } /* This can still happen on top-level files. */ if (st.st_nlink > 1 && inodeHash.count(st.st_ino)) { - debug("'%s' is already linked, with %d other file(s)", path, st.st_nlink - 2); + debug("%s is already linked, with %d other file(s)", PathFmt(path), st.st_nlink - 2); return; } @@ -157,19 +157,19 @@ void LocalStore::optimisePath_( the contents of the target (which may not even exist). */ Hash hash = ({ hashPath( - {make_ref(), CanonPath(path)}, + {make_ref(), CanonPath(path.string())}, FileSerialisationMethod::NixArchive, HashAlgorithm::SHA256) .hash; }); - debug("'%1%' has hash '%2%'", path, hash.to_string(HashFormat::Nix32, true)); + debug("%s has hash '%s'", PathFmt(path), hash.to_string(HashFormat::Nix32, true)); /* Check if this is a known hash. */ std::filesystem::path linkPath = std::filesystem::path{linksDir} / hash.to_string(HashFormat::Nix32, false); /* Maybe delete the link, if it has been corrupted. */ if (std::filesystem::exists(std::filesystem::symlink_status(linkPath))) { - auto stLink = lstat(linkPath.string()); + auto stLink = lstat(linkPath); if (st.st_size != stLink.st_size || (repair && hash != ({ hashPath( makeFSSourceAccessor(linkPath), @@ -178,7 +178,7 @@ void LocalStore::optimisePath_( .hash; }))) { // XXX: Consider overwriting linkPath with our valid version. - warn("removing corrupted link %s", linkPath); + warn("removing corrupted link %s", PathFmt(linkPath)); warn( "There may be more corrupted paths." "\nYou should run `nix-store --verify --check-contents --repair` to fix them all"); @@ -201,38 +201,39 @@ void LocalStore::optimisePath_( /* On ext4, that probably means the directory index is full. When that happens, it's fine to ignore it: we just effectively disable deduplication of this - file. */ - printInfo("cannot link %s to '%s': %s", linkPath, path, strerror(errno)); + file. + */ + printInfo("cannot link %s to '%s': %s", PathFmt(linkPath), PathFmt(path), e.code().message()); return; } else - throw; + throw SystemError(e.code(), "creating hard link from %1% to %2%", PathFmt(linkPath), PathFmt(path)); } } /* Yes! We've seen a file with the same contents. Replace the current file with a hard link to that file. */ - auto stLink = lstat(linkPath.string()); + auto stLink = lstat(linkPath); if (st.st_ino == stLink.st_ino) { - debug("'%1%' is already linked to %2%", path, linkPath); + debug("%1% is already linked to %2%", PathFmt(path), PathFmt(linkPath)); return; } - printMsg(lvlTalkative, "linking '%1%' to %2%", path, linkPath); + printMsg(lvlTalkative, "linking %1% to %2%", PathFmt(path), PathFmt(linkPath)); /* Make the containing directory writable, but only if it's not the store itself (we don't want or need to mess with its permissions). */ - const Path dirOfPath(dirOf(path)); + const auto dirOfPath = path.parent_path(); bool mustToggle = dirOfPath != config->realStoreDir.get(); if (mustToggle) makeWritable(dirOfPath); /* When we're done, make the directory read-only again and reset its timestamp back to 0. */ - MakeReadOnly makeReadOnly(mustToggle ? dirOfPath : ""); + MakeReadOnly makeReadOnly(mustToggle ? dirOfPath : std::filesystem::path{}); std::filesystem::path tempLink = makeTempPath(config->realStoreDir.get(), ".tmp-link"); @@ -245,10 +246,10 @@ void LocalStore::optimisePath_( systems). This is likely to happen with empty files. Just shrug and ignore. */ if (st.st_size) - printInfo("%1% has maximum number of links", linkPath); + printInfo("%1% has maximum number of links", PathFmt(linkPath)); return; } - throw; + throw SystemError(e.code(), "creating hard link from %1% to %2%", PathFmt(linkPath), PathFmt(tempLink)); } /* Atomically replace the old file with the new hard link. */ @@ -259,17 +260,17 @@ void LocalStore::optimisePath_( std::error_code ec; remove(tempLink, ec); /* Clean up after ourselves. */ if (ec) - printError("unable to unlink %1%: %2%", tempLink, ec.message()); + printError("unable to unlink %1%: %2%", PathFmt(tempLink), ec.message()); } if (e.code() == std::errc::too_many_links) { /* Some filesystems generate too many links on the rename, rather than on the original link. (Probably it temporarily increases the st_nlink field before decreasing it again.) */ - debug("%s has reached maximum number of links", linkPath); + debug("%s has reached maximum number of links", PathFmt(linkPath)); return; } - throw; + throw SystemError(e.code(), "renaming %1% to %2%", PathFmt(tempLink), PathFmt(path)); } stats.filesLinked++; @@ -303,7 +304,7 @@ void LocalStore::optimiseStore(OptimiseStats & stats) continue; /* path was GC'ed, probably */ { Activity act(*logger, lvlTalkative, actUnknown, fmt("optimising path '%s'", printStorePath(i))); - optimisePath_(&act, stats, config->realStoreDir + "/" + std::string(i.to_string()), inodeHash, NoRepair); + optimisePath_(&act, stats, config->realStoreDir.get() / i.to_string(), inodeHash, NoRepair); } done++; act.progress(done, paths.size()); @@ -319,12 +320,12 @@ void LocalStore::optimiseStore() printInfo("%s freed by hard-linking %d files", renderSize(stats.bytesFreed), stats.filesLinked); } -void LocalStore::optimisePath(const Path & path, RepairFlag repair) +void LocalStore::optimisePath(const std::filesystem::path & path, RepairFlag repair) { OptimiseStats stats; InodeHash inodeHash; - if (settings.autoOptimiseStore) + if (config->getLocalSettings().autoOptimiseStore) optimisePath_(nullptr, stats, path, inodeHash, repair); } diff --git a/src/libstore/package.nix b/src/libstore/package.nix index a660b9a4646a..ecd5d0ae2e6f 100644 --- a/src/libstore/package.nix +++ b/src/libstore/package.nix @@ -9,10 +9,12 @@ nix-util, boost, curl, + aws-c-common, aws-crt-cpp, libseccomp, nlohmann_json, sqlite, + cmake, # for resolving aws-crt-cpp dep wasmtime, busybox-sandbox-shell ? null, @@ -25,7 +27,7 @@ withAWS ? # Default is this way because there have been issues building this dependency - stdenv.hostPlatform == stdenv.buildPlatform && (stdenv.isLinux || stdenv.isDarwin), + (lib.meta.availableOn stdenv.hostPlatform aws-c-common) && !stdenv.hostPlatform.isStatic, enableWasm ? !stdenv.hostPlatform.isStatic, }: @@ -61,7 +63,8 @@ mkMesonLibrary (finalAttrs: { (fileset.fileFilter (file: file.hasExt "sql") ./.) ]; - nativeBuildInputs = lib.optional embeddedSandboxShell unixtools.hexdump; + nativeBuildInputs = + lib.optional withAWS cmake ++ lib.optional embeddedSandboxShell unixtools.hexdump; buildInputs = [ boost diff --git a/src/libstore/path-info.cc b/src/libstore/path-info.cc index c5a9eba16c86..e1803e1014d0 100644 --- a/src/libstore/path-info.cc +++ b/src/libstore/path-info.cc @@ -130,7 +130,7 @@ size_t ValidPathInfo::checkSignatures(const StoreDirConfig & store, const Public } bool ValidPathInfo::checkSignature( - const StoreDirConfig & store, const PublicKeys & publicKeys, const std::string & sig) const + const StoreDirConfig & store, const PublicKeys & publicKeys, const Signature & sig) const { return verifyDetached(fingerprint(store), sig, publicKeys); } @@ -212,9 +212,7 @@ UnkeyedValidPathInfo::toJSON(const StoreDirConfig * store, bool includeImpureInf jsonObject["ultimate"] = ultimate; - auto & sigsObj = jsonObject["signatures"] = json::array(); - for (auto & sig : sigs) - sigsObj.push_back(sig); + jsonObject["signatures"] = sigs; if (experimentalFeatureSettings.isEnabled(Xp::Provenance)) jsonObject["provenance"] = provenance ? provenance->to_json() : nullptr; @@ -291,7 +289,7 @@ UnkeyedValidPathInfo UnkeyedValidPathInfo::fromJSON(const StoreDirConfig * store res.ultimate = getBoolean(*rawUltimate); if (auto * rawSignatures = optionalValueAt(json, "signatures")) - res.sigs = getStringSet(*rawSignatures); + res.sigs = *rawSignatures; if (experimentalFeatureSettings.isEnabled(Xp::Provenance)) { auto prov = json.find("provenance"); diff --git a/src/libstore/path-references.cc b/src/libstore/path-references.cc index 3d783bbe4be5..409f6de6479f 100644 --- a/src/libstore/path-references.cc +++ b/src/libstore/path-references.cc @@ -47,7 +47,7 @@ StorePathSet PathRefScanSink::getResultPaths() return found; } -StorePathSet scanForReferences(Sink & toTee, const Path & path, const StorePathSet & refs) +StorePathSet scanForReferences(Sink & toTee, const std::filesystem::path & path, const StorePathSet & refs) { PathRefScanSink refsSink = PathRefScanSink::fromPaths(refs); TeeSink sink{refsSink, toTee}; @@ -62,7 +62,7 @@ void scanForReferencesDeep( SourceAccessor & accessor, const CanonPath & rootPath, const StorePathSet & refs, - std::function callback) + fun callback) { // Recursive tree walker auto walk = [&](this auto & self, const CanonPath & path) -> void { @@ -124,7 +124,7 @@ void scanForReferencesDeep( case SourceAccessor::tFifo: case SourceAccessor::tUnknown: default: - throw Error("file '%s' has an unsupported type", path.abs()); + throw Error("file '%s' has an unsupported type", accessor.showPath(path)); } }; diff --git a/src/libstore/posix-fs-canonicalise.cc b/src/libstore/posix-fs-canonicalise.cc index a274468c3295..455cc7c4f5da 100644 --- a/src/libstore/posix-fs-canonicalise.cc +++ b/src/libstore/posix-fs-canonicalise.cc @@ -1,10 +1,10 @@ #include "nix/store/posix-fs-canonicalise.hh" +#include "nix/store/build-result.hh" #include "nix/util/file-system.hh" #include "nix/util/signals.hh" #include "nix/util/util.hh" -#include "nix/store/globals.hh" #include "nix/store/store-api.hh" - +#include "nix/store/globals.hh" #include "store-config-private.hh" #if NIX_SUPPORT_ACL @@ -15,7 +15,7 @@ namespace nix { const time_t mtimeStore = 1; /* 1 second into the epoch */ -static void canonicaliseTimestampAndPermissions(const Path & path, const struct stat & st) +static void canonicaliseTimestampAndPermissions(const std::filesystem::path & path, const PosixStat & st) { if (!S_ISLNK(st.st_mode)) { @@ -24,30 +24,25 @@ static void canonicaliseTimestampAndPermissions(const Path & path, const struct bool isDir = S_ISDIR(st.st_mode); if ((mode != 0444 || isDir) && mode != 0555) { mode = (st.st_mode & S_IFMT) | 0444 | (st.st_mode & S_IXUSR || isDir ? 0111 : 0); - if (chmod(path.c_str(), mode) == -1) - throw SysError("changing mode of '%1%' to %2$o", path, mode); + chmod(path, mode); } } #ifndef _WIN32 // TODO implement if (st.st_mtime != mtimeStore) { - struct stat st2 = st; + PosixStat st2 = st; st2.st_mtime = mtimeStore, setWriteTime(path, st2); } #endif } -void canonicaliseTimestampAndPermissions(const Path & path) +void canonicaliseTimestampAndPermissions(const std::filesystem::path & path) { canonicaliseTimestampAndPermissions(path, lstat(path)); } static void canonicalisePathMetaData_( - const Path & path, -#ifndef _WIN32 - std::optional> uidRange, -#endif - InodesSeen & inodesSeen) + const std::filesystem::path & path, CanonicalizePathMetadataOptions options, InodesSeen & inodesSeen) { checkInterrupt(); @@ -57,7 +52,7 @@ static void canonicalisePathMetaData_( setattrlist() to remove other attributes as well. */ if (lchflags(path.c_str(), 0)) { if (errno != ENOTSUP) - throw SysError("clearing flags of path '%1%'", path); + throw SysError("clearing flags of path %1%", PathFmt(path)); } #endif @@ -65,7 +60,7 @@ static void canonicalisePathMetaData_( /* Really make sure that the path is of a supported type. */ if (!(S_ISREG(st.st_mode) || S_ISDIR(st.st_mode) || S_ISLNK(st.st_mode))) - throw Error("file '%1%' has an unsupported type", path); + throw Error("file %1% has an unsupported type", PathFmt(path)); #if NIX_SUPPORT_ACL /* Remove extended attributes / ACLs. */ @@ -73,18 +68,18 @@ static void canonicalisePathMetaData_( if (eaSize < 0) { if (errno != ENOTSUP && errno != ENODATA) - throw SysError("querying extended attributes of '%s'", path); + throw SysError("querying extended attributes of %s", PathFmt(path)); } else if (eaSize > 0) { std::vector eaBuf(eaSize); if ((eaSize = llistxattr(path.c_str(), eaBuf.data(), eaBuf.size())) < 0) - throw SysError("querying extended attributes of '%s'", path); + throw SysError("querying extended attributes of %s", PathFmt(path)); for (auto & eaName : tokenizeString(std::string(eaBuf.data(), eaSize), std::string("\000", 1))) { - if (settings.ignoredAcls.get().count(eaName)) + if (options.ignoredAcls.count(eaName)) continue; if (lremovexattr(path.c_str(), eaName.c_str()) == -1) - throw SysError("removing extended attribute '%s' from '%s'", eaName, path); + throw SysError("removing extended attribute '%s' from %s", eaName, PathFmt(path)); } } #endif @@ -96,9 +91,9 @@ static void canonicalisePathMetaData_( However, ignore files that we chown'ed ourselves previously to ensure that we don't fail on hard links within the same build (i.e. "touch $out/foo; ln $out/foo $out/bar"). */ - if (uidRange && (st.st_uid < uidRange->first || st.st_uid > uidRange->second)) { + if (options.uidRange && (st.st_uid < options.uidRange->first || st.st_uid > options.uidRange->second)) { if (S_ISDIR(st.st_mode) || !inodesSeen.count(Inode(st.st_dev, st.st_ino))) - throw BuildError(BuildResult::Failure::OutputRejected, "invalid ownership on file '%1%'", path); + throw BuildError(BuildResult::Failure::OutputRejected, "invalid ownership on file %1%", PathFmt(path)); mode_t mode = st.st_mode & ~S_IFMT; assert( S_ISLNK(st.st_mode) @@ -112,77 +107,31 @@ static void canonicalisePathMetaData_( canonicaliseTimestampAndPermissions(path, st); #ifndef _WIN32 - /* Change ownership to the current uid. If it's a symlink, use - lchown if available, otherwise don't bother. Wrong ownership - of a symlink doesn't matter, since the owning user can't change - the symlink and can't delete it because the directory is not - writable. The only exception is top-level paths in the Nix - store (since that directory is group-writable for the Nix build - users group); we check for this case below. */ + /* Change ownership to the current uid. */ if (st.st_uid != geteuid()) { -# if HAVE_LCHOWN if (lchown(path.c_str(), geteuid(), getegid()) == -1) -# else - if (!S_ISLNK(st.st_mode) && chown(path.c_str(), geteuid(), getegid()) == -1) -# endif - throw SysError("changing owner of '%1%' to %2%", path, geteuid()); + throw SysError("changing owner of %1% to %2%", PathFmt(path), geteuid()); } #endif if (S_ISDIR(st.st_mode)) { for (auto & i : DirectoryIterator{path}) { checkInterrupt(); - canonicalisePathMetaData_( - i.path().string(), -#ifndef _WIN32 - uidRange, -#endif - inodesSeen); + canonicalisePathMetaData_(i.path(), options, inodesSeen); } } } void canonicalisePathMetaData( - const Path & path, -#ifndef _WIN32 - std::optional> uidRange, -#endif - InodesSeen & inodesSeen) + const std::filesystem::path & path, CanonicalizePathMetadataOptions options, InodesSeen & inodesSeen) { - canonicalisePathMetaData_( - path, -#ifndef _WIN32 - uidRange, -#endif - inodesSeen); - -#ifndef _WIN32 - /* On platforms that don't have lchown(), the top-level path can't - be a symlink, since we can't change its ownership. */ - auto st = lstat(path); - - if (st.st_uid != geteuid()) { - assert(S_ISLNK(st.st_mode)); - throw Error("wrong ownership of top-level store path '%1%'", path); - } -#endif + canonicalisePathMetaData_(path, options, inodesSeen); } -void canonicalisePathMetaData( - const Path & path -#ifndef _WIN32 - , - std::optional> uidRange -#endif -) +void canonicalisePathMetaData(const std::filesystem::path & path, CanonicalizePathMetadataOptions options) { InodesSeen inodesSeen; - canonicalisePathMetaData_( - path, -#ifndef _WIN32 - uidRange, -#endif - inodesSeen); + canonicalisePathMetaData_(path, options, inodesSeen); } } // namespace nix diff --git a/src/libstore/profiles.cc b/src/libstore/profiles.cc index 22d3f8f8973c..02ecae561684 100644 --- a/src/libstore/profiles.cc +++ b/src/libstore/profiles.cc @@ -103,7 +103,7 @@ static void removeFile(const std::filesystem::path & path) try { std::filesystem::remove(path); } catch (std::filesystem::filesystem_error & e) { - throw SysError("removing file '%1%'", path); + throw SystemError(e.code(), "removing file %1%", PathFmt(path)); } } @@ -141,7 +141,7 @@ void deleteGenerations( auto [gens, curGen] = findGenerations(profile); if (gensToDelete.count(*curGen)) - throw Error("cannot delete current version of profile %1%'", profile); + throw Error("cannot delete current version of profile %1%", PathFmt(profile)); for (auto & i : gens) { if (!gensToDelete.count(i.number)) @@ -234,7 +234,7 @@ time_t parseOlderThanTimeSpec(std::string_view timeSpec) if (timeSpec.empty() || timeSpec[timeSpec.size() - 1] != 'd') throw UsageError("invalid number of days specifier '%1%', expected something like '14d'", timeSpec); - time_t curTime = time(0); + time_t curTime = time(nullptr); auto strDays = timeSpec.substr(0, timeSpec.size() - 1); auto days = string2Int(strDays); @@ -282,7 +282,7 @@ void switchGeneration(const std::filesystem::path & profile, std::optional buildHost, - std::map tags, + StringMap tags, std::string system, std::shared_ptr next) : drvPath(drvPath) @@ -51,9 +51,9 @@ Provenance::Register registerBuildProvenance("build", [](nlohmann::json json) { std::optional buildHost; if (auto p = optionalValueAt(obj, "buildHost")) buildHost = p->get>(); - std::map tags; + StringMap tags; if (auto p = optionalValueAt(obj, "tags"); p && !p->is_null()) - tags = p->get>(); + tags = p->get(); auto buildProv = make_ref( StorePath(getString(valueAt(obj, "drv"))), getString(valueAt(obj, "output")), diff --git a/src/libstore/realisation.cc b/src/libstore/realisation.cc index 4aeb05874fb6..433d67f26ba2 100644 --- a/src/libstore/realisation.cc +++ b/src/libstore/realisation.cc @@ -1,6 +1,5 @@ #include "nix/store/realisation.hh" #include "nix/store/store-api.hh" -#include "nix/util/closure.hh" #include "nix/util/signature/local-keys.hh" #include "nix/util/json-utils.hh" #include @@ -26,41 +25,6 @@ std::string DrvOutput::to_string() const return strHash() + "!" + outputName; } -std::set Realisation::closure(Store & store, const std::set & startOutputs) -{ - std::set res; - Realisation::closure(store, startOutputs, res); - return res; -} - -void Realisation::closure(Store & store, const std::set & startOutputs, std::set & res) -{ - auto getDeps = [&](const Realisation & current) -> std::set { - std::set res; - for (auto & [currentDep, _] : current.dependentRealisations) { - if (auto currentRealisation = store.queryRealisation(currentDep)) - res.insert({*currentRealisation, currentDep}); - else - throw Error("Unrealised derivation '%s'", currentDep.to_string()); - } - return res; - }; - - computeClosure( - startOutputs, - res, - [&](const Realisation & current, std::function> &)> processEdges) { - std::promise> promise; - try { - auto res = getDeps(current); - promise.set_value(res); - } catch (...) { - promise.set_exception(std::current_exception()); - } - return processEdges(promise); - }); -} - std::string UnkeyedRealisation::fingerprint(const DrvOutput & key) const { nlohmann::json serialized = Realisation{*this, key}; @@ -74,7 +38,7 @@ void UnkeyedRealisation::sign(const DrvOutput & key, const Signer & signer) } bool UnkeyedRealisation::checkSignature( - const DrvOutput & key, const PublicKeys & publicKeys, const std::string & sig) const + const DrvOutput & key, const PublicKeys & publicKeys, const Signature & sig) const { return verifyDetached(fingerprint(key), sig, publicKeys); } @@ -99,43 +63,7 @@ const StorePath & RealisedPath::path() const & bool Realisation::isCompatibleWith(const UnkeyedRealisation & other) const { - if (outPath == other.outPath) { - if (dependentRealisations.empty() != other.dependentRealisations.empty()) { - warn( - "Encountered a realisation for '%s' with an empty set of " - "dependencies. This is likely an artifact from an older Nix. " - "I’ll try to fix the realisation if I can", - id.to_string()); - return true; - } else if (dependentRealisations == other.dependentRealisations) { - return true; - } - } - return false; -} - -void RealisedPath::closure(Store & store, const RealisedPath::Set & startPaths, RealisedPath::Set & ret) -{ - // FIXME: This only builds the store-path closure, not the real realisation - // closure - StorePathSet initialStorePaths, pathsClosure; - for (auto & path : startPaths) - initialStorePaths.insert(path.path()); - store.computeFSClosure(initialStorePaths, pathsClosure); - ret.insert(startPaths.begin(), startPaths.end()); - ret.insert(pathsClosure.begin(), pathsClosure.end()); -} - -void RealisedPath::closure(Store & store, RealisedPath::Set & ret) const -{ - RealisedPath::closure(store, {*this}, ret); -} - -RealisedPath::Set RealisedPath::closure(Store & store) const -{ - RealisedPath::Set ret; - closure(store, ret); - return ret; + return outPath == other.outPath; } } // namespace nix @@ -158,31 +86,23 @@ UnkeyedRealisation adl_serializer::from_json(const json & js { auto json = getObject(json0); - StringSet signatures; - if (auto signaturesOpt = optionalValueAt(json, "signatures")) - signatures = *signaturesOpt; - - std::map dependentRealisations; - if (auto jsonDependencies = optionalValueAt(json, "dependentRealisations")) - for (auto & [jsonDepId, jsonDepOutPath] : getObject(*jsonDependencies)) - dependentRealisations.insert({DrvOutput::parse(jsonDepId), jsonDepOutPath}); - return UnkeyedRealisation{ .outPath = valueAt(json, "outPath"), - .signatures = signatures, - .dependentRealisations = dependentRealisations, + .signatures = [&] -> std::set { + if (auto signaturesOpt = optionalValueAt(json, "signatures")) + return *signaturesOpt; + return {}; + }(), }; } void adl_serializer::to_json(json & json, const UnkeyedRealisation & r) { - auto jsonDependentRealisations = nlohmann::json::object(); - for (auto & [depId, depOutPath] : r.dependentRealisations) - jsonDependentRealisations.emplace(depId.to_string(), depOutPath); json = { {"outPath", r.outPath}, {"signatures", r.signatures}, - {"dependentRealisations", jsonDependentRealisations}, + // back-compat + {"dependentRealisations", json::object()}, }; } diff --git a/src/libstore/remote-fs-accessor.cc b/src/libstore/remote-fs-accessor.cc index 51bab9953548..31356a506a09 100644 --- a/src/libstore/remote-fs-accessor.cc +++ b/src/libstore/remote-fs-accessor.cc @@ -1,52 +1,12 @@ -#include #include "nix/store/remote-fs-accessor.hh" -#include "nix/util/nar-accessor.hh" - -#include -#include -#include namespace nix { -RemoteFSAccessor::RemoteFSAccessor(ref store, bool requireValidPath, const Path & cacheDir) +RemoteFSAccessor::RemoteFSAccessor(ref store, bool requireValidPath, std::optional cacheDir) : store(store) + , narCache(cacheDir) , requireValidPath(requireValidPath) - , cacheDir(cacheDir) -{ - if (cacheDir != "") - createDirs(cacheDir); -} - -Path RemoteFSAccessor::makeCacheFile(std::string_view hashPart, const std::string & ext) -{ - assert(cacheDir != ""); - return fmt("%s/%s.%s", cacheDir, hashPart, ext); -} - -ref RemoteFSAccessor::addToCache(std::string_view hashPart, std::string && nar) { - if (cacheDir != "") { - try { - /* FIXME: do this asynchronously. */ - writeFile(makeCacheFile(hashPart, "nar"), nar); - } catch (...) { - ignoreExceptionExceptInterrupt(); - } - } - - auto narAccessor = makeNarAccessor(std::move(nar)); - nars.emplace(hashPart, narAccessor); - - if (cacheDir != "") { - try { - nlohmann::json j = listNarDeep(*narAccessor, CanonPath::root); - writeFile(makeCacheFile(hashPart, "ls"), j.dump()); - } catch (...) { - ignoreExceptionExceptInterrupt(); - } - } - - return narAccessor; } std::pair, CanonPath> RemoteFSAccessor::fetch(const CanonPath & path) @@ -54,46 +14,29 @@ std::pair, CanonPath> RemoteFSAccessor::fetch(const CanonPat auto [storePath, restPath] = store->toStorePath(store->storeDir + path.abs()); if (requireValidPath && !store->isValidPath(storePath)) throw InvalidPath("path '%1%' is not a valid store path", store->printStorePath(storePath)); - return {ref{accessObject(storePath)}, CanonPath{restPath}}; + return {ref{accessObject(storePath)}, restPath}; } std::shared_ptr RemoteFSAccessor::accessObject(const StorePath & storePath) { - auto i = nars.find(std::string(storePath.hashPart())); - if (i != nars.end()) - return i->second; - - std::string listing; - Path cacheFile; - - if (cacheDir != "" && nix::pathExists(cacheFile = makeCacheFile(storePath.hashPart(), "nar"))) { - - try { - listing = nix::readFile(makeCacheFile(storePath.hashPart(), "ls")); - auto listingJson = nlohmann::json::parse(listing); - auto narAccessor = makeLazyNarAccessor(listingJson, seekableGetNarBytes(cacheFile)); - - nars.emplace(storePath.hashPart(), narAccessor); - return narAccessor; + // Check if we already have the NAR hash for this store path + if (auto * narHash = get(narHashes, storePath.hashPart())) + return narCache.getOrInsert(*narHash, [&](Sink & sink) { store->narFromPath(storePath, sink); }); - } catch (SystemError &) { - } + // Query the path info to get the NAR hash + auto info = store->queryPathInfo(storePath); - try { - auto narAccessor = makeNarAccessor(nix::readFile(cacheFile)); - nars.emplace(storePath.hashPart(), narAccessor); - return narAccessor; - } catch (SystemError &) { - } - } + // Cache the mapping from store path to NAR hash + narHashes.emplace(storePath.hashPart(), info->narHash); - StringSink sink; - store->narFromPath(storePath, sink); - return addToCache(storePath.hashPart(), std::move(sink.s)); + // Get or create the NAR accessor + return narCache.getOrInsert(info->narHash, [&](Sink & sink) { store->narFromPath(storePath, sink); }); } std::optional RemoteFSAccessor::maybeLstat(const CanonPath & path) { + if (path.isRoot()) + return Stat{.type = tDirectory}; auto res = fetch(path); return res.first->maybeLstat(res.second); } @@ -104,10 +47,10 @@ SourceAccessor::DirEntries RemoteFSAccessor::readDirectory(const CanonPath & pat return res.first->readDirectory(res.second); } -std::string RemoteFSAccessor::readFile(const CanonPath & path) +void RemoteFSAccessor::readFile(const CanonPath & path, Sink & sink, fun sizeCallback) { auto res = fetch(path); - return res.first->readFile(res.second); + res.first->readFile(res.second, sink, sizeCallback); } std::string RemoteFSAccessor::readLink(const CanonPath & path) diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 0274df18cbbc..3af877bf1776 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -18,8 +18,13 @@ #include "nix/util/callback.hh" #include "nix/store/filetransfer.hh" #include "nix/util/signals.hh" +#include "nix/util/socket.hh" #include "nix/util/provenance.hh" +#ifndef _WIN32 +# include +#endif + #include namespace nix { @@ -39,6 +44,8 @@ RemoteStore::RemoteStore(const Config & config) failed = true; throw; } + /* Track the connection FD for shutdownConnections() */ + connectionFds.lock()->insert(conn->from.fd); return conn; }, [this](const ref & r) { @@ -53,8 +60,13 @@ RemoteStore::RemoteStore(const Config & config) ref RemoteStore::openConnectionWrapper() { - if (failed) + if (failed) { + checkInterrupt(); + /* Throw Interrupted instead of the following error to silence pesky + warning messages that ThreadPool prints on shutdown if other threads + failed. */ throw Error("opening a connection to remote store '%s' previously failed", config.getHumanReadableURI()); + } try { return openConnection(); } catch (...) { @@ -72,15 +84,12 @@ void RemoteStore::initConnection(Connection & conn) StringSink saved; TeeSource tee(conn.from, saved); try { - auto myFeatures = WorkerProto::allFeatures; + auto version = WorkerProto::latest; if (!experimentalFeatureSettings.isEnabled(Xp::Provenance)) - myFeatures.erase(std::string(WorkerProto::featureProvenance)); - auto [protoVersion, features] = - WorkerProto::BasicClientConnection::handshake(conn.to, tee, PROTOCOL_VERSION, myFeatures); - if (protoVersion < MINIMUM_PROTOCOL_VERSION) + version.features.erase(std::string(WorkerProto::featureProvenance)); + conn.protoVersion = WorkerProto::BasicClientConnection::handshake(conn.to, tee, version); + if (conn.protoVersion.number < WorkerProto::minimum.number) throw Error("the Nix daemon version is too old"); - conn.protoVersion = protoVersion; - conn.features = features; } catch (SerialisationError & e) { /* In case the other side is waiting for our input, close it. */ @@ -94,7 +103,7 @@ void RemoteStore::initConnection(Connection & conn) static_cast(conn) = conn.postHandshake(*this); - for (auto & feature : conn.features) + for (auto & feature : conn.protoVersion.features) debug("negotiated feature '%s'", feature); auto ex = conn.processStderrReturn(); @@ -109,22 +118,23 @@ void RemoteStore::initConnection(Connection & conn) void RemoteStore::setOptions(Connection & conn) { - conn.to << WorkerProto::Op::SetOptions << settings.keepFailed << settings.keepGoing << settings.tryFallback - << verbosity << settings.maxBuildJobs << settings.maxSilentTime << true - << (settings.verboseBuild ? lvlError : lvlVomit) << 0 // obsolete log type - << 0 /* obsolete print build trace */ - << settings.buildCores << settings.useSubstitutes; + conn.to << WorkerProto::Op::SetOptions << settings.keepFailed << settings.getWorkerSettings().keepGoing + << settings.getWorkerSettings().tryFallback << verbosity << settings.getWorkerSettings().maxBuildJobs + << settings.getWorkerSettings().maxSilentTime << true << (settings.verboseBuild ? lvlError : lvlVomit) + << 0 // obsolete log type + << 0 /* obsolete print build trace */ + << settings.getLocalSettings().buildCores << settings.getWorkerSettings().useSubstitutes; std::map overrides; settings.getSettings(overrides, true); // libstore settings fileTransferSettings.getSettings(overrides, true); overrides.erase(settings.keepFailed.name); - overrides.erase(settings.keepGoing.name); - overrides.erase(settings.tryFallback.name); - overrides.erase(settings.maxBuildJobs.name); - overrides.erase(settings.maxSilentTime.name); - overrides.erase(settings.buildCores.name); - overrides.erase(settings.useSubstitutes.name); + overrides.erase(settings.getWorkerSettings().keepGoing.name); + overrides.erase(settings.getWorkerSettings().tryFallback.name); + overrides.erase(settings.getWorkerSettings().maxBuildJobs.name); + overrides.erase(settings.getWorkerSettings().maxSilentTime.name); + overrides.erase(settings.getLocalSettings().buildCores.name); + overrides.erase(settings.getWorkerSettings().useSubstitutes.name); overrides.erase(loggerSettings.showTrace.name); overrides.erase(experimentalFeatureSettings.experimentalFeatures.name); overrides.erase("plugin-files"); @@ -200,7 +210,7 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S auto conn(getConnection()); conn->to << WorkerProto::Op::QuerySubstitutablePathInfos; - if (GET_PROTOCOL_MINOR(conn->protoVersion) < 22) { + if (conn->protoVersion.number < WorkerProto::Version::Number{1, 22}) { StorePathSet paths; for (auto & path : pathsMap) paths.insert(path.first); @@ -256,7 +266,7 @@ StorePathSet RemoteStore::queryValidDerivers(const StorePath & path) StorePathSet RemoteStore::queryDerivationOutputs(const StorePath & path) { - if (GET_PROTOCOL_MINOR(getProtocol()) >= 0x16) { + if (WorkerProto::Version::Number::fromWire(getProtocol()) >= WorkerProto::Version::Number{1, 22}) { return Store::queryDerivationOutputs(path); } auto conn(getConnection()); @@ -269,7 +279,7 @@ StorePathSet RemoteStore::queryDerivationOutputs(const StorePath & path) std::map> RemoteStore::queryPartialDerivationOutputMap(const StorePath & path, Store * evalStore_) { - if (GET_PROTOCOL_MINOR(getProtocol()) >= 0x16) { + if (WorkerProto::Version::Number::fromWire(getProtocol()) >= WorkerProto::Version::Number{1, 22}) { if (!evalStore_) { auto conn(getConnection()); conn->to << WorkerProto::Op::QueryDerivationOutputMap; @@ -321,12 +331,12 @@ ref RemoteStore::addCAToStore( std::optional conn_(getConnection()); auto & conn = *conn_; - if (GET_PROTOCOL_MINOR(conn->protoVersion) >= 25) { + if (conn->protoVersion >= WorkerProto::Version{.number = {1, 25}}) { conn->to << WorkerProto::Op::AddToStore << name << caMethod.renderWithAlgo(hashAlgo); WorkerProto::write(*this, *conn, references); conn->to << repair; - if (conn->features.contains(WorkerProto::featureProvenance)) + if (conn->protoVersion.features.contains(WorkerProto::featureProvenance)) conn->to << (provenance ? provenance->to_json_str() : ""); // The dump source may invoke the store, so we need to make some room. @@ -378,10 +388,10 @@ ref RemoteStore::addCAToStore( } } conn.processStderr(); - } catch (SysError & e) { + } catch (SystemError & e) { /* Daemon closed while we were sending the path. Probably OOM or I/O error. */ - if (e.errNo == EPIPE) + if (e.is(std::errc::broken_pipe)) try { conn.processStderr(); } catch (EndOfFile & e) { @@ -439,14 +449,16 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, Repair WorkerProto::write(*this, *conn, info.deriver); conn->to << info.narHash.to_string(HashFormat::Base16, false); WorkerProto::write(*this, *conn, info.references); - conn->to << info.registrationTime << info.narSize << info.ultimate << info.sigs << renderContentAddress(info.ca); - if (conn->features.contains(WorkerProto::featureProvenance)) + conn->to << info.registrationTime << info.narSize << info.ultimate; + WorkerProto::write(*this, *conn, info.sigs); + conn->to << renderContentAddress(info.ca); + if (conn->protoVersion.features.contains(WorkerProto::featureProvenance)) conn->to << (info.provenance ? info.provenance->to_json_str() : ""); conn->to << repair << !checkSigs; - if (GET_PROTOCOL_MINOR(conn->protoVersion) >= 23) { + if (conn->protoVersion >= WorkerProto::Version{.number = {1, 23}}) { conn.withFramedSink([&](Sink & sink) { copyNAR(source, sink); }); - } else if (GET_PROTOCOL_MINOR(conn->protoVersion) >= 21) { + } else if (conn->protoVersion >= WorkerProto::Version{.number = {1, 21}}) { conn.processStderr(0, &source); } else { copyNAR(source, conn->to); @@ -457,6 +469,13 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, Repair void RemoteStore::addMultipleToStore( PathsSource && pathsToCopy, Activity & act, RepairFlag repair, CheckSigsFlag checkSigs) { + if (getConnection()->protoVersion < WorkerProto::Version{.number = {1, 32}}) { + Store::addMultipleToStore(std::move(pathsToCopy), act, repair, checkSigs); + return; + } + + auto conn(getConnection()); + // `addMultipleToStore` is single threaded size_t bytesExpected = 0; for (auto & [pathInfo, _] : pathsToCopy) { @@ -477,7 +496,9 @@ void RemoteStore::addMultipleToStore( *this, WorkerProto::WriteConn{ .to = sink, - .version = 16, + .version = conn->protoVersion.features.contains(WorkerProto::featureVersionedAddToStoreMultiple) + ? conn->protoVersion + : WorkerProto::Version{.number = {.major = 1, .minor = 16}}, }, pathInfo); pathSource->drainInto(sink); @@ -485,24 +506,15 @@ void RemoteStore::addMultipleToStore( } }); - addMultipleToStore(*source, repair, checkSigs); -} - -void RemoteStore::addMultipleToStore(Source & source, RepairFlag repair, CheckSigsFlag checkSigs) -{ - if (GET_PROTOCOL_MINOR(getConnection()->protoVersion) >= 32) { - auto conn(getConnection()); - conn->to << WorkerProto::Op::AddMultipleToStore << repair << !checkSigs; - conn.withFramedSink([&](Sink & sink) { source.drainInto(sink); }); - } else - Store::addMultipleToStore(source, repair, checkSigs); + conn->to << WorkerProto::Op::AddMultipleToStore << repair << !checkSigs; + conn.withFramedSink([&](Sink & sink) { source->drainInto(sink); }); } void RemoteStore::registerDrvOutput(const Realisation & info) { auto conn(getConnection()); conn->to << WorkerProto::Op::RegisterDrvOutput; - if (GET_PROTOCOL_MINOR(conn->protoVersion) < 31) { + if (conn->protoVersion.number < WorkerProto::Version::Number{1, 31}) { WorkerProto::write(*this, *conn, info.id); conn->to << std::string(info.outPath.to_string()); } else { @@ -517,7 +529,7 @@ void RemoteStore::queryRealisationUncached( try { auto conn(getConnection()); - if (GET_PROTOCOL_MINOR(conn->protoVersion) < 27) { + if (conn->protoVersion.number < WorkerProto::Version::Number{1, 27}) { warn("the daemon is too old to support content-addressing derivations, please upgrade it to 2.4"); return callback(nullptr); } @@ -527,7 +539,7 @@ void RemoteStore::queryRealisationUncached( conn.processStderr(); auto real = [&]() -> std::shared_ptr { - if (GET_PROTOCOL_MINOR(conn->protoVersion) < 31) { + if (conn->protoVersion.number < WorkerProto::Version::Number{1, 31}) { auto outPaths = WorkerProto::Serialise>::read(*this, *conn); if (outPaths.empty()) return nullptr; @@ -587,7 +599,7 @@ std::vector RemoteStore::buildPathsWithResults( std::optional conn_(getConnection()); auto & conn = *conn_; - if (GET_PROTOCOL_MINOR(conn->protoVersion) >= 34) { + if (conn->protoVersion >= WorkerProto::Version{.number = {1, 34}}) { conn->to << WorkerProto::Op::BuildPathsWithResults; WorkerProto::write(*this, *conn, paths); conn->to << buildMode; @@ -694,7 +706,7 @@ Roots RemoteStore::findRoots(bool censor) size_t count = readNum(conn->from); Roots result; while (count--) { - Path link = readString(conn->from); + std::string link = readString(conn->from); result[WorkerProto::Serialise::read(*this, *conn)].emplace(link); } return result; @@ -714,7 +726,7 @@ void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results) conn.processStderr(); - results.paths = readStrings(conn->from); + results.paths = readStrings(conn->from); results.bytesFreed = readLongLong(conn->from); readLongLong(conn->from); // obsolete @@ -737,12 +749,12 @@ bool RemoteStore::verifyStore(bool checkContents, RepairFlag repair) return readInt(conn->from); } -void RemoteStore::addSignatures(const StorePath & storePath, const StringSet & sigs) +void RemoteStore::addSignatures(const StorePath & storePath, const std::set & sigs) { auto conn(getConnection()); conn->to << WorkerProto::Op::AddSignatures; WorkerProto::write(*this, *conn, storePath); - conn->to << sigs; + WorkerProto::write(*this, *conn, sigs); conn.processStderr(); readInt(conn->from); } @@ -751,7 +763,7 @@ MissingPaths RemoteStore::queryMissing(const std::vector & targets) { { auto conn(getConnection()); - if (GET_PROTOCOL_MINOR(conn->protoVersion) < 19) + if (conn->protoVersion.number < WorkerProto::Version::Number{1, 19}) // Don't hold the connection handle in the fallback case // to prevent a deadlock. goto fallback; @@ -782,7 +794,7 @@ void RemoteStore::addBuildLog(const StorePath & drvPath, std::string_view log) std::vector RemoteStore::queryActiveBuilds() { auto conn(getConnection()); - if (!conn->features.count(WorkerProto::featureQueryActiveBuilds)) + if (!conn->protoVersion.features.count(WorkerProto::featureQueryActiveBuilds)) throw Error("remote store does not support querying active builds"); conn->to << WorkerProto::Op::QueryActiveBuilds; conn.processStderr(); @@ -803,7 +815,7 @@ void RemoteStore::connect() unsigned int RemoteStore::getProtocol() { auto conn(connections->get()); - return conn->protoVersion; + return conn->protoVersion.number.toWire(); } std::optional RemoteStore::isTrustedClient() @@ -817,6 +829,18 @@ void RemoteStore::flushBadConnections() connections->flushBad(); } +void RemoteStore::shutdownConnections() +{ + auto fds = connectionFds.lock(); + for (auto fd : *fds) { + /* Use shutdown() instead of close() to signal EOF to any blocking + reads/writes without actually closing the FD (which would cause + issues if the connection is still in use). This breaks circular + waits when the client disconnects during long-running operations. */ + ::shutdown(toSocket(fd), SHUT_RDWR); + } +} + void RemoteStore::narFromPath(const StorePath & path, Sink & sink) { auto conn(getConnection()); @@ -838,7 +862,7 @@ std::shared_ptr RemoteStore::getFSAccessor(const StorePath & pat return getRemoteFSAccessor(requireValidPath)->accessObject(path); } -void RemoteStore::ConnectionHandle::withFramedSink(std::function fun) +void RemoteStore::ConnectionHandle::withFramedSink(fun sendData) { (*this)->to.flush(); @@ -848,7 +872,7 @@ void RemoteStore::ConnectionHandle::withFramedSink(std::functionconfig->realStoreDir; } @@ -126,7 +126,7 @@ struct RestrictedStore : public virtual IndirectRootStore, public virtual GcStor void addTempRoot(const StorePath & path) override {} - void addIndirectRoot(const Path & path) override {} + void addIndirectRoot(const std::filesystem::path & path) override {} Roots findRoots(bool censor) override { @@ -135,7 +135,7 @@ struct RestrictedStore : public virtual IndirectRootStore, public virtual GcStor void collectGarbage(const GCOptions & options, GCResults & results) override {} - void addSignatures(const StorePath & storePath, const StringSet & sigs) override + void addSignatures(const StorePath & storePath, const std::set & sigs) override { unsupported("addSignatures"); } @@ -259,8 +259,7 @@ void RestrictedStore::buildPaths( const std::vector & paths, BuildMode buildMode, std::shared_ptr evalStore) { for (auto & result : buildPathsWithResults(paths, buildMode, evalStore)) - if (auto * failureP = result.tryGetFailure()) - failureP->rethrow(); + result.tryThrowBuildError(); } std::vector RestrictedStore::buildPathsWithResults( @@ -294,7 +293,7 @@ std::vector RestrictedStore::buildPathsWithResults( next->computeFSClosure(newPaths, closure); for (auto & path : closure) goal.addDependency(path); - for (auto & real : Realisation::closure(*next, newRealisations)) + for (auto & real : newRealisations) goal.addedDrvOutputs.insert(real.id); return results; diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index fea5e467f7b7..8c3ce56fcfc9 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -138,6 +138,20 @@ void S3BinaryCacheStore::upsertFile( if (auto storageClass = s3Config->storageClass.get()) { uploadHeaders.emplace_back("x-amz-storage-class", *storageClass); } + + { + HashSink hashSink(HashAlgorithm::MD5); + src.drainInto(hashSink); + auto [hash, gotLength] = hashSink.finish(); + /* Use this opportunity to check that the upload size matches what we expect. */ + if (gotLength != size) + throw Error("unexpected size for upload '%s', expected %d, got: %d", path, size, gotLength); + /* The Base64 encoded 128-bit MD5 digest of the message (without the headers) according to RFC 1864: + https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObject.html */ + uploadHeaders.push_back({"Content-MD5", hash.to_string(HashFormat::Base64, /*includeAlgo=*/false)}); + src.restart(); /* Seek to the beginning. */ + } + if (s3Config->multipartUpload && size > s3Config->multipartThreshold) { uploadMultipart(path, src, size, mimeType, std::move(uploadHeaders)); } else { @@ -148,7 +162,9 @@ void S3BinaryCacheStore::upsertFile( try { if (auto compressionMethod = getCompressionMethod(path)) { CompressedSource compressed(source, *compressionMethod); - Headers headers = {{"Content-Encoding", *compressionMethod}}; + /* TODO: Validate that this is a valid content encoding. We probably shouldn't set non-standard values here. + */ + Headers headers = {{"Content-Encoding", showCompressionAlgo(*compressionMethod)}}; doUpload(compressed, compressed.size(), std::move(headers)); } else { doUpload(source, sizeHint, std::nullopt); @@ -400,10 +416,9 @@ StringSet S3BinaryCacheStoreConfig::uriSchemes() return {"s3"}; } -S3BinaryCacheStoreConfig::S3BinaryCacheStoreConfig( - std::string_view scheme, std::string_view _cacheUri, const Params & params) +S3BinaryCacheStoreConfig::S3BinaryCacheStoreConfig(ParsedURL cacheUri_, const Params & params) : StoreConfig(params) - , HttpBinaryCacheStoreConfig(scheme, _cacheUri, params) + , HttpBinaryCacheStoreConfig(std::move(cacheUri_), params) { assert(cacheUri.query.empty()); assert(cacheUri.scheme == "s3"); @@ -439,6 +454,12 @@ S3BinaryCacheStoreConfig::S3BinaryCacheStoreConfig( } } +S3BinaryCacheStoreConfig::S3BinaryCacheStoreConfig(std::string_view bucketName, const Params & params) + : S3BinaryCacheStoreConfig( + ParsedURL{.scheme = "s3", .authority = ParsedURL::Authority{.host = std::string(bucketName)}}, params) +{ +} + std::string S3BinaryCacheStoreConfig::getHumanReadableURI() const { auto reference = getReference(); diff --git a/src/libstore/serve-protocol-connection.cc b/src/libstore/serve-protocol-connection.cc index baa3bf0ce719..5e5e0ff5eb98 100644 --- a/src/libstore/serve-protocol-connection.cc +++ b/src/libstore/serve-protocol-connection.cc @@ -8,14 +8,14 @@ namespace nix { ServeProto::Version ServeProto::BasicClientConnection::handshake( BufferedSink & to, Source & from, ServeProto::Version localVersion, std::string_view host) { - to << SERVE_MAGIC_1 << localVersion; + to << SERVE_MAGIC_1 << localVersion.toWire(); to.flush(); unsigned int magic = readInt(from); if (magic != SERVE_MAGIC_2) throw Error("'nix-store --serve' protocol mismatch from '%s'", host); - auto remoteVersion = readInt(from); - if (GET_PROTOCOL_MAJOR(remoteVersion) != 0x200 || GET_PROTOCOL_MINOR(remoteVersion) < 5) + auto remoteVersion = ServeProto::Version::fromWire(readInt(from)); + if (remoteVersion.major != 2 || remoteVersion < ServeProto::Version{2, 5}) throw Error("unsupported 'nix-store --serve' protocol version on '%s'", host); return std::min(remoteVersion, localVersion); } @@ -26,9 +26,9 @@ ServeProto::BasicServerConnection::handshake(BufferedSink & to, Source & from, S unsigned int magic = readInt(from); if (magic != SERVE_MAGIC_1) throw Error("protocol mismatch"); - to << SERVE_MAGIC_2 << localVersion; + to << SERVE_MAGIC_2 << localVersion.toWire(); to.flush(); - auto remoteVersion = readInt(from); + auto remoteVersion = ServeProto::Version::fromWire(readInt(from)); return std::min(remoteVersion, localVersion); } @@ -85,18 +85,18 @@ BuildResult ServeProto::BasicClientConnection::getBuildDerivationResponse(const } void ServeProto::BasicClientConnection::narFromPath( - const StoreDirConfig & store, const StorePath & path, std::function fun) + const StoreDirConfig & store, const StorePath & path, fun receiveNar) { to << ServeProto::Command::DumpStorePath << store.printStorePath(path); to.flush(); - fun(from); + receiveNar(from); } -void ServeProto::BasicClientConnection::importPaths(const StoreDirConfig & store, std::function fun) +void ServeProto::BasicClientConnection::importPaths(const StoreDirConfig & store, fun sendPaths) { to << ServeProto::Command::ImportPaths; - fun(to); + sendPaths(to); to.flush(); if (readInt(from) != 1) diff --git a/src/libstore/serve-protocol.cc b/src/libstore/serve-protocol.cc index e6c9eabd22c9..ec7b85ed8826 100644 --- a/src/libstore/serve-protocol.cc +++ b/src/libstore/serve-protocol.cc @@ -2,6 +2,7 @@ #include "nix/store/path-with-outputs.hh" #include "nix/store/store-api.hh" #include "nix/store/build-result.hh" +#include "nix/store/common-protocol.hh" #include "nix/store/serve-protocol.hh" #include "nix/store/serve-protocol-impl.hh" #include "nix/util/archive.hh" @@ -15,30 +16,41 @@ namespace nix { BuildResult ServeProto::Serialise::read(const StoreDirConfig & store, ServeProto::ReadConn conn) { - BuildResult status; + BuildResult res; BuildResult::Success success; - BuildResult::Failure failure; - auto rawStatus = readInt(conn.from); - conn.from >> failure.errorMsg; + // Temp variables for failure fields since BuildError uses methods + std::string errorMsg; + bool isNonDeterministic = false; - if (GET_PROTOCOL_MINOR(conn.version) >= 3) - conn.from >> status.timesBuilt >> failure.isNonDeterministic >> status.startTime >> status.stopTime; - if (GET_PROTOCOL_MINOR(conn.version) >= 6) { + auto status = ServeProto::Serialise::read(store, {conn.from}); + conn.from >> errorMsg; + + if (conn.version >= ServeProto::Version{2, 3}) + conn.from >> res.timesBuilt >> isNonDeterministic >> res.startTime >> res.stopTime; + if (conn.version >= ServeProto::Version{2, 6}) { auto builtOutputs = ServeProto::Serialise::read(store, conn); for (auto && [output, realisation] : builtOutputs) success.builtOutputs.insert_or_assign(std::move(output.outputName), std::move(realisation)); } - if (BuildResult::Success::statusIs(rawStatus)) { - success.status = static_cast(rawStatus); - status.inner = std::move(success); - } else { - failure.status = static_cast(rawStatus); - status.inner = std::move(failure); - } + res.inner = std::visit( + overloaded{ + [&](BuildResult::Success::Status s) -> decltype(res.inner) { + success.status = s; + return std::move(success); + }, + [&](BuildResult::Failure::Status s) -> decltype(res.inner) { + return BuildResult::Failure{{ + .status = s, + .msg = HintFmt(std::move(errorMsg)), + .isNonDeterministic = isNonDeterministic, + }}; + }, + }, + status); - return status; + return res; } void ServeProto::Serialise::write( @@ -51,9 +63,9 @@ void ServeProto::Serialise::write( default value for the fields that don't exist in that case. */ auto common = [&](std::string_view errorMsg, bool isNonDeterministic, const auto & builtOutputs) { conn.to << errorMsg; - if (GET_PROTOCOL_MINOR(conn.version) >= 3) + if (conn.version >= ServeProto::Version{2, 3}) conn.to << res.timesBuilt << isNonDeterministic << res.startTime << res.stopTime; - if (GET_PROTOCOL_MINOR(conn.version) >= 6) { + if (conn.version >= ServeProto::Version{2, 6}) { DrvOutputs builtOutputsFullKey; for (auto & [output, realisation] : builtOutputs) builtOutputsFullKey.insert_or_assign(realisation.id, realisation); @@ -63,11 +75,11 @@ void ServeProto::Serialise::write( std::visit( overloaded{ [&](const BuildResult::Failure & failure) { - conn.to << failure.status; - common(failure.errorMsg, failure.isNonDeterministic, decltype(BuildResult::Success::builtOutputs){}); + ServeProto::write(store, {conn.to}, BuildResultStatus{failure.status}); + common(failure.message(), failure.isNonDeterministic, decltype(BuildResult::Success::builtOutputs){}); }, [&](const BuildResult::Success & success) { - conn.to << success.status; + ServeProto::write(store, {conn.to}, BuildResultStatus{success.status}); common(/*errorMsg=*/"", /*isNonDeterministic=*/false, success.builtOutputs); }, }, @@ -88,12 +100,12 @@ UnkeyedValidPathInfo ServeProto::Serialise::read(const Sto readLongLong(conn.from); // download size, unused info.narSize = readLongLong(conn.from); - if (GET_PROTOCOL_MINOR(conn.version) >= 4) { + if (conn.version >= ServeProto::Version{2, 4}) { auto s = readString(conn.from); if (!s.empty()) info.narHash = Hash::parseAnyPrefixed(s); info.ca = ContentAddress::parseOpt(readString(conn.from)); - info.sigs = readStrings(conn.from); + info.sigs = ServeProto::Serialise>::read(store, conn); } return info; @@ -108,8 +120,10 @@ void ServeProto::Serialise::write( // !!! Maybe we want compression? conn.to << info.narSize // downloadSize, lie a little << info.narSize; - if (GET_PROTOCOL_MINOR(conn.version) >= 4) - conn.to << info.narHash.to_string(HashFormat::Nix32, true) << renderContentAddress(info.ca) << info.sigs; + if (conn.version >= ServeProto::Version{2, 4}) { + conn.to << info.narHash.to_string(HashFormat::Nix32, true) << renderContentAddress(info.ca); + ServeProto::write(store, conn, info.sigs); + } } ServeProto::BuildOptions @@ -118,13 +132,13 @@ ServeProto::Serialise::read(const StoreDirConfig & sto BuildOptions options; options.maxSilentTime = readInt(conn.from); options.buildTimeout = readInt(conn.from); - if (GET_PROTOCOL_MINOR(conn.version) >= 2) + if (conn.version >= ServeProto::Version{2, 2}) options.maxLogSize = readNum(conn.from); - if (GET_PROTOCOL_MINOR(conn.version) >= 3) { + if (conn.version >= ServeProto::Version{2, 3}) { options.nrRepeats = readInt(conn.from); options.enforceDeterminism = readInt(conn.from); } - if (GET_PROTOCOL_MINOR(conn.version) >= 7) { + if (conn.version >= ServeProto::Version{2, 7}) { options.keepFailed = (bool) readInt(conn.from); } return options; @@ -134,12 +148,12 @@ void ServeProto::Serialise::write( const StoreDirConfig & store, WriteConn conn, const ServeProto::BuildOptions & options) { conn.to << options.maxSilentTime << options.buildTimeout; - if (GET_PROTOCOL_MINOR(conn.version) >= 2) + if (conn.version >= ServeProto::Version{2, 2}) conn.to << options.maxLogSize; - if (GET_PROTOCOL_MINOR(conn.version) >= 3) + if (conn.version >= ServeProto::Version{2, 3}) conn.to << options.nrRepeats << options.enforceDeterminism; - if (GET_PROTOCOL_MINOR(conn.version) >= 7) { + if (conn.version >= ServeProto::Version{2, 7}) { conn.to << ((int) options.keepFailed); } } diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc index 5f0b3ce51a13..5f6119a427d0 100644 --- a/src/libstore/sqlite.cc +++ b/src/libstore/sqlite.cc @@ -17,7 +17,7 @@ namespace nix { SQLiteError::SQLiteError( const char * path, const char * errMsg, int errNo, int extendedErrNo, int offset, HintFmt && hf) - : Error("") + : CloneableError("") , path(path) , errMsg(errMsg) , errNo(errNo) @@ -62,7 +62,7 @@ static void traceSQL(void * x, const char * sql) notice("SQL<[%1%]>", sql); }; -SQLite::SQLite(const std::filesystem::path & path, SQLiteOpenMode mode) +SQLite::SQLite(const std::filesystem::path & path, Settings && settings) { // Work around a ZFS issue where SQLite's truncate() call on // db.sqlite-shm can randomly take up to a few seconds. See @@ -77,9 +77,9 @@ SQLite::SQLite(const std::filesystem::path & path, SQLiteOpenMode mode) if (fd) { struct statfs fs; if (fstatfs(fd.get(), &fs)) - throw SysError("statfs() on '%s'", shmFile); + throw SysError("statfs() on %s", PathFmt(shmFile)); if (fs.f_type == /* ZFS_SUPER_MAGIC */ 801189825 && fdatasync(fd.get()) != 0) - throw SysError("fsync() on '%s'", shmFile); + throw SysError("fsync() on %s", PathFmt(shmFile)); } } catch (...) { throw; @@ -89,16 +89,16 @@ SQLite::SQLite(const std::filesystem::path & path, SQLiteOpenMode mode) // useSQLiteWAL also indicates what virtual file system we need. Using // `unix-dotfile` is needed on NFS file systems and on Windows' Subsystem // for Linux (WSL) where useSQLiteWAL should be false by default. - const char * vfs = settings.useSQLiteWAL ? 0 : "unix-dotfile"; - bool immutable = mode == SQLiteOpenMode::Immutable; + const char * vfs = settings.useWAL ? 0 : "unix-dotfile"; + bool immutable = settings.mode == SQLiteOpenMode::Immutable; int flags = immutable ? SQLITE_OPEN_READONLY : SQLITE_OPEN_READWRITE; - if (mode == SQLiteOpenMode::Normal) + if (settings.mode == SQLiteOpenMode::Normal) flags |= SQLITE_OPEN_CREATE; auto uri = "file:" + percentEncode(path.string()) + "?immutable=" + (immutable ? "1" : "0"); int ret = sqlite3_open_v2(uri.c_str(), &db, SQLITE_OPEN_URI | flags, vfs); if (ret != SQLITE_OK) { const char * err = sqlite3_errstr(ret); - throw Error("cannot open SQLite database '%s': %s", path, err); + throw Error("cannot open SQLite database %s: %s", PathFmt(path), err); } if (sqlite3_busy_timeout(db, 60 * 60 * 1000) != SQLITE_OK) @@ -278,7 +278,7 @@ SQLiteTxn::~SQLiteTxn() void handleSQLiteBusy(const SQLiteBusy & e, time_t & nextWarning) { - time_t now = time(0); + time_t now = time(nullptr); if (now > nextWarning) { nextWarning = now + 10; logWarning({.msg = e.info().msg}); diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc index 1d69ca640f2a..d0da581fc0f0 100644 --- a/src/libstore/ssh-store.cc +++ b/src/libstore/ssh-store.cc @@ -11,10 +11,10 @@ namespace nix { -SSHStoreConfig::SSHStoreConfig(std::string_view scheme, std::string_view authority, const Params & params) +SSHStoreConfig::SSHStoreConfig(const ParsedURL::Authority & authority, const Params & params) : Store::Config{params} , RemoteStore::Config{params} - , CommonSSHStoreConfig{scheme, authority, params} + , CommonSSHStoreConfig{authority, params} { } @@ -102,11 +102,11 @@ MountedSSHStoreConfig::MountedSSHStoreConfig(StringMap params) { } -MountedSSHStoreConfig::MountedSSHStoreConfig(std::string_view scheme, std::string_view host, StringMap params) +MountedSSHStoreConfig::MountedSSHStoreConfig(const ParsedURL::Authority & authority, StringMap params) : StoreConfig(params) , RemoteStoreConfig(params) - , CommonSSHStoreConfig(scheme, host, params) - , SSHStoreConfig(scheme, host, params) + , CommonSSHStoreConfig(authority, params) + , SSHStoreConfig(authority, params) , LocalFSStoreConfig(params) { } @@ -182,12 +182,12 @@ struct MountedSSHStore : virtual SSHStore, virtual LocalFSStore * privilege escalation / symlinks in directories owned by the * originating requester that they cannot delete. */ - Path addPermRoot(const StorePath & path, const Path & gcRoot) override + std::filesystem::path addPermRoot(const StorePath & path, const std::filesystem::path & gcRoot) override { auto conn(getConnection()); conn->to << WorkerProto::Op::AddPermRoot; WorkerProto::write(*this, *conn, path); - WorkerProto::write(*this, *conn, gcRoot); + WorkerProto::write(*this, *conn, gcRoot.string()); conn.processStderr(); return readString(conn->from); } @@ -213,7 +213,7 @@ ref SSHStore::openConnection() command.push_back(config->remoteStore.get()); } command.insert(command.end(), extraRemoteProgramArgs.begin(), extraRemoteProgramArgs.end()); - conn->sshConn = master.startCommand(std::move(command)); + conn->sshConn = master.startCommand(toOsStrings(std::move(command))); conn->to = FdSink(conn->sshConn->in.get()); conn->from = FdSource(conn->sshConn->out.get()); return conn; diff --git a/src/libstore/ssh.cc b/src/libstore/ssh.cc index 1a99083669cf..849fcff2fdfc 100644 --- a/src/libstore/ssh.cc +++ b/src/libstore/ssh.cc @@ -2,6 +2,7 @@ #include "nix/util/finally.hh" #include "nix/util/current-process.hh" #include "nix/util/environment-variables.hh" +#include "nix/util/os-string.hh" #include "nix/util/util.hh" #include "nix/util/exec.hh" #include "nix/util/base-n.hh" @@ -18,11 +19,11 @@ static std::string parsePublicHostKey(std::string_view host, std::string_view ss } } -class InvalidSSHAuthority : public Error +class InvalidSSHAuthority final : public CloneableError { public: InvalidSSHAuthority(const ParsedURL::Authority & authority, std::string_view reason) - : Error("invalid SSH authority: '%s': %s", authority.to_string(), reason) + : CloneableError("invalid SSH authority: '%s': %s", authority.to_string(), reason) { } }; @@ -51,12 +52,12 @@ static void checkValidAuthority(const ParsedURL::Authority & authority) } } -Strings getNixSshOpts() +OsStrings getNixSshOpts() { std::string sshOpts = getEnv("NIX_SSHOPTS").value_or(""); try { - return shellSplitString(sshOpts); + return toOsStrings(shellSplitString(sshOpts)); } catch (Error & e) { e.addTrace({}, "while splitting NIX_SSHOPTS '%s'", sshOpts); throw; @@ -65,7 +66,7 @@ Strings getNixSshOpts() SSHMaster::SSHMaster( const ParsedURL::Authority & authority, - std::string_view keyFile, + std::filesystem::path keyFile, std::string_view sshPublicHostKey, bool useMaster, bool compress, @@ -79,7 +80,7 @@ SSHMaster::SSHMaster( return std::move(oss).str(); }()) , fakeSSH(authority.to_string() == "localhost") - , keyFile(keyFile) + , keyFile(std::move(keyFile)) , sshPublicHostKey(parsePublicHostKey(authority.host, sshPublicHostKey)) , useMaster(useMaster && !fakeSSH) , compress(compress) @@ -89,38 +90,38 @@ SSHMaster::SSHMaster( checkValidAuthority(authority); } -void SSHMaster::addCommonSSHOpts(Strings & args) +void SSHMaster::addCommonSSHOpts(OsStrings & args) { auto sshArgs = getNixSshOpts(); args.insert(args.end(), sshArgs.begin(), sshArgs.end()); if (!keyFile.empty()) - args.insert(args.end(), {"-i", keyFile}); + args.insert(args.end(), {OS_STR("-i"), keyFile.native()}); if (!sshPublicHostKey.empty()) { std::filesystem::path fileName = tmpDir->path() / "host-key"; - writeFile(fileName.string(), authority.host + " " + sshPublicHostKey + "\n"); - args.insert(args.end(), {"-oUserKnownHostsFile=" + fileName.string()}); + writeFile(fileName, authority.host + " " + sshPublicHostKey + "\n"); + args.insert(args.end(), {OS_STR("-oUserKnownHostsFile=") + fileName.native()}); } if (compress) - args.push_back("-C"); + args.push_back(OS_STR("-C")); if (authority.port) - args.push_back(fmt("-p%d", *authority.port)); + args.push_back(string_to_os_string(fmt("-p%d", *authority.port))); // We use this to make ssh signal back to us that the connection is established. // It really does run locally; see createSSHEnv which sets up SHELL to make // it launch more reliably. The local command runs synchronously, so presumably // the remote session won't be garbled if the local command is slow. - args.push_back("-oPermitLocalCommand=yes"); - args.push_back("-oLocalCommand=echo started"); + args.push_back(OS_STR("-oPermitLocalCommand=yes")); + args.push_back(OS_STR("-oLocalCommand=echo started")); } bool SSHMaster::isMasterRunning() { - Strings args = {"-O", "check", hostnameAndUser}; + OsStrings args = {OS_STR("-O"), OS_STR("check"), string_to_os_string(hostnameAndUser)}; addCommonSSHOpts(args); - auto res = runProgram(RunOptions{.program = "ssh", .args = args, .mergeStderrToStdout = true}); + auto res = runProgram(RunOptions{.program = "ssh", .args = std::move(args), .mergeStderrToStdout = true}); return res.first == 0; } @@ -145,12 +146,12 @@ Strings createSSHEnv() return r; } -std::unique_ptr SSHMaster::startCommand(Strings && command, Strings && extraSshArgs) +std::unique_ptr SSHMaster::startCommand(OsStrings && command, OsStrings && extraSshArgs) { #ifdef _WIN32 // TODO re-enable on Windows, once we can start processes. throw UnimplementedError("cannot yet SSH on windows because spawning processes is not yet implemented"); #else - Path socketPath = startMaster(); + std::filesystem::path socketPath = startMaster(); Pipe in, out; in.create(); @@ -179,13 +180,13 @@ std::unique_ptr SSHMaster::startCommand(Strings && comman if (logFD != -1 && dup2(logFD, STDERR_FILENO) == -1) throw SysError("duping over stderr"); - Strings args; + OsStrings args; if (!fakeSSH) { args = {"ssh", hostnameAndUser.c_str(), "-x"}; addCommonSSHOpts(args); - if (socketPath != "") - args.insert(args.end(), {"-S", socketPath}); + if (!socketPath.empty()) + args.insert(args.end(), {"-S", socketPath.string()}); if (verbosity >= lvlChatty) args.push_back("-v"); args.splice(args.end(), std::move(extraSshArgs)); @@ -228,17 +229,17 @@ std::unique_ptr SSHMaster::startCommand(Strings && comman #ifndef _WIN32 // TODO re-enable on Windows, once we can start processes. -Path SSHMaster::startMaster() +std::filesystem::path SSHMaster::startMaster() { if (!useMaster) - return ""; + return {}; auto state(state_.lock()); if (state->sshMaster != INVALID_DESCRIPTOR) return state->socketPath; - state->socketPath = (Path) *tmpDir + "/ssh.sock"; + state->socketPath = tmpDir->path() / "ssh.sock"; Pipe out; out.create(); @@ -260,7 +261,7 @@ Path SSHMaster::startMaster() if (dup2(out.writeSide.get(), STDOUT_FILENO) == -1) throw SysError("duping over stdout"); - Strings args = {"ssh", hostnameAndUser.c_str(), "-M", "-N", "-S", state->socketPath}; + OsStrings args = {"ssh", hostnameAndUser.c_str(), "-M", "-N", "-S", state->socketPath.string()}; if (verbosity >= lvlChatty) args.push_back("-v"); addCommonSSHOpts(args); diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index b3ec0ccbb01a..57952b1dfd4a 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -14,12 +14,13 @@ #include "nix/util/callback.hh" #include "nix/util/git.hh" #include "nix/util/posix-source-accessor.hh" -// FIXME this should not be here, see TODO below on -// `addMultipleToStore`. -#include "nix/store/worker-protocol.hh" #include "nix/util/signals.hh" +#include "nix/util/environment-variables.hh" +#include "nix/util/file-system.hh" #include "nix/store/provenance.hh" +#include "store-config-private.hh" + #include #include @@ -27,9 +28,35 @@ namespace nix { -Path StoreConfigBase::getDefaultNixStoreDir() +std::string StoreConfigBase::getDefaultNixStoreDir() +{ + return +#ifndef _WIN32 + canonPath +#endif + (getEnvNonEmpty("NIX_STORE_DIR").value_or(getEnvNonEmpty("NIX_STORE").value_or(NIX_STORE_DIR))) +#ifndef _WIN32 + .string() +#endif + ; +} + +StoreDirSetting::StoreDirSetting( + Config * options, + const std::string & def, + const std::string & name, + const std::string & description, + const StringSet & aliases) + : BaseSetting(def, true, name, description, aliases) +{ + options->addSetting(this); +} + +std::string StoreDirSetting::parse(const std::string & str) const { - return settings.nixStore; + if (str.empty()) + throw UsageError("setting '%s' is a path and paths cannot be empty", name); + return canonPath(str).string(); } StoreConfig::StoreConfig(const Params & params) @@ -38,31 +65,31 @@ StoreConfig::StoreConfig(const Params & params) { } -bool StoreDirConfig::isInStore(PathView path) const +bool StoreDirConfig::isInStore(std::string_view path) const { return isInDir(path, storeDir); } -std::pair StoreDirConfig::toStorePath(PathView path) const +std::pair StoreDirConfig::toStorePath(std::string_view path) const { if (!isInStore(path)) throw Error("path '%1%' is not in the Nix store", path); auto slash = path.find('/', storeDir.size() + 1); - if (slash == Path::npos) - return {parseStorePath(path), ""}; + if (slash == std::string::npos) + return {parseStorePath(path), CanonPath::root}; else - return {parseStorePath(path.substr(0, slash)), (Path) path.substr(slash)}; + return {parseStorePath(path.substr(0, slash)), CanonPath{path.substr(slash)}}; } -Path Store::followLinksToStore(std::string_view _path) const +std::filesystem::path Store::followLinksToStore(std::string_view _path) const { - Path path = absPath(std::string(_path)); + auto path = absPath(std::string(_path)); // Limit symlink follows to prevent infinite loops unsigned int followCount = 0; const unsigned int maxFollow = 1024; - while (!isInStore(path)) { + while (!isInStore(path.string())) { if (!std::filesystem::is_symlink(path)) break; @@ -70,17 +97,18 @@ Path Store::followLinksToStore(std::string_view _path) const throw Error("too many symbolic links encountered while resolving '%s'", _path); auto target = readLink(path); - path = absPath(target, dirOf(path)); + auto parentPath = path.parent_path(); + path = absPath(target, &parentPath); } - if (!isInStore(path)) - throw BadStorePath("path '%1%' is not in the Nix store", path); + if (!isInStore(path.string())) + throw BadStorePath("path %1% is not in the Nix store", PathFmt(path)); return path; } StorePath Store::followLinksToStorePath(std::string_view path) const { - return toStorePath(followLinksToStore(path)).first; + return toStorePath(followLinksToStore(path).string()).first; } StorePath Store::addToStore( @@ -180,7 +208,7 @@ void Store::addMultipleToStore(PathsSource && pathsToCopy, Activity & act, Repai addToStore(info, *source, repair, checkSigs); } catch (Error & e) { nrFailed++; - if (!settings.keepGoing) + if (!settings.getWorkerSettings().keepGoing) throw e; printMsg(lvlError, "could not copy %s: %s", printStorePath(path), e.what()); showProgress(); @@ -193,24 +221,6 @@ void Store::addMultipleToStore(PathsSource && pathsToCopy, Activity & act, Repai }); } -void Store::addMultipleToStore(Source & source, RepairFlag repair, CheckSigsFlag checkSigs) -{ - auto expected = readNum(source); - for (uint64_t i = 0; i < expected; ++i) { - // FIXME we should not be using the worker protocol here, let - // alone the worker protocol with a hard-coded version! - auto info = WorkerProto::Serialise::read( - *this, - WorkerProto::ReadConn{ - .from = source, - .version = 16, - }); - info.ultimate = false; - EnsureRead wrapper{source, info.narSize}; - addToStore(info, wrapper, repair, checkSigs); - } -} - /* The aim of this function is to compute in one pass the correct ValidPathInfo for the files that we are trying to add to the store. To accomplish that in one @@ -339,10 +349,15 @@ StoreReference StoreConfig::getReference() const return {.variant = StoreReference::Auto{}}; } -bool Store::PathInfoCacheValue::isKnownNow() +bool StoreConfig::getReadOnly() const { - std::chrono::duration ttl = didExist() ? std::chrono::seconds(settings.ttlPositiveNarInfoCache) - : std::chrono::seconds(settings.ttlNegativeNarInfoCache); + return settings.readOnlyMode; +} + +bool Store::PathInfoCacheValue::isKnownNow(const NarInfoDiskCacheSettings & settings) +{ + std::chrono::duration ttl = + didExist() ? std::chrono::seconds(settings.ttlPositive) : std::chrono::seconds(settings.ttlNegative); return std::chrono::steady_clock::now() < time_point + ttl; } @@ -413,7 +428,7 @@ StorePathSet Store::queryDerivationOutputs(const StorePath & path) void Store::querySubstitutablePathInfos(const StorePathCAMap & paths, SubstitutablePathInfos & infos) { - if (!settings.useSubstitutes) + if (!settings.getWorkerSettings().useSubstitutes) return; for (auto & path : paths) { @@ -469,7 +484,7 @@ void Store::querySubstitutablePathInfos(const StorePathCAMap & paths, Substituta } } if (lastStoresException.has_value()) { - if (!settings.tryFallback) { + if (!settings.getWorkerSettings().tryFallback) { throw *lastStoresException; } else logError(lastStoresException->info()); @@ -477,10 +492,44 @@ void Store::querySubstitutablePathInfos(const StorePathCAMap & paths, Substituta } } +StorePathSet Store::querySubstitutablePaths(const StorePathSet & paths) +{ + if (!settings.getWorkerSettings().useSubstitutes) + return StorePathSet(); + + StorePathSet remaining; + for (auto & i : paths) + remaining.insert(i); + + StorePathSet res; + + for (auto & sub : getDefaultSubstituters()) { + if (remaining.empty()) + break; + if (sub->storeDir != storeDir) + continue; + if (!sub->config.wantMassQuery) + continue; + + auto valid = sub->queryValidPaths(remaining); + + StorePathSet remaining2; + for (auto & path : remaining) + if (valid.count(path)) + res.insert(path); + else + remaining2.insert(path); + + std::swap(remaining, remaining2); + } + + return res; +} + bool Store::isValidPath(const StorePath & storePath) { auto res = pathInfoCache->lock()->get(storePath); - if (res && res->isKnownNow()) { + if (res && res->isKnownNow(settings.getNarInfoDiskCacheSettings())) { stats.narInfoReadAverted++; return res->didExist(); } @@ -563,7 +612,7 @@ std::optional> Store::queryPathInfoFromClie auto hashPart = std::string(storePath.hashPart()); auto res = pathInfoCache->lock()->get(storePath); - if (res && res->isKnownNow()) { + if (res && res->isKnownNow(settings.getNarInfoDiskCacheSettings())) { stats.narInfoReadAverted++; if (res->didExist()) return std::make_optional(res->value); @@ -955,36 +1004,21 @@ std::map copyPaths( SubstituteFlag substitute) { StorePathSet storePaths; - std::set toplevelRealisations; + std::vector realisations; for (auto & path : paths) { storePaths.insert(path.path()); if (auto * realisation = std::get_if(&path.raw)) { experimentalFeatureSettings.require(Xp::CaDerivations); - toplevelRealisations.insert(*realisation); + realisations.push_back(realisation); } } auto pathsMap = copyPaths(srcStore, dstStore, storePaths, repair, checkSigs, substitute); try { - // Copy the realisation closure - processGraph( - Realisation::closure(srcStore, toplevelRealisations), - [&](const Realisation & current) -> std::set { - std::set children; - for (const auto & [drvOutput, _] : current.dependentRealisations) { - auto currentChild = srcStore.queryRealisation(drvOutput); - if (!currentChild) - throw Error( - "incomplete realisation closure: '%s' is a " - "dependency of '%s' but isn't registered", - drvOutput.to_string(), - current.id.to_string()); - children.insert({*currentChild, drvOutput}); - } - return children; - }, - [&](const Realisation & current) -> void { dstStore.registerDrvOutput(current, checkSigs); }); + // Copy the realisations. TODO batch this + for (const auto * realisation : realisations) + dstStore.registerDrvOutput(*realisation, checkSigs); } catch (MissingExperimentalFeature & e) { // Don't fail if the remote doesn't support CA derivations is it might // not be within our control to change that, and we might still want @@ -1096,8 +1130,19 @@ void copyClosure( if (&srcStore == &dstStore) return; - RealisedPath::Set closure; - RealisedPath::closure(srcStore, paths, closure); + StorePathSet closure0; + for (auto & path : paths) { + if (auto * opaquePath = std::get_if(&path.raw)) { + closure0.insert(opaquePath->path); + } + } + + StorePathSet closure1; + srcStore.computeFSClosure(closure0, closure1); + + RealisedPath::Set closure = paths; + for (auto && path : closure1) + closure.insert({std::move(path)}); copyPaths(srcStore, dstStore, closure, repair, checkSigs, substitute); } @@ -1156,27 +1201,6 @@ decodeValidPathInfo(const Store & store, std::istream & str, std::optional(std::move(info)); } -std::string StoreDirConfig::showPaths(const StorePathSet & paths) const -{ - std::string s; - for (auto & i : paths) { - if (s.size() != 0) - s += ", "; - s += "'" + printStorePath(i) + "'"; - } - return s; -} - -std::string showPaths(const std::set paths) -{ - return concatStringsSep(", ", quoteFSPaths(paths)); -} - -std::string showPaths(const PathSet & paths) -{ - return concatStringsSep(", ", quoteStrings(paths)); -} - Derivation Store::derivationFromPath(const StorePath & drvPath) { ensurePath(drvPath); @@ -1186,10 +1210,16 @@ Derivation Store::derivationFromPath(const StorePath & drvPath) static Derivation readDerivationCommon(Store & store, const StorePath & drvPath, bool requireValidPath) { auto accessor = store.requireStoreObjectAccessor(drvPath, requireValidPath); + auto contents = accessor->readFile(CanonPath::root); + try { - return parseDerivation(store, accessor->readFile(CanonPath::root), Derivation::nameFromPath(drvPath)); + /* Special case for an empty file to show the user a better message */ + if (contents.empty()) + throw FormatError("file is empty (possible filesystem corruption)"); + + return parseDerivation(store, std::move(contents), Derivation::nameFromPath(drvPath)); } catch (FormatError & e) { - throw Error("error parsing derivation '%s': %s", store.printStorePath(drvPath), e.msg()); + throw Error("error parsing derivation '%s': %s", store.printStorePath(drvPath), e.message()); } } @@ -1216,7 +1246,7 @@ std::optional Store::getBuildDerivationPath(const StorePath & path) // resolved derivation, so we need to get it first auto resolvedDrv = drv.tryResolve(*this); if (resolvedDrv) - return ::nix::writeDerivation(*this, *resolvedDrv, NoRepair, true); + return nix::computeStorePath(*this, Derivation{*resolvedDrv}); } return path; diff --git a/src/libstore/store-dir-config.cc b/src/libstore/store-dir-config.cc index 8c756ff58193..61f82029d79f 100644 --- a/src/libstore/store-dir-config.cc +++ b/src/libstore/store-dir-config.cc @@ -15,14 +15,14 @@ StorePath StoreDirConfig::parseStorePath(std::string_view path) const // Windows <-> Unix ssh-ing). auto p = #ifdef _WIN32 - path + std::filesystem::path(path) #else canonPath(std::string(path)) #endif ; - if (dirOf(p) != storeDir) - throw BadStorePath("path '%s' is not in the Nix store", p); - return StorePath(baseNameOf(p)); + if (p.parent_path() != storeDir) + throw BadStorePath("path %s is not in the Nix store", PathFmt(p)); + return StorePath(p.filename().string()); } std::optional StoreDirConfig::maybeParseStorePath(std::string_view path) const @@ -39,7 +39,7 @@ bool StoreDirConfig::isStorePath(std::string_view path) const return (bool) maybeParseStorePath(path); } -StorePathSet StoreDirConfig::parseStorePathSet(const PathSet & paths) const +StorePathSet StoreDirConfig::parseStorePathSet(const StringSet & paths) const { StorePathSet res; for (auto & i : paths) @@ -52,9 +52,9 @@ std::string StoreDirConfig::printStorePath(const StorePath & path) const return (storeDir + "/").append(path.to_string()); } -PathSet StoreDirConfig::printStorePathSet(const StorePathSet & paths) const +StringSet StoreDirConfig::printStorePathSet(const StorePathSet & paths) const { - PathSet res; + StringSet res; for (auto & i : paths) res.insert(printStorePath(i)); return res; diff --git a/src/libstore/store-reference.cc b/src/libstore/store-reference.cc index 01e197be76dc..8f72a3d33081 100644 --- a/src/libstore/store-reference.cc +++ b/src/libstore/store-reference.cc @@ -1,4 +1,5 @@ #include "nix/util/error.hh" +#include "nix/util/file-path-impl.hh" #include "nix/util/split.hh" #include "nix/util/url.hh" #include "nix/store/store-reference.hh" @@ -6,6 +7,7 @@ #include "nix/util/util.hh" #include +#include namespace nix { @@ -16,7 +18,7 @@ static bool isNonUriPath(const std::string & spec) spec.find("://") == std::string::npos // Has at least one path separator, and so isn't a single word that // might be special like "auto" - && spec.find("/") != std::string::npos; + && OsPathTrait::findPathSep(spec) != std::string::npos; } std::string StoreReference::render(bool withParams) const @@ -110,7 +112,7 @@ StoreReference StoreReference::parse(const std::string & uri, const StoreReferen .variant = Specified{ .scheme = "local", - .authority = absPath(baseURI), + .authority = encodeUrlPath(pathToUrlPath(absPath(std::filesystem::path{baseURI}))), }, .params = std::move(params), }; @@ -184,3 +186,19 @@ std::pair splitUriAndParams(const std::stri } } // namespace nix + +namespace nlohmann { + +using namespace nix; + +StoreReference adl_serializer::from_json(const json & json) +{ + return StoreReference::parse(json.get()); +} + +void adl_serializer::to_json(json & json, const StoreReference & ref) +{ + json = ref.render(); +} + +} // namespace nlohmann diff --git a/src/libstore/store-registration.cc b/src/libstore/store-registration.cc index cfaf86b1e8bb..e4d5843ed9bd 100644 --- a/src/libstore/store-registration.cc +++ b/src/libstore/store-registration.cc @@ -8,7 +8,7 @@ namespace nix { ref openStore() { - return openStore(settings.storeUri.get()); + return openStore(StoreReference{settings.storeUri.get()}); } ref openStore(const std::string & uri, const Store::Config::Params & extraParams) @@ -30,7 +30,7 @@ ref resolveStoreConfig(StoreReference && storeURI) auto storeConfig = std::visit( overloaded{ [&](const StoreReference::Auto &) -> ref { - auto stateDir = getOr(params, "state", settings.nixStateDir); + auto stateDir = getOr(params, "state", settings.nixStateDir.string()); if (access(stateDir.c_str(), R_OK | W_OK) == 0) return make_ref(params); else if (pathExists(settings.nixDaemonSocketFile)) @@ -42,17 +42,18 @@ ref resolveStoreConfig(StoreReference && storeURI) /* If /nix doesn't exist, there is no daemon socket, and we're not root, then automatically set up a chroot store in ~/.local/share/nix/root. */ - auto chrootStore = getDataDir() + "/root"; + auto chrootStore = getDataDir() / "root"; if (!pathExists(chrootStore)) { try { createDirs(chrootStore); } catch (SystemError & e) { return make_ref(params); } - warn("'%s' does not exist, so Nix will use '%s' as a chroot store", stateDir, chrootStore); + warn("%s does not exist, so Nix will use %s as a chroot store", stateDir, PathFmt(chrootStore)); } else - debug("'%s' does not exist, so Nix will use '%s' as a chroot store", stateDir, chrootStore); - return make_ref("local", chrootStore, params); + debug( + "%s does not exist, so Nix will use %s as a chroot store", stateDir, PathFmt(chrootStore)); + return make_ref(std::filesystem::path(chrootStore), params); } #endif else @@ -79,20 +80,20 @@ std::list> getDefaultSubstituters() static auto stores([]() { std::list> stores; - StringSet done; + std::set done; - auto addStore = [&](const std::string & uri) { - if (!done.insert(uri).second) + auto addStore = [&](const StoreReference & ref) { + if (!done.insert(ref).second) return; try { - stores.push_back(openStore(uri)); + stores.push_back(openStore(StoreReference{ref})); } catch (Error & e) { logWarning(e.info()); } }; - for (const auto & uri : settings.substituters.get()) - addStore(uri); + for (const auto & ref : settings.getWorkerSettings().substituters.get()) + addStore(ref); stores.sort([](ref & a, ref & b) { return a->config.priority < b->config.priority; }); diff --git a/src/libstore/uds-remote-store.cc b/src/libstore/uds-remote-store.cc index 6106a99ce385..98e6a29332b9 100644 --- a/src/libstore/uds-remote-store.cc +++ b/src/libstore/uds-remote-store.cc @@ -19,16 +19,12 @@ namespace nix { -UDSRemoteStoreConfig::UDSRemoteStoreConfig( - std::string_view scheme, std::string_view authority, const StoreReference::Params & params) +UDSRemoteStoreConfig::UDSRemoteStoreConfig(const std::filesystem::path & path, const StoreReference::Params & params) : Store::Config{params} , LocalFSStore::Config{params} , RemoteStore::Config{params} - , path{authority.empty() ? settings.nixDaemonSocketFile : authority} + , path{path.empty() ? settings.nixDaemonSocketFile : path} { - if (uriSchemes().count(scheme) == 0) { - throw UsageError("Scheme must be 'unix'"); - } } std::string UDSRemoteStoreConfig::doc() @@ -43,7 +39,7 @@ std::string UDSRemoteStoreConfig::doc() // don't we just wire it all through? I believe there are cases where it // will live reload so we want to continue to account for that. UDSRemoteStoreConfig::UDSRemoteStoreConfig(const Params & params) - : UDSRemoteStoreConfig(*uriSchemes().begin(), "", params) + : UDSRemoteStoreConfig("", params) { } @@ -69,7 +65,7 @@ StoreReference UDSRemoteStoreConfig::getReference() const .variant = StoreReference::Specified{ .scheme = *uriSchemes().begin(), - .authority = path, + .authority = encodeUrlPath(pathToUrlPath(path)), }, .params = getQueryParams(), }; @@ -95,10 +91,10 @@ ref UDSRemoteStore::openConnection() return conn; } -void UDSRemoteStore::addIndirectRoot(const Path & path) +void UDSRemoteStore::addIndirectRoot(const std::filesystem::path & path) { auto conn(getConnection()); - conn->to << WorkerProto::Op::AddIndirectRoot << path; + conn->to << WorkerProto::Op::AddIndirectRoot << path.string(); conn.processStderr(); readInt(conn->from); } diff --git a/src/libstore/unix/build/chroot-derivation-builder.cc b/src/libstore/unix/build/chroot-derivation-builder.cc index a7bf94cf934f..eb95c3aeb2fe 100644 --- a/src/libstore/unix/build/chroot-derivation-builder.cc +++ b/src/libstore/unix/build/chroot-derivation-builder.cc @@ -13,7 +13,7 @@ struct ChrootDerivationBuilder : virtual DerivationBuilderImpl /** * The root of the chroot environment. */ - Path chrootRootDir; + std::filesystem::path chrootRootDir; /** * RAII object to delete the chroot directory. @@ -36,15 +36,15 @@ struct ChrootDerivationBuilder : virtual DerivationBuilderImpl On macOS, we don't use an actual chroot, so this isn't possible. Any mitigation along these lines would have to be done directly in the sandbox profile. */ - tmpDir = topTmpDir + "/build"; + tmpDir = topTmpDir / "build"; createDir(tmpDir, 0700); } - Path tmpDirInSandbox() override + std::filesystem::path tmpDirInSandbox() override { /* In a sandbox, for determinism, always use the same temporary directory. */ - return settings.sandboxBuildDir; + return store.config->getLocalSettings().sandboxBuildDir.get(); } virtual gid_t sandboxGid() @@ -58,57 +58,51 @@ struct ChrootDerivationBuilder : virtual DerivationBuilderImpl environment using bind-mounts. We put it in the Nix store so that the build outputs can be moved efficiently from the chroot to their final location. */ - auto chrootParentDir = store.toRealPath(drvPath) + ".chroot"; + auto chrootParentDir = store.toRealPath(drvPath); + chrootParentDir += ".chroot"; deletePath(chrootParentDir); /* Clean up the chroot directory automatically. */ autoDelChroot = std::make_shared(chrootParentDir); - printMsg(lvlChatty, "setting up chroot environment in '%1%'", chrootParentDir); + printMsg(lvlChatty, "setting up chroot environment in %1%", PathFmt(chrootParentDir)); if (mkdir(chrootParentDir.c_str(), 0700) == -1) - throw SysError("cannot create '%s'", chrootRootDir); + throw SysError("cannot create %s", PathFmt(chrootRootDir)); - chrootRootDir = chrootParentDir + "/root"; + chrootRootDir = chrootParentDir / "root"; if (mkdir(chrootRootDir.c_str(), buildUser && buildUser->getUIDCount() != 1 ? 0755 : 0750) == -1) - throw SysError("cannot create '%1%'", chrootRootDir); + throw SysError("cannot create %1%", PathFmt(chrootRootDir)); if (buildUser && chown( chrootRootDir.c_str(), buildUser->getUIDCount() != 1 ? buildUser->getUID() : 0, buildUser->getGID()) == -1) - throw SysError("cannot change ownership of '%1%'", chrootRootDir); + throw SysError("cannot change ownership of %1%", PathFmt(chrootRootDir)); /* Create a writable /tmp in the chroot. Many builders need this. (Of course they should really respect $TMPDIR instead.) */ - Path chrootTmpDir = chrootRootDir + "/tmp"; + std::filesystem::path chrootTmpDir = chrootRootDir / "tmp"; createDirs(chrootTmpDir); - chmod_(chrootTmpDir, 01777); + chmod(chrootTmpDir, 01777); /* Create a /etc/passwd with entries for the build user and the nobody account. The latter is kind of a hack to support Samba-in-QEMU. */ - createDirs(chrootRootDir + "/etc"); + createDirs(chrootRootDir / "etc"); if (drvOptions.useUidRange(drv)) - chownToBuilder(chrootRootDir + "/etc"); + chownToBuilder(chrootRootDir / "etc"); if (drvOptions.useUidRange(drv) && (!buildUser || buildUser->getUIDCount() < 65536)) - throw Error("feature 'uid-range' requires the setting '%s' to be enabled", settings.autoAllocateUids.name); - - /* Declare the build user's group so that programs get a consistent - view of the system (e.g., "id -gn"). */ - writeFile( - chrootRootDir + "/etc/group", - fmt("root:x:0:\n" - "nixbld:!:%1%:\n" - "nogroup:x:65534:\n", - sandboxGid())); + throw Error( + "feature 'uid-range' requires the setting '%s' to be enabled", + store.config->getLocalSettings().autoAllocateUids.name); /* Create /etc/hosts with localhost entry. */ if (derivationType.isSandboxed()) - writeFile(chrootRootDir + "/etc/hosts", "127.0.0.1 localhost\n::1 localhost\n"); + writeFile(chrootRootDir / "etc" / "hosts", "127.0.0.1 localhost\n::1 localhost\n"); /* Make the closure of the inputs available in the chroot, rather than the whole Nix store. This prevents any access @@ -117,18 +111,18 @@ struct ChrootDerivationBuilder : virtual DerivationBuilderImpl can be bind-mounted). !!! As an extra security precaution, make the fake Nix store only writable by the build user. */ - Path chrootStoreDir = chrootRootDir + store.storeDir; + std::filesystem::path chrootStoreDir = chrootRootDir / std::filesystem::path(store.storeDir).relative_path(); createDirs(chrootStoreDir); - chmod_(chrootStoreDir, 01775); + chmod(chrootStoreDir, 01775); if (buildUser && chown(chrootStoreDir.c_str(), 0, buildUser->getGID()) == -1) - throw SysError("cannot change ownership of '%1%'", chrootStoreDir); + throw SysError("cannot change ownership of %1%", PathFmt(chrootStoreDir)); pathsInChroot = getPathsInSandbox(); for (auto & i : inputPaths) { auto p = store.printStorePath(i); - pathsInChroot.insert_or_assign(p, ChrootPath{.source = store.toRealPath(p)}); + pathsInChroot.insert_or_assign(p, ChrootPath{.source = store.toRealPath(i)}); } /* If we're repairing, checking or rebuilding part of a @@ -150,13 +144,14 @@ struct ChrootDerivationBuilder : virtual DerivationBuilderImpl Strings getPreBuildHookArgs() override { assert(!chrootRootDir.empty()); - return Strings({store.printStorePath(drvPath), chrootRootDir}); + return Strings({store.printStorePath(drvPath), chrootRootDir.native()}); } - Path realPathInHost(const Path & p) override + std::filesystem::path realPathInHost(const std::filesystem::path & p) override { // FIXME: why the needsHashRewrite() conditional? - return !needsHashRewrite() ? chrootRootDir + p : store.toRealPath(p); + return !needsHashRewrite() ? chrootRootDir / p.relative_path() + : std::filesystem::path(store.toRealPath(store.parseStorePath(p.native()))); } void cleanupBuild(bool force) override @@ -171,26 +166,28 @@ struct ChrootDerivationBuilder : virtual DerivationBuilderImpl continue; if (buildMode != bmCheck && status.known->isValid()) continue; - auto p = store.toRealPath(status.known->path); - if (pathExists(chrootRootDir + p)) - std::filesystem::rename((chrootRootDir + p), p); + std::filesystem::path p = store.toRealPath(status.known->path); + std::filesystem::path chrootPath = chrootRootDir / p.relative_path(); + if (pathExists(chrootPath)) + std::filesystem::rename(chrootPath, p); } autoDelChroot.reset(); /* this runs the destructor */ } - std::pair addDependencyPrep(const StorePath & path) + std::pair addDependencyPrep(const StorePath & path) { DerivationBuilderImpl::addDependencyImpl(path); debug("materialising '%s' in the sandbox", store.printStorePath(path)); - Path source = store.toRealPath(path); - Path target = chrootRootDir + store.printStorePath(path); + std::filesystem::path source = store.toRealPath(path); + std::filesystem::path target = + chrootRootDir / std::filesystem::path(store.printStorePath(path)).relative_path(); if (pathExists(target)) { // There is a similar debug message in doBind, so only run it in this block to not have double messages. - debug("bind-mounting %s -> %s", target, source); + debug("bind-mounting %s -> %s", PathFmt(target), PathFmt(source)); throw Error("store path '%s' already exists in the sandbox", store.printStorePath(path)); } diff --git a/src/libstore/unix/build/darwin-derivation-builder.cc b/src/libstore/unix/build/darwin-derivation-builder.cc index 24329bffce1a..e9999d455b30 100644 --- a/src/libstore/unix/build/darwin-derivation-builder.cc +++ b/src/libstore/unix/build/darwin-derivation-builder.cc @@ -67,25 +67,25 @@ struct DarwinDerivationBuilder : DerivationBuilderImpl if (useSandbox) { /* Lots and lots and lots of file functions freak out if they can't stat their full ancestry */ - PathSet ancestry; + StringSet ancestry; /* We build the ancestry before adding all inputPaths to the store because we know they'll all have the same parents (the store), and there might be lots of inputs. This isn't particularly efficient... I doubt it'll be a bottleneck in practice */ for (auto & i : pathsInChroot) { - Path cur = i.first; - while (cur.compare("/") != 0) { - cur = dirOf(cur); - ancestry.insert(cur); + std::filesystem::path cur = i.first; + while (cur != "/") { + cur = cur.parent_path(); + ancestry.insert(cur.native()); } } /* And we want the store in there regardless of how empty pathsInChroot. We include the innermost path component this time, since it's typically /nix/store and we care about that. */ - Path cur = store.storeDir; - while (cur.compare("/") != 0) { - ancestry.insert(cur); - cur = dirOf(cur); + std::filesystem::path cur = store.storeDir; + while (cur != "/") { + ancestry.insert(cur.native()); + cur = cur.parent_path(); } /* Add all our input paths to the chroot */ @@ -96,7 +96,7 @@ struct DarwinDerivationBuilder : DerivationBuilderImpl /* Violations will go to the syslog if you set this. Unfortunately the destination does not appear to be * configurable */ - if (settings.darwinLogSandboxViolations) { + if (store.config->getLocalSettings().darwinLogSandboxViolations) { sandboxProfile += "(deny default)\n"; } else { sandboxProfile += "(deny default (with no-log))\n"; @@ -137,9 +137,9 @@ struct DarwinDerivationBuilder : DerivationBuilderImpl if (i.first != i.second.source) throw Error( - "can't map '%1%' to '%2%': mismatched impure paths not supported on Darwin", - i.first, - i.second.source); + "can't map %1% to %2%: mismatched impure paths not supported on Darwin", + PathFmt(i.first), + PathFmt(i.second.source)); std::string path = i.first; auto optSt = maybeLstat(path.c_str()); @@ -174,18 +174,19 @@ struct DarwinDerivationBuilder : DerivationBuilderImpl /* The tmpDir in scope points at the temporary build directory for our derivation. Some packages try different mechanisms to find temporary directories, so we want to open up a broader place for them to put their files, if needed. */ - Path globalTmpDir = canonPath(defaultTempDir().string(), true); + std::filesystem::path globalTmpDir = canonPath(defaultTempDir().native(), true); /* They don't like trailing slashes on subpath directives */ - while (!globalTmpDir.empty() && globalTmpDir.back() == '/') - globalTmpDir.pop_back(); + std::string globalTmpDirStr = globalTmpDir.native(); + while (!globalTmpDirStr.empty() && globalTmpDirStr.back() == '/') + globalTmpDirStr.pop_back(); if (getEnv("_NIX_TEST_NO_SANDBOX") != "1") { Strings sandboxArgs; sandboxArgs.push_back("_NIX_BUILD_TOP"); - sandboxArgs.push_back(tmpDir); + sandboxArgs.push_back(tmpDir.native()); sandboxArgs.push_back("_GLOBAL_TMP_DIR"); - sandboxArgs.push_back(globalTmpDir); + sandboxArgs.push_back(globalTmpDirStr); if (drvOptions.allowLocalNetworking) { sandboxArgs.push_back("_ALLOW_LOCAL_NETWORKING"); sandboxArgs.push_back("1"); diff --git a/src/libstore/unix/build/derivation-builder.cc b/src/libstore/unix/build/derivation-builder.cc index d47776655e5d..4c93315ff81c 100644 --- a/src/libstore/unix/build/derivation-builder.cc +++ b/src/libstore/unix/build/derivation-builder.cc @@ -1,11 +1,11 @@ #include "nix/store/build/derivation-builder.hh" +#include "nix/util/file-system-at.hh" #include "nix/util/file-system.hh" #include "nix/store/local-store.hh" #include "nix/store/active-builds.hh" #include "nix/util/processes.hh" #include "nix/store/builtins.hh" #include "nix/store/path-references.hh" -#include "nix/util/finally.hh" #include "nix/util/util.hh" #include "nix/util/archive.hh" #include "nix/util/git.hh" @@ -20,10 +20,9 @@ #include "nix/store/globals.hh" #include "nix/store/build/derivation-env-desugar.hh" #include "nix/util/terminal.hh" +#include "nix/store/filetransfer.hh" #include "nix/store/provenance.hh" -#include - #include #include #include @@ -31,6 +30,9 @@ #include #include #include +#ifdef __linux__ +# include +#endif #include "store-config-private.hh" @@ -56,14 +58,46 @@ namespace nix { -struct NotDeterministic : BuildError +struct NotDeterministic final : CloneableError { NotDeterministic(auto &&... args) - : BuildError(BuildResult::Failure::NotDeterministic, args...) + : CloneableError(BuildResult::Failure::NotDeterministic, args...) { + isNonDeterministic = true; } }; +void preserveDeathSignal(fun setCredentials) +{ +#ifdef __linux__ + /* Record the old parent pid. This is to avoid a race in case the parent + gets killed after setuid, but before we restored the death signal. It is + zero if the parent isn't visible inside the PID namespace. + See: https://stackoverflow.com/questions/284325/how-to-make-child-process-die-after-parent-exits */ + auto parentPid = getppid(); + + int oldDeathSignal; + if (prctl(PR_GET_PDEATHSIG, &oldDeathSignal) == -1) + throw SysError("getting death signal"); + + setCredentials(); /* Invoke the callback that does setuid etc. */ + + /* Set the old death signal. SIGKILL is set by default in startProcess, + but it gets cleared after setuid. Without this we end up with runaway + build processes if we get killed. */ + if (prctl(PR_SET_PDEATHSIG, oldDeathSignal) == -1) + throw SysError("setting death signal"); + + /* The parent got killed and we got reparented. Commit seppuku. This check + doesn't help much with PID namespaces, but it's still useful without + sandboxing. */ + if (oldDeathSignal && getppid() != parentPid) + raise(oldDeathSignal); +#else + setCredentials(); /* Just call the function on non-Linux. */ +#endif +} + /** * This class represents the state for building locally. * @@ -92,6 +126,8 @@ class DerivationBuilderImpl : public DerivationBuilder, public DerivationBuilder LocalStore & store; + const LocalSettings & localSettings = store.config->getLocalSettings(); + std::unique_ptr miscMethods; public: @@ -139,13 +175,13 @@ class DerivationBuilderImpl : public DerivationBuilder, public DerivationBuilder /** * The temporary directory used for the build. */ - Path tmpDir; + std::filesystem::path tmpDir; /** * The top-level temporary directory. `tmpDir` is either equal to * or a child of this directory. */ - Path topTmpDir; + std::filesystem::path topTmpDir; /** * The file descriptor of the temporary directory. @@ -185,7 +221,7 @@ class DerivationBuilderImpl : public DerivationBuilder, public DerivationBuilder */ OutputPathMap scratchOutputs; - const static Path homeDir; + const static std::filesystem::path homeDir; /** * The recursive Nix daemon socket. @@ -242,7 +278,7 @@ class DerivationBuilderImpl : public DerivationBuilder, public DerivationBuilder */ virtual std::unique_ptr getBuildUser() { - return acquireUserLock(1, false); + return acquireUserLock(settings.nixStateDir, localSettings, 1, false); } /** @@ -269,7 +305,7 @@ class DerivationBuilderImpl : public DerivationBuilder, public DerivationBuilder /** * Return the path of the temporary directory in the sandbox. */ - virtual Path tmpDirInSandbox() + virtual std::filesystem::path tmpDirInSandbox() { assert(!topTmpDir.empty()); return topTmpDir; @@ -295,9 +331,9 @@ class DerivationBuilderImpl : public DerivationBuilder, public DerivationBuilder return Strings({store.printStorePath(drvPath)}); } - virtual Path realPathInHost(const Path & p) + virtual std::filesystem::path realPathInHost(const std::filesystem::path & p) { - return store.toRealPath(p); + return store.toRealPath(store.parseStorePath(p.native())); } /** @@ -357,15 +393,20 @@ class DerivationBuilderImpl : public DerivationBuilder, public DerivationBuilder * SAFETY: this function is prone to TOCTOU as it receives a path and not a descriptor. * It's only safe to call in a child of a directory only visible to the owner. */ - void chownToBuilder(const Path & path); + void chownToBuilder(const std::filesystem::path & path); /** * Make a file owned by the builder addressed by its file descriptor. + * + * @param path Only used for error messages. */ - void chownToBuilder(int fd, const Path & path); + void chownToBuilder(int fd, const std::filesystem::path & path); /** * Create a file in `tmpDir` owned by the builder. + * + * @param Name must not contain more than one path segment and none of them must be `..`, `.` + * Otherwise this function throws an Error. */ void writeBuilderFile(const std::string & name, std::string_view contents); @@ -415,8 +456,8 @@ class DerivationBuilderImpl : public DerivationBuilder, public DerivationBuilder /** * Delete the temporary directory, if we have one. * - * @param force We know the build suceeded, so don't attempt to - * preseve anything for debugging. + * @param force We know the build succeeded, so don't attempt to + * preserve anything for debugging. */ virtual void cleanupBuild(bool force); @@ -451,36 +492,39 @@ class DerivationBuilderImpl : public DerivationBuilder, public DerivationBuilder StorePath makeFallbackPath(OutputNameView outputName); }; -void handleDiffHook( - uid_t uid, uid_t gid, const Path & tryA, const Path & tryB, const Path & drvPath, const Path & tmpDir) +static void handleDiffHook( + const std::filesystem::path & diffHook, + uid_t uid, + uid_t gid, + const std::filesystem::path & tryA, + const std::filesystem::path & tryB, + const std::filesystem::path & drvPath, + const std::filesystem::path & tmpDir) { - auto & diffHookOpt = settings.diffHook.get(); - if (diffHookOpt && settings.runDiffHook) { - auto & diffHook = *diffHookOpt; - try { - auto diffRes = runProgram( - RunOptions{ - .program = diffHook, - .lookupPath = true, - .args = {tryA, tryB, drvPath, tmpDir}, - .uid = uid, - .gid = gid, - .chdir = "/"}); - if (!statusOk(diffRes.first)) - throw ExecError(diffRes.first, "diff-hook program '%1%' %2%", diffHook, statusToString(diffRes.first)); - - if (diffRes.second != "") - printError(chomp(diffRes.second)); - } catch (Error & error) { - ErrorInfo ei = error.info(); - // FIXME: wrap errors. - ei.msg = HintFmt("diff hook execution failed: %s", ei.msg.str()); - logError(ei); - } + try { + auto diffRes = runProgram( + RunOptions{ + .program = diffHook, + .lookupPath = true, + .args = {tryA, tryB, drvPath, tmpDir}, + .uid = uid, + .gid = gid, + .chdir = "/"}); + if (!statusOk(diffRes.first)) + throw ExecError( + diffRes.first, "diff-hook program %s %2%", PathFmt(diffHook), statusToString(diffRes.first)); + + if (diffRes.second != "") + printError(chomp(diffRes.second)); + } catch (Error & error) { + ErrorInfo ei = error.info(); + // FIXME: wrap errors. + ei.msg = HintFmt("diff hook execution failed: %s", ei.msg.str()); + logError(ei); } } -const Path DerivationBuilderImpl::homeDir = "/homeless-shelter"; +const std::filesystem::path DerivationBuilderImpl::homeDir = "/homeless-shelter"; void DerivationBuilderImpl::killSandbox(bool getStats) { @@ -507,6 +551,8 @@ bool DerivationBuilderImpl::killChild() pid.wait(); activeBuildHandle.reset(); + + miscMethods->childTerminated(); } return ret; } @@ -522,7 +568,7 @@ SingleDrvOutputs DerivationBuilderImpl::unprepareBuild() debug("builder process for '%s' finished", store.printStorePath(drvPath)); buildResult.timesBuilt++; - buildResult.stopTime = time(0); + buildResult.stopTime = time(nullptr); /* So the child is gone now. */ miscMethods->childTerminated(); @@ -579,37 +625,31 @@ SingleDrvOutputs DerivationBuilderImpl::unprepareBuild() return builtOutputs; } -static void chmod_(const Path & path, mode_t mode) -{ - if (chmod(path.c_str(), mode) == -1) - throw SysError("setting permissions on '%s'", path); -} - /* Move/rename path 'src' to 'dst'. Temporarily make 'src' writable if it's a directory and we're not root (to be able to update the directory's parent link ".."). */ -static void movePath(const Path & src, const Path & dst) +static void movePath(const std::filesystem::path & src, const std::filesystem::path & dst) { auto st = lstat(src); bool changePerm = (geteuid() && S_ISDIR(st.st_mode) && !(st.st_mode & S_IWUSR)); if (changePerm) - chmod_(src, st.st_mode | S_IWUSR); + chmod(src, st.st_mode | S_IWUSR); std::filesystem::rename(src, dst); if (changePerm) - chmod_(dst, st.st_mode); + chmod(dst, st.st_mode); } -static void replaceValidPath(const Path & storePath, const Path & tmpPath) +static void replaceValidPath(const std::filesystem::path & storePath, const std::filesystem::path & tmpPath) { /* We can't atomically replace storePath (the original) with tmpPath (the replacement), so we have to move it out of the way first. We'd better not be interrupted here, because if we're repairing (say) Glibc, we end up with a broken system. */ - Path oldPath; + std::filesystem::path oldPath; if (pathExists(storePath)) { // why do we loop here? @@ -702,7 +742,7 @@ static void checkNotWorldWritable(std::filesystem::path path) while (true) { auto st = lstat(path); if (st.st_mode & S_IWOTH) - throw Error("Path %s is world-writable or a symlink. That's not allowed for security.", path); + throw Error("Path %s is world-writable or a symlink. That's not allowed for security.", PathFmt(path)); if (path == path.parent_path()) break; path = path.parent_path(); @@ -712,7 +752,7 @@ static void checkNotWorldWritable(std::filesystem::path path) std::optional DerivationBuilderImpl::startBuild() { - if (useBuildUsers()) { + if (useBuildUsers(localSettings)) { if (!buildUser) buildUser = getBuildUser(); @@ -742,7 +782,7 @@ std::optional DerivationBuilderImpl::startBuild() POSIX semantics.*/ tmpDirFd = AutoCloseFD{open(tmpDir.c_str(), O_RDONLY | O_NOFOLLOW | O_DIRECTORY)}; if (!tmpDirFd) - throw SysError("failed to open the build temporary directory descriptor '%1%'", tmpDir); + throw SysError("failed to open the build temporary directory descriptor %1%", PathFmt(tmpDir)); chownToBuilder(tmpDirFd.get(), tmpDir); @@ -808,7 +848,8 @@ std::optional DerivationBuilderImpl::startBuild() if (needsHashRewrite() && pathExists(homeDir)) throw Error( - "home directory '%1%' exists; please remove it to assure purity of builds without sandboxing", homeDir); + "home directory %1% exists; please remove it to assure purity of builds without sandboxing", + PathFmt(homeDir)); /* Fire up a Nix daemon to process recursive Nix calls from the builder. */ @@ -832,8 +873,7 @@ std::optional DerivationBuilderImpl::startBuild() std::string slaveName = getPtsName(builderOut.get()); if (buildUser) { - if (chmod(slaveName.c_str(), 0600)) - throw SysError("changing mode of pseudoterminal slave"); + chmod(slaveName, 0600); if (chown(slaveName.c_str(), buildUser->getUID(), 0)) throw SysError("changing owner of pseudoterminal slave"); @@ -848,7 +888,7 @@ std::optional DerivationBuilderImpl::startBuild() if (unlockpt(builderOut.get())) throw SysError("unlocking pseudoterminal"); - buildResult.startTime = time(0); + buildResult.startTime = time(nullptr); /* Start a child process to build the derivation. */ startChild(); @@ -890,14 +930,15 @@ PathsInChroot DerivationBuilderImpl::getPathsInSandbox() #endif && !maybeLstat(p.second.source)) throw SysError( - "path '%s' is configured as part of the `sandbox-paths` option, but is inaccessible", p.second.source); + "path %s is configured as part of the `sandbox-paths` option, but is inaccessible", + PathFmt(p.second.source)); - if (hasPrefix(store.storeDir, tmpDirInSandbox())) { + if (hasPrefix(store.storeDir, tmpDirInSandbox().native())) { throw Error("`sandbox-build-dir` must not contain the storeDir"); } pathsInChroot[tmpDirInSandbox()] = {.source = tmpDir}; - PathSet allowedPaths = settings.allowedImpureHostPrefixes; + auto allowedPaths = localSettings.allowedImpureHostPrefixes.get(); /* This works like the above, except on a per-derivation level */ auto impurePaths = drvOptions.impureHostDeps; @@ -907,10 +948,10 @@ PathsInChroot DerivationBuilderImpl::getPathsInSandbox() /* Note: we're not resolving symlinks here to prevent giving a non-root user info about inaccessible files. */ - Path canonI = canonPath(i); + std::filesystem::path canonI = canonPath(i); /* If only we had a trie to do this more efficiently :) luckily, these are generally going to be pretty small */ for (auto & a : allowedPaths) { - Path canonA = canonPath(a); + std::filesystem::path canonA = canonPath(a); if (isDirOrInDir(canonI, canonA)) { found = true; break; @@ -927,13 +968,13 @@ PathsInChroot DerivationBuilderImpl::getPathsInSandbox() pathsInChroot[i] = {i, true}; } - if (settings.preBuildHook != "") { - printMsg(lvlChatty, "executing pre-build hook '%1%'", settings.preBuildHook); + if (localSettings.preBuildHook != "") { + printMsg(lvlChatty, "executing pre-build hook '%1%'", localSettings.preBuildHook); enum BuildHookState { stBegin, stExtraChrootDirs }; auto state = stBegin; - auto lines = runProgram(settings.preBuildHook, false, getPreBuildHookArgs()); + auto lines = runProgram(localSettings.preBuildHook.get(), false, getPreBuildHookArgs()); auto lastPos = std::string::size_type{0}; for (auto nlPos = lines.find('\n'); nlPos != std::string::npos; nlPos = lines.find('\n', lastPos)) { auto line = lines.substr(lastPos, nlPos - lastPos); @@ -1084,13 +1125,15 @@ void DerivationBuilderImpl::initEnv() env["NIX_STORE"] = store.storeDir; /* The maximum number of cores to utilize for parallel building. */ - env["NIX_BUILD_CORES"] = fmt("%d", settings.buildCores ? settings.buildCores : settings.getDefaultCores()); + env["NIX_BUILD_CORES"] = fmt( + "%d", + settings.getLocalSettings().buildCores ? settings.getLocalSettings().buildCores : settings.getDefaultCores()); /* Write the final environment. Note that this is intentionally *not* `drv.env`, because we've desugared things like like "passAFile", "expandReferencesGraph", structured attrs, etc. */ for (const auto & [name, info] : desugaredEnv.variables) { - env[name] = info.prependBuildDirectory ? tmpDirInSandbox() + "/" + info.value : info.value; + env[name] = info.prependBuildDirectory ? (tmpDirInSandbox() / info.value).string() : info.value; } /* Add extra files, similar to `finalEnv` */ @@ -1128,7 +1171,7 @@ void DerivationBuilderImpl::initEnv() fixed-output derivations is by definition pure (since we already know the cryptographic hash of the output). */ if (!derivationType.isSandboxed()) { - auto & impureEnv = settings.impureEnv.get(); + auto & impureEnv = localSettings.impureEnv.get(); if (!impureEnv.empty()) experimentalFeatureSettings.require(Xp::ConfigurableImpureEnv); @@ -1169,8 +1212,8 @@ void DerivationBuilderImpl::startDaemon() addedPaths.clear(); auto socketName = ".nix-socket"; - Path socketPath = tmpDir + "/" + socketName; - env["NIX_REMOTE"] = "unix://" + tmpDirInSandbox() + "/" + socketName; + std::filesystem::path socketPath = tmpDir / socketName; + env["NIX_REMOTE"] = "unix://" + (tmpDirInSandbox() / socketName).native(); daemonSocket = createUnixDomainSocket(socketPath, 0600); @@ -1252,30 +1295,34 @@ void DerivationBuilderImpl::addDependencyImpl(const StorePath & path) addedPaths.insert(path); } -void DerivationBuilderImpl::chownToBuilder(const Path & path) +void DerivationBuilderImpl::chownToBuilder(const std::filesystem::path & path) { if (!buildUser) return; if (chown(path.c_str(), buildUser->getUID(), buildUser->getGID()) == -1) - throw SysError("cannot change ownership of '%1%'", path); + throw SysError("cannot change ownership of %1%", PathFmt(path)); } -void DerivationBuilderImpl::chownToBuilder(int fd, const Path & path) +void DerivationBuilderImpl::chownToBuilder(int fd, const std::filesystem::path & path) { if (!buildUser) return; if (fchown(fd, buildUser->getUID(), buildUser->getGID()) == -1) - throw SysError("cannot change ownership of file '%1%'", path); + throw SysError("cannot change ownership of file %1%", PathFmt(path)); } void DerivationBuilderImpl::writeBuilderFile(const std::string & name, std::string_view contents) { - auto path = std::filesystem::path(tmpDir) / name; - AutoCloseFD fd{ - openat(tmpDirFd.get(), name.c_str(), O_WRONLY | O_TRUNC | O_CREAT | O_CLOEXEC | O_EXCL | O_NOFOLLOW, 0666)}; + /* Path must be the same after normalisation. This is an additional sanity check in addition to + existing parsing checks for non-structured attrs exportReferencesGraph. In practice we only expect + a single path component without any `..`, `.` components. */ + auto relPath = CanonPath::fromFilename(name); + AutoCloseFD fd = openFileEnsureBeneathNoSymlinks( + tmpDirFd.get(), relPath, O_WRONLY | O_TRUNC | O_CREAT | O_CLOEXEC | O_EXCL | O_NOFOLLOW, 0666); + auto path = tmpDir / relPath.rel(); /* This is used only for error messages. */ if (!fd) - throw SysError("creating file %s", path); - writeFile(fd, path, contents); + throw SysError("creating file %s", PathFmt(path)); + writeFile(fd.get(), contents); chownToBuilder(fd.get(), path); } @@ -1295,6 +1342,7 @@ void DerivationBuilderImpl::runChild(RunChildArgs args) different uid and/or in a sandbox). */ BuiltinBuilderContext ctx{ .drv = drv, + .hashedMirrors = settings.getLocalSettings().hashedMirrors, .tmpDirInSandbox = tmpDirInSandbox(), #if NIX_WITH_AWS_AUTH .awsCredentials = args.awsCredentials, @@ -1303,20 +1351,21 @@ void DerivationBuilderImpl::runChild(RunChildArgs args) if (drv.isBuiltin() && drv.builder == "builtin:fetchurl") { try { - ctx.netrcData = readFile(settings.netrcFile); + ctx.netrcData = readFile(fileTransferSettings.netrcFile.get()); } catch (SystemError &) { } - try { - ctx.caFileData = readFile(settings.caFile); - } catch (SystemError &) { - } + if (auto & caFile = fileTransferSettings.caFile.get()) + try { + ctx.caFileData = readFile(*caFile); + } catch (SystemError &) { + } } enterChroot(); if (chdir(tmpDirInSandbox().c_str()) == -1) - throw SysError("changing into '%1%'", tmpDir); + throw SysError("changing into %1%", PathFmt(tmpDir)); /* Close all other file descriptors. */ unix::closeExtraFDs(); @@ -1386,17 +1435,21 @@ void DerivationBuilderImpl::setUser() setuid() when run as root sets the real, effective and saved UIDs. */ if (buildUser) { - /* Preserve supplementary groups of the build user, to allow - admins to specify groups such as "kvm". */ - auto gids = buildUser->getSupplementaryGIDs(); - if (setgroups(gids.size(), gids.data()) == -1) - throw SysError("cannot set supplementary groups of build user"); - - if (setgid(buildUser->getGID()) == -1 || getgid() != buildUser->getGID() || getegid() != buildUser->getGID()) - throw SysError("setgid failed"); - - if (setuid(buildUser->getUID()) == -1 || getuid() != buildUser->getUID() || geteuid() != buildUser->getUID()) - throw SysError("setuid failed"); + preserveDeathSignal([this]() { + /* Preserve supplementary groups of the build user, to allow + admins to specify groups such as "kvm". */ + auto gids = buildUser->getSupplementaryGIDs(); + if (setgroups(gids.size(), gids.data()) == -1) + throw SysError("cannot set supplementary groups of build user"); + + if (setgid(buildUser->getGID()) == -1 || getgid() != buildUser->getGID() + || getegid() != buildUser->getGID()) + throw SysError("setgid failed"); + + if (setuid(buildUser->getUID()) == -1 || getuid() != buildUser->getUID() + || geteuid() != buildUser->getUID()) + throw SysError("setuid failed"); + }); } } @@ -1453,7 +1506,7 @@ SingleDrvOutputs DerivationBuilderImpl::registerOutputs() scratchOutputsInverse.insert_or_assign(path, outputName); std::map> outputReferencesIfUnregistered; - std::map outputStats; + std::map outputStats; for (auto & [outputName, _] : drv.outputs) { auto scratchOutput = get(scratchOutputs, outputName); assert(scratchOutput); @@ -1474,15 +1527,15 @@ SingleDrvOutputs DerivationBuilderImpl::registerOutputs() continue; } - auto optSt = maybeLstat(actualPath.c_str()); + auto optSt = maybeLstat(actualPath); if (!optSt) throw BuildError( BuildResult::Failure::OutputRejected, - "builder for '%s' failed to produce output path for output '%s' at '%s'", + "builder for '%s' failed to produce output path for output '%s' at %s", store.printStorePath(drvPath), outputName, - actualPath); - struct stat & st = *optSt; + PathFmt(actualPath)); + PosixStat & st = *optSt; #ifndef __CYGWIN__ /* Check that the output is not group or world writable, as @@ -1493,8 +1546,8 @@ SingleDrvOutputs DerivationBuilderImpl::registerOutputs() || (buildUser && st.st_uid != buildUser->getUID())) throw BuildError( BuildResult::Failure::OutputRejected, - "suspicious ownership or permission on '%s' for output '%s'; rejecting this build output", - actualPath, + "suspicious ownership or permission on %s for output '%s'; rejecting this build output", + PathFmt(actualPath), outputName); #endif @@ -1502,7 +1555,13 @@ SingleDrvOutputs DerivationBuilderImpl::registerOutputs() rewriting doesn't contain a hard link to /etc/shadow or something like that. */ canonicalisePathMetaData( - actualPath, buildUser ? std::optional(buildUser->getUIDRange()) : std::nullopt, inodesSeen); + actualPath, + { +#ifndef _WIN32 + .uidRange = buildUser ? std::optional(buildUser->getUIDRange()) : std::nullopt, +#endif + NIX_WHEN_SUPPORT_ACLS(localSettings.ignoredAcls)}, + inodesSeen); bool discardReferences = false; if (auto udr = get(drvOptions.unsafeDiscardReferences, outputName)) { @@ -1513,7 +1572,7 @@ SingleDrvOutputs DerivationBuilderImpl::registerOutputs() if (discardReferences) debug("discarding references of output '%s'", outputName); else { - debug("scanning for references for output '%s' in temp location '%s'", outputName, actualPath); + debug("scanning for references for output '%s' in temp location %s", outputName, PathFmt(actualPath)); /* Pass blank Sink as we are not ready to hash data at this stage. */ NullSink blank; @@ -1609,7 +1668,7 @@ SingleDrvOutputs DerivationBuilderImpl::registerOutputs() auto rewriteOutput = [&](const StringMap & rewrites) { /* Apply hash rewriting if necessary. */ if (!rewrites.empty()) { - debug("rewriting hashes in '%1%'; cross fingers", actualPath); + debug("rewriting hashes in %1%; cross fingers", PathFmt(actualPath)); /* FIXME: Is this actually streaming? */ auto source = sinkToSource([&](Sink & nextSink) { @@ -1617,14 +1676,22 @@ SingleDrvOutputs DerivationBuilderImpl::registerOutputs() dumpPath(actualPath, rsink); rsink.flush(); }); - Path tmpPath = actualPath + ".tmp"; + std::filesystem::path tmpPath = actualPath.native() + ".tmp"; restorePath(tmpPath, *source); deletePath(actualPath); movePath(tmpPath, actualPath); /* FIXME: set proper permissions in restorePath() so we don't have to do another traversal. */ - canonicalisePathMetaData(actualPath, {}, inodesSeen); + canonicalisePathMetaData( + actualPath, + { +#ifndef _WIN32 + // builder UIDs are already dealt with + .uidRange = std::nullopt, +#endif + NIX_WHEN_SUPPORT_ACLS(localSettings.ignoredAcls)}, + inodesSeen); } }; @@ -1657,15 +1724,17 @@ SingleDrvOutputs DerivationBuilderImpl::registerOutputs() auto st = get(outputStats, outputName); if (!st) throw BuildError( - BuildResult::Failure::OutputRejected, "output path %1% without valid stats info", actualPath); + BuildResult::Failure::OutputRejected, + "output path %1% without valid stats info", + PathFmt(actualPath)); if (outputHash.method.getFileIngestionMethod() == FileIngestionMethod::Flat) { /* The output path should be a regular file without execute permission. */ if (!S_ISREG(st->st_mode) || (st->st_mode & S_IXUSR) != 0) throw BuildError( BuildResult::Failure::OutputRejected, - "output path '%1%' should be a non-executable regular file " + "output path %1% should be a non-executable regular file " "since recursive hashing is not enabled (one of outputHashMode={flat,text} is true)", - actualPath); + PathFmt(actualPath)); } rewriteOutput(outputRewrites); /* FIXME optimize and deduplicate with addToStore */ @@ -1677,11 +1746,13 @@ SingleDrvOutputs DerivationBuilderImpl::registerOutputs() case FileIngestionMethod::NixArchive: { HashModuloSink caSink{outputHash.hashAlgo, oldHashPart}; auto fim = outputHash.method.getFileIngestionMethod(); - dumpPath({getFSSourceAccessor(), CanonPath(actualPath)}, caSink, (FileSerialisationMethod) fim); + dumpPath( + {getFSSourceAccessor(), CanonPath(actualPath.native())}, caSink, (FileSerialisationMethod) fim); return caSink.finish().hash; } case FileIngestionMethod::Git: { - return git::dumpHash(outputHash.hashAlgo, {getFSSourceAccessor(), CanonPath(actualPath)}).hash; + return git::dumpHash(outputHash.hashAlgo, {getFSSourceAccessor(), CanonPath(actualPath.native())}) + .hash; } } assert(false); @@ -1703,7 +1774,7 @@ SingleDrvOutputs DerivationBuilderImpl::registerOutputs() { HashResult narHashAndSize = hashPath( - {getFSSourceAccessor(), CanonPath(actualPath)}, + {getFSSourceAccessor(), CanonPath(actualPath.native())}, FileSerialisationMethod::NixArchive, HashAlgorithm::SHA256); newInfo0.narHash = narHashAndSize.hash; @@ -1727,7 +1798,7 @@ SingleDrvOutputs DerivationBuilderImpl::registerOutputs() std::string{scratchPath->hashPart()}, std::string{requiredFinalPath.hashPart()}); rewriteOutput(outputRewrites); HashResult narHashAndSize = hashPath( - {getFSSourceAccessor(), CanonPath(actualPath)}, + {getFSSourceAccessor(), CanonPath(actualPath.native())}, FileSerialisationMethod::NixArchive, HashAlgorithm::SHA256); ValidPathInfo newInfo0{requiredFinalPath, {store, narHashAndSize.hash}}; @@ -1744,8 +1815,8 @@ SingleDrvOutputs DerivationBuilderImpl::registerOutputs() // Replace the output by a fresh copy of itself to make sure // that there's no stale file descriptor pointing to it - Path tmpOutput = actualPath + ".tmp"; - copyFile(std::filesystem::path(actualPath), std::filesystem::path(tmpOutput), true); + std::filesystem::path tmpOutput = actualPath.native() + ".tmp"; + copyFile(actualPath, tmpOutput, true); std::filesystem::rename(tmpOutput, actualPath); @@ -1777,7 +1848,15 @@ SingleDrvOutputs DerivationBuilderImpl::registerOutputs() /* FIXME: set proper permissions in restorePath() so we don't have to do another traversal. */ - canonicalisePathMetaData(actualPath, {}, inodesSeen); + canonicalisePathMetaData( + actualPath, + { +#ifndef _WIN32 + // builder UIDs are already dealt with + .uidRange = std::nullopt, +#endif + NIX_WHEN_SUPPORT_ACLS(localSettings.ignoredAcls)}, + inodesSeen); /* Calculate where we'll move the output files. In the checking case we will leave leave them where they are, for now, rather than move to @@ -1792,14 +1871,14 @@ SingleDrvOutputs DerivationBuilderImpl::registerOutputs() auto optFixedPath = output->path(store, drv.name, outputName); if (!optFixedPath || store.printStorePath(*optFixedPath) != finalDestPath) { assert(newInfo.ca); - dynamicOutputLock.lockPaths({store.toRealPath(finalDestPath)}); + dynamicOutputLock.lockPaths({store.toRealPath(newInfo.path)}); } /* Move files, if needed */ - if (store.toRealPath(finalDestPath) != actualPath) { + if (store.toRealPath(newInfo.path) != actualPath) { if (buildMode == bmRepair) { /* Path already exists, need to replace it */ - replaceValidPath(store.toRealPath(finalDestPath), actualPath); + replaceValidPath(store.toRealPath(newInfo.path), actualPath); } else if (buildMode == bmCheck) { /* Path already exists, and we want to compare, so we leave out new path in place. */ @@ -1810,7 +1889,7 @@ SingleDrvOutputs DerivationBuilderImpl::registerOutputs() /* Can delete our scratch copy now. */ deletePath(actualPath); } else { - auto destPath = store.toRealPath(finalDestPath); + auto destPath = store.toRealPath(newInfo.path); deletePath(destPath); movePath(actualPath, destPath); } @@ -1822,29 +1901,34 @@ SingleDrvOutputs DerivationBuilderImpl::registerOutputs() if (store.isValidPath(newInfo.path)) { ValidPathInfo oldInfo(*store.queryPathInfo(newInfo.path)); if (newInfo.narHash != oldInfo.narHash) { - if (settings.runDiffHook || settings.keepFailed) { - auto dst = store.toRealPath(finalDestPath + ".check"); + auto * diffHook = localSettings.getDiffHook(); + if (diffHook || settings.keepFailed) { + auto dst = store.toRealPath(newInfo.path); + dst += ".check"; deletePath(dst); movePath(actualPath, dst); - handleDiffHook( - buildUser ? buildUser->getUID() : getuid(), - buildUser ? buildUser->getGID() : getgid(), - finalDestPath, - dst, - store.printStorePath(drvPath), - tmpDir); + if (diffHook) { + handleDiffHook( + *diffHook, + buildUser ? buildUser->getUID() : getuid(), + buildUser ? buildUser->getGID() : getgid(), + finalDestPath, + dst, + store.printStorePath(drvPath), + tmpDir); + } throw NotDeterministic( - "derivation '%s' may not be deterministic: output '%s' differs from '%s'", + "derivation '%s' may not be deterministic: output %s differs from %s", store.printStorePath(drvPath), - store.toRealPath(finalDestPath), - dst); + PathFmt(store.toRealPath(newInfo.path)), + PathFmt(dst)); } else throw NotDeterministic( - "derivation '%s' may not be deterministic: output '%s' differs", + "derivation '%s' may not be deterministic: output %s differs", store.printStorePath(drvPath), - store.toRealPath(finalDestPath)); + PathFmt(store.toRealPath(newInfo.path))); } /* Since we verified the build, it's now ultimately trusted. */ @@ -1866,8 +1950,7 @@ SingleDrvOutputs DerivationBuilderImpl::registerOutputs() } if (!store.isValidPath(newInfo.path)) - store.optimisePath( - store.toRealPath(finalDestPath), NoRepair); // FIXME: combine with scanForReferences() + store.optimisePath(store.toRealPath(newInfo.path), NoRepair); // FIXME: combine with scanForReferences() newInfo.deriver = drvPath; newInfo.ultimate = true; @@ -1875,8 +1958,8 @@ SingleDrvOutputs DerivationBuilderImpl::registerOutputs() newInfo.provenance = std::make_shared( drvPath, outputName, - settings.getHostName(), - settings.buildProvenanceTags.get(), + settings.getWorkerSettings().getHostName(), + settings.getWorkerSettings().buildProvenanceTags.get(), drv.platform, drvProvenance); store.signPathInfo(newInfo); @@ -1964,14 +2047,14 @@ void DerivationBuilderImpl::cleanupBuild(bool force) * This hardens against an attack which smuggles a file descriptor * to make use of the temporary directory. */ - chmod(topTmpDir.c_str(), 0000); + chmod(topTmpDir, 0000); /* Don't keep temporary directories for builtins because they might have privileged stuff (like a copy of netrc). */ if (settings.keepFailed && !force && !drv.isBuiltin()) { - printError("note: keeping build directory '%s'", tmpDir); - chmod(topTmpDir.c_str(), 0755); - chmod(tmpDir.c_str(), 0755); + printError("note: keeping build directory %s", PathFmt(tmpDir)); + chmod(topTmpDir, 0755); + chmod(tmpDir, 0755); } else deletePath(topTmpDir); topTmpDir = ""; @@ -2036,10 +2119,11 @@ std::unique_ptr makeDerivationBuild LocalStore & store, std::unique_ptr miscMethods, DerivationBuilderParams params) { bool useSandbox = false; + const LocalSettings & localSettings = store.config->getLocalSettings(); /* Are we doing a sandboxed build? */ { - if (settings.sandboxMode == smEnabled) { + if (localSettings.sandboxMode == smEnabled) { if (params.drvOptions.noChroot) throw Error( "derivation '%s' has '__noChroot' set, " @@ -2053,9 +2137,9 @@ std::unique_ptr makeDerivationBuild store.printStorePath(params.drvPath)); #endif useSandbox = true; - } else if (settings.sandboxMode == smDisabled) + } else if (localSettings.sandboxMode == smDisabled) useSandbox = false; - else if (settings.sandboxMode == smRelaxed) + else if (localSettings.sandboxMode == smRelaxed) // FIXME: cache derivationType useSandbox = params.drv.type().isSandboxed() && !params.drvOptions.noChroot; } @@ -2075,7 +2159,7 @@ std::unique_ptr makeDerivationBuild #ifdef __linux__ if (useSandbox && !mountAndPidNamespacesSupported()) { - if (!settings.sandboxFallback) + if (!localSettings.sandboxFallback) throw Error( "this system does not support the kernel namespaces that are required for sandboxing; use '--no-sandbox' to disable sandboxing"); debug("auto-disabling sandboxing because the prerequisite namespaces are not available"); diff --git a/src/libstore/unix/build/external-derivation-builder.cc b/src/libstore/unix/build/external-derivation-builder.cc index 6c1a00f91068..96301af1d906 100644 --- a/src/libstore/unix/build/external-derivation-builder.cc +++ b/src/libstore/unix/build/external-derivation-builder.cc @@ -15,7 +15,7 @@ struct ExternalDerivationBuilder : DerivationBuilderImpl experimentalFeatureSettings.require(Xp::ExternalBuilders); } - Path tmpDirInSandbox() override + std::filesystem::path tmpDirInSandbox() override { /* In a sandbox, for determinism, always use the same temporary directory. */ @@ -24,7 +24,7 @@ struct ExternalDerivationBuilder : DerivationBuilderImpl void setBuildTmpDir() override { - tmpDir = topTmpDir + "/build"; + tmpDir = topTmpDir / "build"; createDir(tmpDir, 0700); } @@ -49,9 +49,9 @@ struct ExternalDerivationBuilder : DerivationBuilderImpl j.emplace(name, rewriteStrings(value, inputRewrites)); json.emplace("env", std::move(j)); } - json.emplace("topTmpDir", topTmpDir); - json.emplace("tmpDir", tmpDir); - json.emplace("tmpDirInSandbox", tmpDirInSandbox()); + json.emplace("topTmpDir", topTmpDir.native()); + json.emplace("tmpDir", tmpDir.native()); + json.emplace("tmpDirInSandbox", tmpDirInSandbox().native()); json.emplace("storeDir", store.storeDir); json.emplace("realStoreDir", store.config->realStoreDir.get()); json.emplace("system", drv.platform); @@ -88,7 +88,7 @@ struct ExternalDerivationBuilder : DerivationBuilderImpl args.insert(args.end(), jsonFile); if (chdir(tmpDir.c_str()) == -1) - throw SysError("changing into '%1%'", tmpDir); + throw SysError("changing into %1%", PathFmt(tmpDir)); chownToBuilder(topTmpDir); @@ -97,7 +97,7 @@ struct ExternalDerivationBuilder : DerivationBuilderImpl debug("executing external builder: %s", concatStringsSep(" ", args)); execv(externalBuilder.program.c_str(), stringsToCharPtrs(args).data()); - throw SysError("executing '%s'", externalBuilder.program); + throw SysError("executing %s", PathFmt(externalBuilder.program)); } catch (...) { handleChildException(true); _exit(1); diff --git a/src/libstore/unix/build/hook-instance.cc b/src/libstore/unix/build/hook-instance.cc index 83824b51f755..39be05440f37 100644 --- a/src/libstore/unix/build/hook-instance.cc +++ b/src/libstore/unix/build/hook-instance.cc @@ -1,18 +1,16 @@ -#include "nix/store/globals.hh" #include "nix/util/config-global.hh" #include "nix/store/build/hook-instance.hh" -#include "nix/util/file-system.hh" #include "nix/store/build/child.hh" #include "nix/util/strings.hh" #include "nix/util/executable-path.hh" namespace nix { -HookInstance::HookInstance() +HookInstance::HookInstance(const Strings & _buildHook) { - debug("starting build hook '%s'", concatStringsSep(" ", settings.buildHook.get())); + debug("starting build hook '%s'", concatStringsSep(" ", _buildHook)); - auto buildHookArgs = settings.buildHook.get(); + auto buildHookArgs = _buildHook; if (buildHookArgs.empty()) throw Error("'build-hook' setting is empty"); @@ -69,7 +67,7 @@ HookInstance::HookInstance() execv(buildHook.native().c_str(), stringsToCharPtrs(args).data()); - throw SysError("executing '%s'", buildHook); + throw SysError("executing %s", PathFmt(buildHook)); }); pid.setSeparatePG(true); @@ -88,8 +86,11 @@ HookInstance::~HookInstance() { try { toHook.writeSide = -1; - if (pid != -1) + if (pid != -1) { pid.kill(); + if (onKillChild) + onKillChild(); + } } catch (...) { ignoreExceptionInDestructor(); } diff --git a/src/libstore/unix/build/linux-derivation-builder.cc b/src/libstore/unix/build/linux-derivation-builder.cc index fc2140817d7a..d89aa23f3eec 100644 --- a/src/libstore/unix/build/linux-derivation-builder.cc +++ b/src/libstore/unix/build/linux-derivation-builder.cc @@ -1,9 +1,12 @@ #ifdef __linux__ +# include "nix/store/globals.hh" # include "nix/store/personality.hh" +# include "nix/store/filetransfer.hh" # include "nix/util/cgroup.hh" # include "nix/util/linux-namespaces.hh" # include "nix/util/logging.hh" +# include "nix/util/serialise.hh" # include "linux/fchmodat2-compat.hh" # include @@ -23,9 +26,9 @@ namespace nix { -static void setupSeccomp() +static void setupSeccomp(const LocalSettings & localSettings) { - if (!settings.filterSyscalls) + if (!localSettings.filterSyscalls) return; # if HAVE_SECCOMP @@ -110,7 +113,7 @@ static void setupSeccomp() || seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(fsetxattr), 0) != 0) throw SysError("unable to add seccomp rule"); - if (seccomp_attr_set(ctx, SCMP_FLTATR_CTL_NNP, settings.allowNewPrivileges ? 0 : 1) != 0) + if (seccomp_attr_set(ctx, SCMP_FLTATR_CTL_NNP, localSettings.allowNewPrivileges ? 0 : 1) != 0) throw SysError("unable to set 'no new privileges' seccomp attribute"); if (seccomp_load(ctx) != 0) @@ -122,13 +125,13 @@ static void setupSeccomp() # endif } -static void doBind(const Path & source, const Path & target, bool optional = false) +static void doBind(const std::filesystem::path & source, const std::filesystem::path & target, bool optional = false) { - debug("bind mounting '%1%' to '%2%'", source, target); + debug("bind mounting %1% to %2%", PathFmt(source), PathFmt(target)); auto bindMount = [&]() { if (mount(source.c_str(), target.c_str(), "", MS_BIND | MS_REC, 0) == -1) - throw SysError("bind mount from '%1%' to '%2%' failed", source, target); + throw SysError("bind mount from %1% to %2% failed", PathFmt(source), PathFmt(target)); }; auto maybeSt = maybeLstat(source); @@ -136,7 +139,7 @@ static void doBind(const Path & source, const Path & target, bool optional = fal if (optional) return; else - throw SysError("getting attributes of path '%1%'", source); + throw SysError("getting attributes of path %1%", PathFmt(source)); } auto st = *maybeSt; @@ -145,10 +148,10 @@ static void doBind(const Path & source, const Path & target, bool optional = fal bindMount(); } else if (S_ISLNK(st.st_mode)) { // Symlinks can (apparently) not be bind-mounted, so just copy it - createDirs(dirOf(target)); - copyFile(std::filesystem::path(source), std::filesystem::path(target), false); + createDirs(target.parent_path()); + copyFile(source, target, false); } else { - createDirs(dirOf(target)); + createDirs(target.parent_path()); writeFile(target, ""); bindMount(); } @@ -160,12 +163,19 @@ struct LinuxDerivationBuilder : virtual DerivationBuilderImpl void enterChroot() override { - setupSeccomp(); + auto & localSettings = store.config->getLocalSettings(); - linux::setPersonality(drv.platform); + setupSeccomp(localSettings); + + linux::setPersonality({ + .system = drv.platform, + .impersonateLinux26 = localSettings.impersonateLinux26, + }); } }; +static const std::filesystem::path procPath = "/proc"; + struct ChrootLinuxDerivationBuilder : ChrootDerivationBuilder, LinuxDerivationBuilder { /** @@ -189,7 +199,7 @@ struct ChrootLinuxDerivationBuilder : ChrootDerivationBuilder, LinuxDerivationBu /** * The cgroup of the builder, if any. */ - std::optional cgroup; + std::optional cgroup; ChrootLinuxDerivationBuilder( LocalStore & store, std::unique_ptr miscMethods, DerivationBuilderParams params) @@ -212,47 +222,48 @@ struct ChrootLinuxDerivationBuilder : ChrootDerivationBuilder, LinuxDerivationBu std::unique_ptr getBuildUser() override { - return acquireUserLock(drvOptions.useUidRange(drv) ? 65536 : 1, true); + return acquireUserLock( + settings.nixStateDir, store.config->getLocalSettings(), drvOptions.useUidRange(drv) ? 65536 : 1, true); } void prepareUser() override { - if ((buildUser && buildUser->getUIDCount() != 1) || settings.useCgroups) { + if ((buildUser && buildUser->getUIDCount() != 1) || store.config->getLocalSettings().useCgroups) { experimentalFeatureSettings.require(Xp::Cgroups); /* If we're running from the daemon, then this will return the root cgroup of the service. Otherwise, it will return the current cgroup. */ - auto rootCgroup = getRootCgroup(); auto cgroupFS = getCgroupFS(); if (!cgroupFS) throw Error("cannot determine the cgroups file system"); - auto rootCgroupPath = canonPath(*cgroupFS + "/" + rootCgroup); + auto rootCgroupPath = *cgroupFS / getRootCgroup().rel(); if (!pathExists(rootCgroupPath)) - throw Error("expected cgroup directory '%s'", rootCgroupPath); + throw Error("expected cgroup directory %s", PathFmt(rootCgroupPath)); static std::atomic counter{0}; - cgroup = buildUser ? fmt("%s/nix-build-uid-%d", rootCgroupPath, buildUser->getUID()) - : fmt("%s/nix-build-pid-%d-%d", rootCgroupPath, getpid(), counter++); + cgroup = rootCgroupPath + / (buildUser ? fmt("nix-build-uid-%d", buildUser->getUID()) + : fmt("nix-build-pid-%d-%d", getpid(), counter++)); - debug("using cgroup '%s'", *cgroup); + debug("using cgroup %s", PathFmt(*cgroup)); /* When using a build user, record the cgroup we used for that user so that if we got interrupted previously, we can kill any left-over cgroup first. */ if (buildUser) { - auto cgroupsDir = settings.nixStateDir + "/cgroups"; + auto cgroupsDir = std::filesystem::path{settings.nixStateDir} / "cgroups"; createDirs(cgroupsDir); - auto cgroupFile = fmt("%s/%d", cgroupsDir, buildUser->getUID()); + auto cgroupFile = cgroupsDir / std::to_string(buildUser->getUID()); if (pathExists(cgroupFile)) { auto prevCgroup = readFile(cgroupFile); destroyCgroup(prevCgroup); } - writeFile(cgroupFile, *cgroup); + writeFile(cgroupFile, cgroup->native()); } } @@ -266,11 +277,11 @@ struct ChrootLinuxDerivationBuilder : ChrootDerivationBuilder, LinuxDerivationBu if (cgroup) { if (mkdir(cgroup->c_str(), 0755) != 0) - throw SysError("creating cgroup '%s'", *cgroup); + throw SysError("creating cgroup %s", PathFmt(*cgroup)); chownToBuilder(*cgroup); - chownToBuilder(*cgroup + "/cgroup.procs"); - chownToBuilder(*cgroup + "/cgroup.threads"); - // chownToBuilder(*cgroup + "/cgroup.subtree_control"); + chownToBuilder(*cgroup / "cgroup.procs"); + chownToBuilder(*cgroup / "cgroup.threads"); + // chownToBuilder(*cgroup / "cgroup.subtree_control"); } } @@ -337,7 +348,7 @@ struct ChrootLinuxDerivationBuilder : ChrootDerivationBuilder, LinuxDerivationBu if (setgroups(0, 0) == -1) { if (errno != EPERM) throw SysError("setgroups failed"); - if (settings.requireDropSupplementaryGroups) + if (store.config->getLocalSettings().requireDropSupplementaryGroups) throw Error( "setgroups failed. Set the require-drop-supplementary-groups option to false to skip this step."); } @@ -385,9 +396,11 @@ struct ChrootLinuxDerivationBuilder : ChrootDerivationBuilder, LinuxDerivationBu userNamespaceSync.writeSide = -1; }); - auto ss = tokenizeString>(readLine(sendPid.readSide.get())); + FdSource sendPidSource(sendPid.readSide.get()); + auto ss = tokenizeString>(sendPidSource.readLine()); assert(ss.size() == 1); pid = string2Int(ss[0]).value(); + auto thisProcPath = procPath / std::to_string(static_cast(pid)); if (usingUserNamespace) { /* Set the UID/GID mapping of the builder's user namespace @@ -397,12 +410,12 @@ struct ChrootLinuxDerivationBuilder : ChrootDerivationBuilder, LinuxDerivationBu uid_t hostGid = buildUser ? buildUser->getGID() : getgid(); uid_t nrIds = buildUser ? buildUser->getUIDCount() : 1; - writeFile("/proc/" + std::to_string(pid) + "/uid_map", fmt("%d %d %d", sandboxUid(), hostUid, nrIds)); + writeFile(thisProcPath / "uid_map", fmt("%d %d %d", sandboxUid(), hostUid, nrIds)); if (!buildUser || buildUser->getUIDCount() == 1) - writeFile("/proc/" + std::to_string(pid) + "/setgroups", "deny"); + writeFile(thisProcPath / "setgroups", "deny"); - writeFile("/proc/" + std::to_string(pid) + "/gid_map", fmt("%d %d %d", sandboxGid(), hostGid, nrIds)); + writeFile(thisProcPath / "gid_map", fmt("%d %d %d", sandboxGid(), hostGid, nrIds)); } else { debug("note: not using a user namespace"); if (!buildUser) @@ -410,32 +423,40 @@ struct ChrootLinuxDerivationBuilder : ChrootDerivationBuilder, LinuxDerivationBu "cannot perform a sandboxed build because user namespaces are not enabled; check /proc/sys/user/max_user_namespaces"); } - /* Now that we now the sandbox uid, we can write - /etc/passwd. */ + /* Now that we know the sandbox uid/gid, we can write + /etc/passwd and /etc/group. */ writeFile( - chrootRootDir + "/etc/passwd", + chrootRootDir / "etc" / "passwd", fmt("root:x:0:0:Nix build user:%3%:/noshell\n" "nixbld:x:%1%:%2%:Nix build user:%3%:/noshell\n" "nobody:x:65534:65534:Nobody:/:/noshell\n", sandboxUid(), sandboxGid(), - settings.sandboxBuildDir)); + store.config->getLocalSettings().sandboxBuildDir.get().native())); + + writeFile( + chrootRootDir / "etc" / "group", + fmt("root:x:0:\n" + "nixbld:!:%1%:\n" + "nogroup:x:65534:\n", + sandboxGid())); /* Save the mount- and user namespace of the child. We have to do this *before* the child does a chroot. */ - sandboxMountNamespace = open(fmt("/proc/%d/ns/mnt", (pid_t) pid).c_str(), O_RDONLY); + auto sandboxPath = thisProcPath / "ns"; + sandboxMountNamespace = open((sandboxPath / "mnt").c_str(), O_RDONLY); if (sandboxMountNamespace.get() == -1) throw SysError("getting sandbox mount namespace"); if (usingUserNamespace) { - sandboxUserNamespace = open(fmt("/proc/%d/ns/user", (pid_t) pid).c_str(), O_RDONLY); + sandboxUserNamespace = open((sandboxPath / "user").c_str(), O_RDONLY); if (sandboxUserNamespace.get() == -1) throw SysError("getting sandbox user namespace"); } /* Move the child into its own cgroup. */ if (cgroup) - writeFile(*cgroup + "/cgroup.procs", fmt("%d", (pid_t) pid)); + writeFile(*cgroup / "cgroup.procs", fmt("%d", (pid_t) pid)); /* Signal the builder that we've updated its user namespace. */ writeFull(userNamespaceSync.writeSide.get(), "1\n"); @@ -487,7 +508,7 @@ struct ChrootLinuxDerivationBuilder : ChrootDerivationBuilder, LinuxDerivationBu /* Bind-mount chroot directory to itself, to treat it as a different filesystem from /, as needed for pivot_root. */ if (mount(chrootRootDir.c_str(), chrootRootDir.c_str(), 0, MS_BIND, 0) == -1) - throw SysError("unable to bind mount '%1%'", chrootRootDir); + throw SysError("unable to bind mount %1%", PathFmt(chrootRootDir)); /* Bind-mount the sandbox's Nix store onto itself so that we can mark it as a "shared" subtree, allowing bind @@ -497,20 +518,20 @@ struct ChrootLinuxDerivationBuilder : ChrootDerivationBuilder, LinuxDerivationBu Marking chrootRootDir as MS_SHARED causes pivot_root() to fail with EINVAL. Don't know why. */ - Path chrootStoreDir = chrootRootDir + store.storeDir; + std::filesystem::path chrootStoreDir = chrootRootDir / std::filesystem::path(store.storeDir).relative_path(); if (mount(chrootStoreDir.c_str(), chrootStoreDir.c_str(), 0, MS_BIND, 0) == -1) - throw SysError("unable to bind mount the Nix store", chrootStoreDir); + throw SysError("unable to bind mount the Nix store at %1%", PathFmt(chrootStoreDir)); if (mount(0, chrootStoreDir.c_str(), 0, MS_SHARED, 0) == -1) - throw SysError("unable to make '%s' shared", chrootStoreDir); + throw SysError("unable to make %s shared", PathFmt(chrootStoreDir)); /* Set up a nearly empty /dev, unless the user asked to bind-mount the host /dev. */ Strings ss; if (pathsInChroot.find("/dev") == pathsInChroot.end()) { - createDirs(chrootRootDir + "/dev/shm"); - createDirs(chrootRootDir + "/dev/pts"); + createDirs(chrootRootDir / "dev" / "shm"); + createDirs(chrootRootDir / "dev" / "pts"); ss.push_back("/dev/full"); if (systemFeatures.count("kvm")) { if (pathExists("/dev/kvm")) { @@ -527,10 +548,10 @@ struct ChrootLinuxDerivationBuilder : ChrootDerivationBuilder, LinuxDerivationBu ss.push_back("/dev/tty"); ss.push_back("/dev/urandom"); ss.push_back("/dev/zero"); - createSymlink("/proc/self/fd", chrootRootDir + "/dev/fd"); - createSymlink("/proc/self/fd/0", chrootRootDir + "/dev/stdin"); - createSymlink("/proc/self/fd/1", chrootRootDir + "/dev/stdout"); - createSymlink("/proc/self/fd/2", chrootRootDir + "/dev/stderr"); + createSymlink("/proc/self/fd", chrootRootDir / "dev" / "fd"); + createSymlink("/proc/self/fd/0", chrootRootDir / "dev" / "stdin"); + createSymlink("/proc/self/fd/1", chrootRootDir / "dev" / "stdout"); + createSymlink("/proc/self/fd/2", chrootRootDir / "dev" / "stderr"); } /* Fixed-output derivations typically need to access the @@ -541,7 +562,7 @@ struct ChrootLinuxDerivationBuilder : ChrootDerivationBuilder, LinuxDerivationBu // services. Don’t use it for anything else that may // be configured for this system. This limits the // potential impurities introduced in fixed-outputs. - writeFile(chrootRootDir + "/etc/nsswitch.conf", "hosts: files dns\nservices: files\n"); + writeFile(chrootRootDir / "etc" / "nsswitch.conf", "hosts: files dns\nservices: files\n"); /* N.B. it is realistic that these paths might not exist. It happens when testing Nix building fixed-output derivations @@ -550,10 +571,10 @@ struct ChrootLinuxDerivationBuilder : ChrootDerivationBuilder, LinuxDerivationBu if (pathExists(path)) ss.push_back(path); - if (settings.caFile != "") { - Path caFile = settings.caFile; - if (pathExists(caFile)) - pathsInChroot.try_emplace("/etc/ssl/certs/ca-certificates.crt", canonPath(caFile, true), true); + if (auto & caFile = fileTransferSettings.caFile.get()) { + if (pathExists(*caFile)) + pathsInChroot.try_emplace( + "/etc/ssl/certs/ca-certificates.crt", canonPath(caFile->native(), true), true); } } @@ -576,26 +597,26 @@ struct ChrootLinuxDerivationBuilder : ChrootDerivationBuilder, LinuxDerivationBu static unsigned char sh[] = { # include "embedded-sandbox-shell.gen.hh" }; - auto dst = chrootRootDir + i.first; - createDirs(dirOf(dst)); + auto dst = chrootRootDir / i.first.relative_path(); + createDirs(dst.parent_path()); writeFile(dst, std::string_view((const char *) sh, sizeof(sh))); - chmod_(dst, 0555); + chmod(dst, 0555); } else # endif { - doBind(i.second.source, chrootRootDir + i.first, i.second.optional); + doBind(i.second.source, chrootRootDir / i.first.relative_path(), i.second.optional); } } /* Bind a new instance of procfs on /proc. */ - createDirs(chrootRootDir + "/proc"); - if (mount("none", (chrootRootDir + "/proc").c_str(), "proc", 0, 0) == -1) + createDirs(chrootRootDir / "proc"); + if (mount("none", (chrootRootDir / "proc").c_str(), "proc", 0, 0) == -1) throw SysError("mounting /proc"); /* Mount sysfs on /sys. */ if (buildUser && buildUser->getUIDCount() != 1) { - createDirs(chrootRootDir + "/sys"); - if (mount("none", (chrootRootDir + "/sys").c_str(), "sysfs", 0, 0) == -1) + createDirs(chrootRootDir / "sys"); + if (mount("none", (chrootRootDir / "sys").c_str(), "sysfs", 0, 0) == -1) throw SysError("mounting /sys"); } @@ -604,10 +625,10 @@ struct ChrootLinuxDerivationBuilder : ChrootDerivationBuilder, LinuxDerivationBu if (pathExists("/dev/shm") && mount( "none", - (chrootRootDir + "/dev/shm").c_str(), + (chrootRootDir / "dev" / "shm").c_str(), "tmpfs", 0, - fmt("size=%s", settings.sandboxShmSize).c_str()) + fmt("size=%s", store.config->getLocalSettings().sandboxShmSize).c_str()) == -1) throw SysError("mounting /dev/shm"); @@ -615,25 +636,25 @@ struct ChrootLinuxDerivationBuilder : ChrootDerivationBuilder, LinuxDerivationBu requires the kernel to be compiled with CONFIG_DEVPTS_MULTIPLE_INSTANCES=y (which is the case if /dev/ptx/ptmx exists). */ - if (pathExists("/dev/pts/ptmx") && !pathExists(chrootRootDir + "/dev/ptmx") + if (pathExists("/dev/pts/ptmx") && !pathExists(chrootRootDir / "dev" / "ptmx") && !pathsInChroot.count("/dev/pts")) { - if (mount("none", (chrootRootDir + "/dev/pts").c_str(), "devpts", 0, "newinstance,mode=0620") == 0) { - createSymlink("/dev/pts/ptmx", chrootRootDir + "/dev/ptmx"); + if (mount("none", (chrootRootDir / "dev" / "pts").c_str(), "devpts", 0, "newinstance,mode=0620") == 0) { + createSymlink("/dev/pts/ptmx", chrootRootDir / "dev" / "ptmx"); /* Make sure /dev/pts/ptmx is world-writable. With some Linux versions, it is created with permissions 0. */ - chmod_(chrootRootDir + "/dev/pts/ptmx", 0666); + chmod(chrootRootDir / "dev" / "pts" / "ptmx", 0666); } else { if (errno != EINVAL) throw SysError("mounting /dev/pts"); - doBind("/dev/pts", chrootRootDir + "/dev/pts"); - doBind("/dev/ptmx", chrootRootDir + "/dev/ptmx"); + doBind("/dev/pts", chrootRootDir / "dev" / "pts"); + doBind("/dev/ptmx", chrootRootDir / "dev" / "ptmx"); } } /* Make /etc unwritable */ if (!drvOptions.useUidRange(drv)) - chmod_(chrootRootDir + "/etc", 0555); + chmod(chrootRootDir / "etc", 0555); /* Unshare this mount namespace. This is necessary because pivot_root() below changes the root of the mount @@ -656,16 +677,16 @@ struct ChrootLinuxDerivationBuilder : ChrootDerivationBuilder, LinuxDerivationBu /* Do the chroot(). */ if (chdir(chrootRootDir.c_str()) == -1) - throw SysError("cannot change directory to '%1%'", chrootRootDir); + throw SysError("cannot change directory to %1%", PathFmt(chrootRootDir)); if (mkdir("real-root", 0500) == -1) throw SysError("cannot create real-root directory"); if (pivot_root(".", "real-root") == -1) - throw SysError("cannot pivot old root directory onto '%1%'", (chrootRootDir + "/real-root")); + throw SysError("cannot pivot old root directory onto %1%", PathFmt(chrootRootDir / "real-root")); if (chroot(".") == -1) - throw SysError("cannot change root directory to '%1%'", chrootRootDir); + throw SysError("cannot change root directory to %1%", PathFmt(chrootRootDir)); if (umount2("real-root", MNT_DETACH) == -1) throw SysError("cannot unmount real root filesystem"); @@ -678,13 +699,15 @@ struct ChrootLinuxDerivationBuilder : ChrootDerivationBuilder, LinuxDerivationBu void setUser() override { - /* Switch to the sandbox uid/gid in the user namespace, - which corresponds to the build user or calling user in - the parent namespace. */ - if (setgid(sandboxGid()) == -1) - throw SysError("setgid failed"); - if (setuid(sandboxUid()) == -1) - throw SysError("setuid failed"); + preserveDeathSignal([this]() { + /* Switch to the sandbox uid/gid in the user namespace, + which corresponds to the build user or calling user in + the parent namespace. */ + if (setgid(sandboxGid()) == -1) + throw SysError("setgid failed"); + if (setuid(sandboxUid()) == -1) + throw SysError("setuid failed"); + }); } SingleDrvOutputs unprepareBuild() override @@ -721,10 +744,10 @@ struct ChrootLinuxDerivationBuilder : ChrootDerivationBuilder, LinuxDerivationBu in multithreaded programs. So we do this in a child process.*/ Pid child(startProcess([&]() { - if (usingUserNamespace && (setns(sandboxUserNamespace.get(), 0) == -1)) + if (usingUserNamespace && (setns(sandboxUserNamespace.get(), CLONE_NEWUSER) == -1)) throw SysError("entering sandbox user namespace"); - if (setns(sandboxMountNamespace.get(), 0) == -1) + if (setns(sandboxMountNamespace.get(), CLONE_NEWNS) == -1) throw SysError("entering sandbox mount namespace"); doBind(source, target); diff --git a/src/libstore/unix/include/nix/store/build/hook-instance.hh b/src/libstore/unix/include/nix/store/build/hook-instance.hh index 7657d5dbd082..e53a791d71fc 100644 --- a/src/libstore/unix/include/nix/store/build/hook-instance.hh +++ b/src/libstore/unix/include/nix/store/build/hook-instance.hh @@ -5,6 +5,8 @@ #include "nix/util/serialise.hh" #include "nix/util/processes.hh" +#include + namespace nix { /** @@ -50,7 +52,13 @@ struct HookInstance std::map activities; - HookInstance(); + /** + * Callback to run when the hook process is killed in the destructor. + * Used to call `Worker::childTerminated`. + */ + std::function onKillChild; + + HookInstance(const Strings & buildHook); ~HookInstance(); }; diff --git a/src/libstore/unix/include/nix/store/user-lock.hh b/src/libstore/unix/include/nix/store/user-lock.hh index 828980d6fdb9..0c10906f59a8 100644 --- a/src/libstore/unix/include/nix/store/user-lock.hh +++ b/src/libstore/unix/include/nix/store/user-lock.hh @@ -1,12 +1,15 @@ #pragma once ///@file +#include #include #include #include namespace nix { +struct LocalSettings; + struct UserLock { virtual ~UserLock() {} @@ -36,8 +39,9 @@ struct UserLock * Acquire a user lock for a UID range of size `nrIds`. Note that this * may return nullptr if no user is available. */ -std::unique_ptr acquireUserLock(uid_t nrIds, bool useUserNamespace); +std::unique_ptr acquireUserLock( + const std::filesystem::path & stateDir, const LocalSettings & localSettings, uid_t nrIds, bool useUserNamespace); -bool useBuildUsers(); +bool useBuildUsers(const LocalSettings &); } // namespace nix diff --git a/src/libstore/unix/pathlocks.cc b/src/libstore/unix/pathlocks.cc index 6117b82c8922..ce2adf74debc 100644 --- a/src/libstore/unix/pathlocks.cc +++ b/src/libstore/unix/pathlocks.cc @@ -1,4 +1,5 @@ #include "nix/store/pathlocks.hh" +#include "nix/util/file-system.hh" #include "nix/util/util.hh" #include "nix/util/sync.hh" #include "nix/util/signals.hh" @@ -19,7 +20,7 @@ AutoCloseFD openLockFile(const std::filesystem::path & path, bool create) fd = open(path.c_str(), O_CLOEXEC | O_RDWR | (create ? O_CREAT : 0), 0600); if (!fd && (create || errno != ENOENT)) - throw SysError("opening lock file %1%", path); + throw SysError("opening lock file %1%", PathFmt(path)); return fd; } @@ -30,9 +31,9 @@ void deleteLockFile(const std::filesystem::path & path, Descriptor desc) races. Write a (meaningless) token to the file to indicate to other processes waiting on this lock that the lock is stale (deleted). */ - unlink(path.c_str()); + tryUnlink(path); writeFull(desc, "d"); - /* Note that the result of unlink() is ignored; removing the lock + /* We just try to unlink don't care if it fails; removing the lock file is an optimisation, not a necessity. */ } @@ -81,9 +82,10 @@ bool PathLocks::lockPaths(const std::set & paths, const s preventing deadlocks. */ for (auto & path : paths) { checkInterrupt(); - std::filesystem::path lockPath = path + ".lock"; + auto lockPath = path; + lockPath += ".lock"; - debug("locking path %1%", path); + debug("locking path %1%", PathFmt(path)); AutoCloseFD fd; @@ -106,19 +108,17 @@ bool PathLocks::lockPaths(const std::set & paths, const s } } - debug("lock acquired on %1%", lockPath); + debug("lock acquired on %1%", PathFmt(lockPath)); /* Check that the lock file hasn't become stale (i.e., hasn't been unlinked). */ - struct stat st; - if (fstat(fd.get(), &st) == -1) - throw SysError("statting lock file %1%", lockPath); + auto st = nix::fstat(fd.get()); if (st.st_size != 0) /* This lock file has been unlinked, so we're holding a lock on a deleted file. This means that other processes may create and acquire a lock on `lockPath', and proceed. So we must retry. */ - debug("open lock file %1% has become stale", lockPath); + debug("open lock file %1% has become stale", PathFmt(lockPath)); else break; } @@ -137,9 +137,9 @@ void PathLocks::unlock() deleteLockFile(i.second, i.first); if (close(i.first) == -1) - printError("error (ignored): cannot close lock file on %1%", i.second); + printError("error (ignored): cannot close lock file on %1%", PathFmt(i.second)); - debug("lock released on %1%", i.second); + debug("lock released on %1%", PathFmt(i.second)); } fds.clear(); diff --git a/src/libstore/unix/user-lock.cc b/src/libstore/unix/user-lock.cc index c5e6455e8d9a..c9abdddcea8a 100644 --- a/src/libstore/unix/user-lock.cc +++ b/src/libstore/unix/user-lock.cc @@ -64,15 +64,15 @@ struct SimpleUserLock : UserLock return supplementaryGIDs; } - static std::unique_ptr acquire() + static std::unique_ptr + acquire(const std::filesystem::path & userPoolDir, const std::string & buildUsersGroup) { - assert(settings.buildUsersGroup != ""); - createDirs(settings.nixStateDir + "/userpool"); + assert(buildUsersGroup != ""); /* Get the members of the build-users-group. */ - struct group * gr = getgrnam(settings.buildUsersGroup.get().c_str()); + struct group * gr = getgrnam(buildUsersGroup.c_str()); if (!gr) - throw Error("the group '%s' specified in 'build-users-group' does not exist", settings.buildUsersGroup); + throw Error("the group '%s' specified in 'build-users-group' does not exist", buildUsersGroup); /* Copy the result of getgrnam. */ Strings users; @@ -82,7 +82,7 @@ struct SimpleUserLock : UserLock } if (users.empty()) - throw Error("the build users group '%s' has no members", settings.buildUsersGroup); + throw Error("the build users group '%s' has no members", buildUsersGroup); /* Find a user account that isn't currently in use for another build. */ @@ -91,13 +91,13 @@ struct SimpleUserLock : UserLock struct passwd * pw = getpwnam(i.c_str()); if (!pw) - throw Error("the user '%s' in the group '%s' does not exist", i, settings.buildUsersGroup); + throw Error("the user '%s' in the group '%s' does not exist", i, buildUsersGroup); - auto fnUserLock = fmt("%s/userpool/%s", settings.nixStateDir, pw->pw_uid); + auto fnUserLock = userPoolDir / std::to_string(pw->pw_uid); AutoCloseFD fd = open(fnUserLock.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600); if (!fd) - throw SysError("opening user lock '%s'", fnUserLock); + throw SysError("opening user lock %s", PathFmt(fnUserLock)); if (lockFile(fd.get(), ltWrite, false)) { auto lock = std::make_unique(); @@ -108,7 +108,7 @@ struct SimpleUserLock : UserLock /* Sanity check... */ if (lock->uid == getuid() || lock->uid == geteuid()) - throw Error("the Nix user should not be a member of '%s'", settings.buildUsersGroup); + throw Error("the Nix user should not be a member of '%s'", buildUsersGroup); #ifdef __linux__ /* Get the list of supplementary groups of this user. This is @@ -158,36 +158,37 @@ struct AutoUserLock : UserLock return {}; } - static std::unique_ptr acquire(uid_t nrIds, bool useUserNamespace) + static std::unique_ptr acquire( + const std::filesystem::path & userPoolDir, + const std::string & buildUsersGroup, + uid_t nrIds, + bool useUserNamespace, + const AutoAllocateUidSettings & uidSettings) { #if !defined(__linux__) useUserNamespace = false; #endif experimentalFeatureSettings.require(Xp::AutoAllocateUids); - assert(settings.startId > 0); - assert(settings.uidCount % maxIdsPerBuild == 0); - assert((uint64_t) settings.startId + (uint64_t) settings.uidCount <= std::numeric_limits::max()); + assert(uidSettings.startId > 0); + assert(uidSettings.uidCount % maxIdsPerBuild == 0); + assert((uint64_t) uidSettings.startId + (uint64_t) uidSettings.uidCount <= std::numeric_limits::max()); assert(nrIds <= maxIdsPerBuild); - createDirs(settings.nixStateDir + "/userpool2"); - - size_t nrSlots = settings.uidCount / maxIdsPerBuild; + size_t nrSlots = uidSettings.uidCount / maxIdsPerBuild; for (size_t i = 0; i < nrSlots; i++) { debug("trying user slot '%d'", i); - createDirs(settings.nixStateDir + "/userpool2"); - - auto fnUserLock = fmt("%s/userpool2/slot-%d", settings.nixStateDir, i); + auto fnUserLock = userPoolDir / fmt("slot-%d", i); AutoCloseFD fd = open(fnUserLock.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600); if (!fd) - throw SysError("opening user lock '%s'", fnUserLock); + throw SysError("opening user lock %s", PathFmt(fnUserLock)); if (lockFile(fd.get(), ltWrite, false)) { - auto firstUid = settings.startId + i * maxIdsPerBuild; + auto firstUid = uidSettings.startId + i * maxIdsPerBuild; auto pw = getpwuid(firstUid); if (pw) @@ -199,10 +200,9 @@ struct AutoUserLock : UserLock if (useUserNamespace) lock->firstGid = firstUid; else { - struct group * gr = getgrnam(settings.buildUsersGroup.get().c_str()); + struct group * gr = getgrnam(buildUsersGroup.c_str()); if (!gr) - throw Error( - "the group '%s' specified in 'build-users-group' does not exist", settings.buildUsersGroup); + throw Error("the group '%s' specified in 'build-users-group' does not exist", buildUsersGroup); lock->firstGid = gr->gr_gid; } lock->nrIds = nrIds; @@ -214,21 +214,27 @@ struct AutoUserLock : UserLock } }; -std::unique_ptr acquireUserLock(uid_t nrIds, bool useUserNamespace) +std::unique_ptr acquireUserLock( + const std::filesystem::path & stateDir, const LocalSettings & localSettings, uid_t nrIds, bool useUserNamespace) { - if (settings.autoAllocateUids) - return AutoUserLock::acquire(nrIds, useUserNamespace); - else - return SimpleUserLock::acquire(); + if (auto * uidSettings = localSettings.getAutoAllocateUidSettings()) { + auto userPoolDir = stateDir / "userpool2"; + createDirs(userPoolDir); + return AutoUserLock::acquire(userPoolDir, localSettings.buildUsersGroup, nrIds, useUserNamespace, *uidSettings); + } else { + auto userPoolDir = stateDir / "userpool"; + createDirs(userPoolDir); + return SimpleUserLock::acquire(userPoolDir, localSettings.buildUsersGroup); + } } -bool useBuildUsers() +bool useBuildUsers(const LocalSettings & localSettings) { #ifdef __linux__ - static bool b = (settings.buildUsersGroup != "" || settings.autoAllocateUids) && isRootUser(); + static bool b = (localSettings.buildUsersGroup != "" || localSettings.autoAllocateUids) && isRootUser(); return b; #elif defined(__APPLE__) || defined(__FreeBSD__) - static bool b = settings.buildUsersGroup != "" && isRootUser(); + static bool b = localSettings.buildUsersGroup != "" && isRootUser(); return b; #else return false; diff --git a/src/libstore/windows/pathlocks.cc b/src/libstore/windows/pathlocks.cc index 32d9e7c0fa8d..8021bb072a22 100644 --- a/src/libstore/windows/pathlocks.cc +++ b/src/libstore/windows/pathlocks.cc @@ -1,13 +1,14 @@ +#include "nix/util/file-system.hh" #include "nix/util/logging.hh" #include "nix/store/pathlocks.hh" #include "nix/util/signals.hh" #include "nix/util/util.hh" +#include "nix/util/windows-environment.hh" #ifdef _WIN32 # include # include # include -# include "nix/util/windows-error.hh" namespace nix { @@ -18,7 +19,7 @@ void deleteLockFile(const std::filesystem::path & path, Descriptor desc) int exit = DeleteFileW(path.c_str()); if (exit == 0) - warn("%s: &s", path, std::to_string(GetLastError())); + warn("%s: %s", PathFmt(path), std::to_string(GetLastError())); } void PathLocks::unlock() @@ -28,9 +29,9 @@ void PathLocks::unlock() deleteLockFile(i.second, i.first); if (CloseHandle(i.first) == -1) - printError("error (ignored): cannot close lock file on %1%", i.second); + printError("error (ignored): cannot close lock file on %1%", PathFmt(i.second)); - debug("lock released on %1%", i.second); + debug("lock released on %1%", PathFmt(i.second)); } fds.clear(); @@ -47,36 +48,48 @@ AutoCloseFD openLockFile(const std::filesystem::path & path, bool create) FILE_ATTRIBUTE_NORMAL | FILE_FLAG_POSIX_SEMANTICS, NULL); if (desc.get() == INVALID_HANDLE_VALUE) - warn("%s: %s", path, std::to_string(GetLastError())); + warn("%s: %s", PathFmt(path), std::to_string(GetLastError())); return desc; } +/** + * Throw a WinError, or if running under Wine, just warn and return true. + * Wine has incomplete file locking support, so we degrade gracefully. + */ +template +static bool warnOrThrowWine(DWORD lastError, const std::string & fs, const Args &... args) +{ + if (isWine()) { + warn(fs + ": %s (ignored under Wine)", args..., lastError); + return true; + } + throw WinError(lastError, fs, args...); +} + bool lockFile(Descriptor desc, LockType lockType, bool wait) { switch (lockType) { case ltNone: { OVERLAPPED ov = {0}; - if (!UnlockFileEx(desc, 0, 2, 0, &ov)) { - WinError winError("Failed to unlock file desc %s", desc); - throw winError; - } + if (!UnlockFileEx(desc, 0, 2, 0, &ov)) + return warnOrThrowWine(GetLastError(), "Failed to unlock file %s", PathFmt(descriptorToPath(desc))); return true; } case ltRead: { OVERLAPPED ov = {0}; if (!LockFileEx(desc, wait ? 0 : LOCKFILE_FAIL_IMMEDIATELY, 0, 1, 0, &ov)) { - WinError winError("Failed to lock file desc %s", desc); - if (winError.lastError == ERROR_LOCK_VIOLATION) + auto lastError = GetLastError(); + if (lastError == ERROR_LOCK_VIOLATION) return false; - throw winError; + return warnOrThrowWine(lastError, "Failed to lock file %s", PathFmt(descriptorToPath(desc))); } ov.Offset = 1; if (!UnlockFileEx(desc, 0, 1, 0, &ov)) { - WinError winError("Failed to unlock file desc %s", desc); - if (winError.lastError != ERROR_NOT_LOCKED) - throw winError; + auto lastError = GetLastError(); + if (lastError != ERROR_NOT_LOCKED) + return warnOrThrowWine(lastError, "Failed to unlock file %s", PathFmt(descriptorToPath(desc))); } return true; } @@ -84,17 +97,17 @@ bool lockFile(Descriptor desc, LockType lockType, bool wait) OVERLAPPED ov = {0}; ov.Offset = 1; if (!LockFileEx(desc, LOCKFILE_EXCLUSIVE_LOCK | (wait ? 0 : LOCKFILE_FAIL_IMMEDIATELY), 0, 1, 0, &ov)) { - WinError winError("Failed to lock file desc %s", desc); - if (winError.lastError == ERROR_LOCK_VIOLATION) + auto lastError = GetLastError(); + if (lastError == ERROR_LOCK_VIOLATION) return false; - throw winError; + return warnOrThrowWine(lastError, "Failed to lock file %s", PathFmt(descriptorToPath(desc))); } ov.Offset = 0; if (!UnlockFileEx(desc, 0, 1, 0, &ov)) { - WinError winError("Failed to unlock file desc %s", desc); - if (winError.lastError != ERROR_NOT_LOCKED) - throw winError; + auto lastError = GetLastError(); + if (lastError != ERROR_NOT_LOCKED) + return warnOrThrowWine(lastError, "Failed to unlock file %s", PathFmt(descriptorToPath(desc))); } return true; } @@ -111,7 +124,7 @@ bool PathLocks::lockPaths(const std::set & paths, const s checkInterrupt(); std::filesystem::path lockPath = path; lockPath += L".lock"; - debug("locking path %1%", path); + debug("locking path %1%", PathFmt(path)); AutoCloseFD fd; @@ -128,13 +141,10 @@ bool PathLocks::lockPaths(const std::set & paths, const s } } - debug("lock acquired on %1%", lockPath); + debug("lock acquired on %1%", PathFmt(lockPath)); - struct _stat st; - if (_fstat(fromDescriptorReadOnly(fd.get()), &st) == -1) - throw SysError("statting lock file %1%", lockPath); - if (st.st_size != 0) - debug("open lock file %1% has become stale", lockPath); + if (getFileSize(fd.get()) != 0) + debug("open lock file %1% has become stale", PathFmt(lockPath)); else break; } diff --git a/src/libstore/worker-protocol-connection.cc b/src/libstore/worker-protocol-connection.cc index 7f41b0c47e7e..dcc698293a30 100644 --- a/src/libstore/worker-protocol-connection.cc +++ b/src/libstore/worker-protocol-connection.cc @@ -5,9 +5,6 @@ namespace nix { -const WorkerProto::FeatureSet WorkerProto::allFeatures{ - {std::string(WorkerProto::featureQueryActiveBuilds), std::string(WorkerProto::featureProvenance)}}; - WorkerProto::BasicClientConnection::~BasicClientConnection() { try { @@ -65,7 +62,7 @@ WorkerProto::BasicClientConnection::processStderrReturn(Sink * sink, Source * so } else if (msg == STDERR_ERROR) { - if (GET_PROTOCOL_MINOR(protoVersion) >= 26) { + if (protoVersion >= WorkerProto::Version{.number = {1, 26}}) { ex = std::make_exception_ptr(readError(from)); } else { auto error = readString(from); @@ -123,7 +120,7 @@ WorkerProto::BasicClientConnection::processStderrReturn(Sink * sink, Source * so // explain to users what's going on when their daemon is // older than #4628 (2023). if (experimentalFeatureSettings.isEnabled(Xp::DynamicDerivations) - && GET_PROTOCOL_MINOR(protoVersion) <= 35) { + && protoVersion.number < WorkerProto::Version::Number{1, 36}) { auto m = e.msg(); if (m.find("parsing derivation") != std::string::npos && m.find("expected string") != std::string::npos && m.find("Derive([") != std::string::npos) @@ -147,86 +144,87 @@ void WorkerProto::BasicClientConnection::processStderr( } } -static WorkerProto::FeatureSet intersectFeatures(const WorkerProto::FeatureSet & a, const WorkerProto::FeatureSet & b) +static WorkerProto::Version::FeatureSet +intersectFeatures(const WorkerProto::Version::FeatureSet & a, const WorkerProto::Version::FeatureSet & b) { - WorkerProto::FeatureSet res; + WorkerProto::Version::FeatureSet res; for (auto & x : a) if (b.contains(x)) res.insert(x); return res; } -std::tuple WorkerProto::BasicClientConnection::handshake( - BufferedSink & to, - Source & from, - WorkerProto::Version localVersion, - const WorkerProto::FeatureSet & supportedFeatures) +WorkerProto::Version WorkerProto::BasicClientConnection::handshake( + BufferedSink & to, Source & from, const WorkerProto::Version & localVersion) { - to << WORKER_MAGIC_1 << localVersion; + to << WORKER_MAGIC_1 << localVersion.number.toWire(); to.flush(); unsigned int magic = readInt(from); if (magic != WORKER_MAGIC_2) throw Error("nix-daemon protocol mismatch from"); - auto daemonVersion = readInt(from); + auto daemonVersion = WorkerProto::Version::Number::fromWire(readInt(from)); - if (GET_PROTOCOL_MAJOR(daemonVersion) != GET_PROTOCOL_MAJOR(PROTOCOL_VERSION)) + if (daemonVersion.major != WorkerProto::latest.number.major) throw Error("Nix daemon protocol version not supported"); - if (GET_PROTOCOL_MINOR(daemonVersion) < 10) + if (daemonVersion < WorkerProto::Version::Number{1, 10}) throw Error("the Nix daemon version is too old"); - auto protoVersion = std::min(daemonVersion, localVersion); + auto protoVersionNumber = std::min(daemonVersion, localVersion.number); /* Exchange features. */ - WorkerProto::FeatureSet daemonFeatures; - if (GET_PROTOCOL_MINOR(protoVersion) >= 38) { - to << supportedFeatures; + WorkerProto::Version::FeatureSet daemonFeatures; + if (protoVersionNumber >= WorkerProto::Version::Number{1, 38}) { + to << localVersion.features; to.flush(); - daemonFeatures = readStrings(from); + daemonFeatures = readStrings(from); } - return {protoVersion, intersectFeatures(daemonFeatures, supportedFeatures)}; + return { + .number = protoVersionNumber, + .features = intersectFeatures(daemonFeatures, localVersion.features), + }; } -std::tuple WorkerProto::BasicServerConnection::handshake( - BufferedSink & to, - Source & from, - WorkerProto::Version localVersion, - const WorkerProto::FeatureSet & supportedFeatures) +WorkerProto::Version WorkerProto::BasicServerConnection::handshake( + BufferedSink & to, Source & from, const WorkerProto::Version & localVersion) { unsigned int magic = readInt(from); if (magic != WORKER_MAGIC_1) throw Error("protocol mismatch"); - to << WORKER_MAGIC_2 << localVersion; + to << WORKER_MAGIC_2 << localVersion.number.toWire(); to.flush(); - auto clientVersion = readInt(from); + auto clientVersion = WorkerProto::Version::Number::fromWire(readInt(from)); - auto protoVersion = std::min(clientVersion, localVersion); + auto protoVersionNumber = std::min(clientVersion, localVersion.number); /* Exchange features. */ - WorkerProto::FeatureSet clientFeatures; - if (GET_PROTOCOL_MINOR(protoVersion) >= 38) { - clientFeatures = readStrings(from); - to << supportedFeatures; + WorkerProto::Version::FeatureSet clientFeatures; + if (protoVersionNumber >= WorkerProto::Version::Number{1, 38}) { + clientFeatures = readStrings(from); + to << localVersion.features; to.flush(); } - return {protoVersion, intersectFeatures(clientFeatures, supportedFeatures)}; + return { + .number = protoVersionNumber, + .features = intersectFeatures(clientFeatures, localVersion.features), + }; } WorkerProto::ClientHandshakeInfo WorkerProto::BasicClientConnection::postHandshake(const StoreDirConfig & store) { WorkerProto::ClientHandshakeInfo res; - if (GET_PROTOCOL_MINOR(protoVersion) >= 14) { + if (protoVersion >= WorkerProto::Version{.number = {1, 14}}) { // Obsolete CPU affinity. to << 0; } - if (GET_PROTOCOL_MINOR(protoVersion) >= 11) + if (protoVersion >= WorkerProto::Version{.number = {1, 11}}) to << false; // obsolete reserveSpace - if (GET_PROTOCOL_MINOR(protoVersion) >= 33) + if (protoVersion >= WorkerProto::Version{.number = {1, 33}}) to.flush(); return WorkerProto::Serialise::read(store, *this); @@ -234,12 +232,12 @@ WorkerProto::ClientHandshakeInfo WorkerProto::BasicClientConnection::postHandsha void WorkerProto::BasicServerConnection::postHandshake(const StoreDirConfig & store, const ClientHandshakeInfo & info) { - if (GET_PROTOCOL_MINOR(protoVersion) >= 14 && readInt(from)) { + if (protoVersion >= WorkerProto::Version{.number = {1, 14}} && readInt(from)) { // Obsolete CPU affinity. readInt(from); } - if (GET_PROTOCOL_MINOR(protoVersion) >= 11) + if (protoVersion >= WorkerProto::Version{.number = {1, 11}}) readInt(from); // obsolete reserveSpace WorkerProto::write(store, *this, info); @@ -257,7 +255,7 @@ std::optional WorkerProto::BasicClientConnection::queryPat return std::nullopt; throw; } - if (GET_PROTOCOL_MINOR(protoVersion) >= 17) { + if (protoVersion >= WorkerProto::Version{.number = {1, 17}}) { bool valid; from >> valid; if (!valid) @@ -269,10 +267,10 @@ std::optional WorkerProto::BasicClientConnection::queryPat StorePathSet WorkerProto::BasicClientConnection::queryValidPaths( const StoreDirConfig & store, bool * daemonException, const StorePathSet & paths, SubstituteFlag maybeSubstitute) { - assert(GET_PROTOCOL_MINOR(protoVersion) >= 12); + assert((protoVersion >= WorkerProto::Version{.number = {1, 12}})); to << WorkerProto::Op::QueryValidPaths; WorkerProto::write(store, *this, paths); - if (GET_PROTOCOL_MINOR(protoVersion) >= 27) { + if (protoVersion >= WorkerProto::Version{.number = {1, 27}}) { to << maybeSubstitute; } processStderr(daemonException); @@ -306,12 +304,12 @@ WorkerProto::BasicClientConnection::getBuildDerivationResponse(const StoreDirCon } void WorkerProto::BasicClientConnection::narFromPath( - const StoreDirConfig & store, bool * daemonException, const StorePath & path, std::function fun) + const StoreDirConfig & store, bool * daemonException, const StorePath & path, fun receiveNar) { to << WorkerProto::Op::NarFromPath << store.printStorePath(path); processStderr(daemonException); - fun(from); + receiveNar(from); } } // namespace nix diff --git a/src/libstore/worker-protocol.cc b/src/libstore/worker-protocol.cc index 6dc1f837102e..3c7acbfb4784 100644 --- a/src/libstore/worker-protocol.cc +++ b/src/libstore/worker-protocol.cc @@ -3,6 +3,7 @@ #include "nix/store/store-api.hh" #include "nix/store/gc-store.hh" #include "nix/store/build-result.hh" +#include "nix/store/common-protocol.hh" #include "nix/store/worker-protocol.hh" #include "nix/store/worker-protocol-impl.hh" #include "nix/util/archive.hh" @@ -14,6 +15,43 @@ namespace nix { +const WorkerProto::Version WorkerProto::latest = { + .number = + { + .major = 1, + .minor = 38, + }, + .features = + { + std::string{WorkerProto::featureQueryActiveBuilds}, + std::string{WorkerProto::featureProvenance}, + std::string{WorkerProto::featureVersionedAddToStoreMultiple}, + }, +}; + +const WorkerProto::Version WorkerProto::minimum = { + .number = + { + .major = 1, + .minor = 18, + }, +}; + +std::partial_ordering WorkerProto::Version::operator<=>(const WorkerProto::Version & other) const +{ + auto numCmp = number <=> other.number; + bool thisSubsetEq = std::includes(other.features.begin(), other.features.end(), features.begin(), features.end()); + bool otherSubsetEq = std::includes(features.begin(), features.end(), other.features.begin(), other.features.end()); + + if (numCmp == 0 && thisSubsetEq && otherSubsetEq) + return std::partial_ordering::equivalent; + if (numCmp <= 0 && thisSubsetEq) + return std::partial_ordering::less; + if (numCmp >= 0 && otherSubsetEq) + return std::partial_ordering::greater; + return std::partial_ordering::unordered; +} + /* protocol-specific definitions */ BuildMode WorkerProto::Serialise::read(const StoreDirConfig & store, WorkerProto::ReadConn conn) @@ -153,7 +191,7 @@ void WorkerProto::Serialise>::write( DerivedPath WorkerProto::Serialise::read(const StoreDirConfig & store, WorkerProto::ReadConn conn) { auto s = readString(conn.from); - if (GET_PROTOCOL_MINOR(conn.version) >= 30) { + if (conn.version >= WorkerProto::Version{.number = {1, 30}}) { return DerivedPath::parseLegacy(store, s); } else { return parsePathWithOutputs(store, s).toDerivedPath(); @@ -163,7 +201,7 @@ DerivedPath WorkerProto::Serialise::read(const StoreDirConfig & sto void WorkerProto::Serialise::write( const StoreDirConfig & store, WorkerProto::WriteConn conn, const DerivedPath & req) { - if (GET_PROTOCOL_MINOR(conn.version) >= 30) { + if (conn.version >= WorkerProto::Version{.number = {1, 30}}) { conn.to << req.to_string_legacy(store); } else { auto sOrDrvPath = StorePathWithOutputs::tryFromDerivedPath(req); @@ -174,8 +212,8 @@ void WorkerProto::Serialise::write( throw Error( "trying to request '%s', but daemon protocol %d.%d is too old (< 1.29) to request a derivation file", store.printStorePath(drvPath), - GET_PROTOCOL_MAJOR(conn.version), - GET_PROTOCOL_MINOR(conn.version)); + conn.version.number.major, + conn.version.number.minor); }, [&](std::monostate) { throw Error( @@ -208,31 +246,42 @@ BuildResult WorkerProto::Serialise::read(const StoreDirConfig & sto { BuildResult res; BuildResult::Success success; - BuildResult::Failure failure; - auto rawStatus = readInt(conn.from); - conn.from >> failure.errorMsg; + // Temp variables for failure fields since BuildError uses methods + std::string errorMsg; + bool isNonDeterministic = false; - if (GET_PROTOCOL_MINOR(conn.version) >= 29) { - conn.from >> res.timesBuilt >> failure.isNonDeterministic >> res.startTime >> res.stopTime; + auto status = WorkerProto::Serialise::read(store, conn); + conn.from >> errorMsg; + + if (conn.version >= WorkerProto::Version{.number = {1, 29}}) { + conn.from >> res.timesBuilt >> isNonDeterministic >> res.startTime >> res.stopTime; } - if (GET_PROTOCOL_MINOR(conn.version) >= 37) { + if (conn.version >= WorkerProto::Version{.number = {1, 37}}) { res.cpuUser = WorkerProto::Serialise>::read(store, conn); res.cpuSystem = WorkerProto::Serialise>::read(store, conn); } - if (GET_PROTOCOL_MINOR(conn.version) >= 28) { + if (conn.version >= WorkerProto::Version{.number = {1, 28}}) { auto builtOutputs = WorkerProto::Serialise::read(store, conn); for (auto && [output, realisation] : builtOutputs) success.builtOutputs.insert_or_assign(std::move(output.outputName), std::move(realisation)); } - if (BuildResult::Success::statusIs(rawStatus)) { - success.status = static_cast(rawStatus); - res.inner = std::move(success); - } else { - failure.status = static_cast(rawStatus); - res.inner = std::move(failure); - } + res.inner = std::visit( + overloaded{ + [&](BuildResult::Success::Status s) -> decltype(res.inner) { + success.status = s; + return std::move(success); + }, + [&](BuildResult::Failure::Status s) -> decltype(res.inner) { + return BuildResult::Failure{{ + .status = s, + .msg = HintFmt(std::move(errorMsg)), + .isNonDeterministic = isNonDeterministic, + }}; + }, + }, + status); return res; } @@ -247,14 +296,14 @@ void WorkerProto::Serialise::write( default value for the fields that don't exist in that case. */ auto common = [&](std::string_view errorMsg, bool isNonDeterministic, const auto & builtOutputs) { conn.to << errorMsg; - if (GET_PROTOCOL_MINOR(conn.version) >= 29) { + if (conn.version >= WorkerProto::Version{.number = {1, 29}}) { conn.to << res.timesBuilt << isNonDeterministic << res.startTime << res.stopTime; } - if (GET_PROTOCOL_MINOR(conn.version) >= 37) { + if (conn.version >= WorkerProto::Version{.number = {1, 37}}) { WorkerProto::write(store, conn, res.cpuUser); WorkerProto::write(store, conn, res.cpuSystem); } - if (GET_PROTOCOL_MINOR(conn.version) >= 28) { + if (conn.version >= WorkerProto::Version{.number = {1, 28}}) { DrvOutputs builtOutputsFullKey; for (auto & [output, realisation] : builtOutputs) builtOutputsFullKey.insert_or_assign(realisation.id, realisation); @@ -264,11 +313,11 @@ void WorkerProto::Serialise::write( std::visit( overloaded{ [&](const BuildResult::Failure & failure) { - conn.to << failure.status; - common(failure.errorMsg, failure.isNonDeterministic, decltype(BuildResult::Success::builtOutputs){}); + WorkerProto::write(store, conn, BuildResultStatus{failure.status}); + common(failure.message(), failure.isNonDeterministic, decltype(BuildResult::Success::builtOutputs){}); }, [&](const BuildResult::Success & success) { - conn.to << success.status; + WorkerProto::write(store, conn, BuildResultStatus{success.status}); common(/*errorMsg=*/"", /*isNonDeterministic=*/false, success.builtOutputs); }, }, @@ -299,12 +348,12 @@ UnkeyedValidPathInfo WorkerProto::Serialise::read(const St info.deriver = std::move(deriver); info.references = WorkerProto::Serialise::read(store, conn); conn.from >> info.registrationTime >> info.narSize; - if (GET_PROTOCOL_MINOR(conn.version) >= 16) { + if (conn.version >= WorkerProto::Version{.number = {1, 16}}) { conn.from >> info.ultimate; - info.sigs = readStrings(conn.from); + info.sigs = WorkerProto::Serialise>::read(store, conn); info.ca = ContentAddress::parseOpt(readString(conn.from)); } - if (conn.provenance) + if (conn.version.features.contains(WorkerProto::featureProvenance)) info.provenance = Provenance::from_json_str_optional(readString(conn.from)); return info; } @@ -316,10 +365,12 @@ void WorkerProto::Serialise::write( conn.to << pathInfo.narHash.to_string(HashFormat::Base16, false); WorkerProto::write(store, conn, pathInfo.references); conn.to << pathInfo.registrationTime << pathInfo.narSize; - if (GET_PROTOCOL_MINOR(conn.version) >= 16) { - conn.to << pathInfo.ultimate << pathInfo.sigs << renderContentAddress(pathInfo.ca); + if (conn.version >= WorkerProto::Version{.number = {1, 16}}) { + conn.to << pathInfo.ultimate; + WorkerProto::write(store, conn, pathInfo.sigs); + conn.to << renderContentAddress(pathInfo.ca); } - if (conn.provenance) + if (conn.version.features.contains(WorkerProto::featureProvenance)) conn.to << (pathInfo.provenance ? pathInfo.provenance->to_json_str() : ""); } @@ -328,11 +379,11 @@ WorkerProto::Serialise::read(const StoreDirCon { WorkerProto::ClientHandshakeInfo res; - if (GET_PROTOCOL_MINOR(conn.version) >= 33) { + if (conn.version >= WorkerProto::Version{.number = {1, 33}}) { res.daemonNixVersion = readString(conn.from); } - if (GET_PROTOCOL_MINOR(conn.version) >= 35) { + if (conn.version >= WorkerProto::Version{.number = {1, 35}}) { res.remoteTrustsUs = WorkerProto::Serialise>::read(store, conn); } else { // We don't know the answer; protocol to old. @@ -345,12 +396,12 @@ WorkerProto::Serialise::read(const StoreDirCon void WorkerProto::Serialise::write( const StoreDirConfig & store, WriteConn conn, const WorkerProto::ClientHandshakeInfo & info) { - if (GET_PROTOCOL_MINOR(conn.version) >= 33) { + if (conn.version >= WorkerProto::Version{.number = {1, 33}}) { assert(info.daemonNixVersion); conn.to << *info.daemonNixVersion; } - if (GET_PROTOCOL_MINOR(conn.version) >= 35) { + if (conn.version >= WorkerProto::Version{.number = {1, 35}}) { WorkerProto::write(store, conn, info.remoteTrustsUs); } } diff --git a/src/libutil-c/nix_api_util.h b/src/libutil-c/nix_api_util.h index d301e5743cfd..66ea17522213 100644 --- a/src/libutil-c/nix_api_util.h +++ b/src/libutil-c/nix_api_util.h @@ -109,6 +109,13 @@ enum nix_err { */ NIX_ERR_NIX_ERROR = -4, + /** + * @brief A recoverable error occurred. + * + * This is used primarily by C API *consumers* to communicate that a failed + * primop call should be retried on the next evaluation attempt. + */ + NIX_ERR_RECOVERABLE = -5, }; typedef enum nix_err nix_err; diff --git a/src/libutil-test-support/include/nix/util/tests/characterization.hh b/src/libutil-test-support/include/nix/util/tests/characterization.hh index d8fad1df9258..6dd5f38866db 100644 --- a/src/libutil-test-support/include/nix/util/tests/characterization.hh +++ b/src/libutil-test-support/include/nix/util/tests/characterization.hh @@ -4,20 +4,11 @@ #include #include "nix/util/types.hh" -#include "nix/util/environment-variables.hh" #include "nix/util/file-system.hh" +#include "nix/util/tests/test-data.hh" namespace nix { -/** - * The path to the unit test data directory. See the contributing guide - * in the manual for further details. - */ -static inline std::filesystem::path getUnitTestData() -{ - return getEnv("_NIX_TEST_UNIT_DATA").value(); -} - /** * Whether we should update "golden masters" instead of running tests * against them. See the contributing guide in the manual for further @@ -37,7 +28,7 @@ struct CharacterizationTest : virtual ::testing::Test * While the "golden master" for this characterization test is * located. It should not be shared with any other test. */ - virtual std::filesystem::path goldenMaster(PathView testStem) const = 0; + virtual std::filesystem::path goldenMaster(std::string_view testStem) const = 0; /** * Golden test for reading @@ -45,7 +36,7 @@ struct CharacterizationTest : virtual ::testing::Test * @param test hook that takes the contents of the file and does the * actual work */ - void readTest(PathView testStem, auto && test) + void readTest(std::string_view testStem, auto && test) { auto file = goldenMaster(testStem); @@ -62,7 +53,7 @@ struct CharacterizationTest : virtual ::testing::Test * @param test hook that produces contents of the file and does the * actual work */ - void writeTest(PathView testStem, auto && test, auto && readFile2, auto && writeFile2) + void writeTest(std::string_view testStem, auto && test, auto && readFile2, auto && writeFile2) { auto file = goldenMaster(testStem); @@ -81,7 +72,7 @@ struct CharacterizationTest : virtual ::testing::Test /** * Specialize to `std::string` */ - void writeTest(PathView testStem, auto && test) + void writeTest(std::string_view testStem, auto && test) { writeTest( testStem, diff --git a/src/libutil-test-support/include/nix/util/tests/json-characterization.hh b/src/libutil-test-support/include/nix/util/tests/json-characterization.hh index 6db32c4b6c3e..d0f4f9c495ad 100644 --- a/src/libutil-test-support/include/nix/util/tests/json-characterization.hh +++ b/src/libutil-test-support/include/nix/util/tests/json-characterization.hh @@ -16,10 +16,10 @@ namespace nix { * Golden test for JSON reading */ template -void readJsonTest(CharacterizationTest & test, PathView testStem, const T & expected, auto... args) +void readJsonTest(CharacterizationTest & test, std::string_view testStem, const T & expected, auto... args) { using namespace nlohmann; - test.readTest(Path{testStem} + ".json", [&](const auto & encodedRaw) { + test.readTest(std::string{testStem} + ".json", [&](const auto & encodedRaw) { auto encoded = json::parse(encodedRaw); T decoded = adl_serializer::from_json(encoded, args...); ASSERT_EQ(decoded, expected); @@ -30,11 +30,11 @@ void readJsonTest(CharacterizationTest & test, PathView testStem, const T & expe * Golden test for JSON writing */ template -void writeJsonTest(CharacterizationTest & test, PathView testStem, const T & value) +void writeJsonTest(CharacterizationTest & test, std::string_view testStem, const T & value) { using namespace nlohmann; test.writeTest( - Path{testStem} + ".json", + std::string{testStem} + ".json", [&]() -> json { return static_cast(value); }, [](const auto & file) { return json::parse(readFile(file)); }, [](const auto & file, const auto & got) { return writeFile(file, got.dump(2) + "\n"); }); @@ -49,11 +49,11 @@ void writeJsonTest(CharacterizationTest & test, PathView testStem, const T & val * pointer), so we break the symmetry as the best remaining option. */ template -void writeJsonTest(CharacterizationTest & test, PathView testStem, const ref & value) +void writeJsonTest(CharacterizationTest & test, std::string_view testStem, const ref & value) { using namespace nlohmann; test.writeTest( - Path{testStem} + ".json", + std::string{testStem} + ".json", [&]() -> json { return static_cast(*value); }, [](const auto & file) { return json::parse(readFile(file)); }, [](const auto & file, const auto & got) { return writeFile(file, got.dump(2) + "\n"); }); @@ -63,11 +63,11 @@ void writeJsonTest(CharacterizationTest & test, PathView testStem, const ref * Golden test in the middle of something */ template -void checkpointJson(CharacterizationTest & test, PathView testStem, const T & got) +void checkpointJson(CharacterizationTest & test, std::string_view testStem, const T & got) { using namespace nlohmann; - auto file = test.goldenMaster(Path{testStem} + ".json"); + auto file = test.goldenMaster(std::string{testStem} + ".json"); json gotJson = static_cast(got); @@ -83,6 +83,31 @@ void checkpointJson(CharacterizationTest & test, PathView testStem, const T & go } } +/** + * Specialization for when we need to do "JSON -> `ref`" in one + * direction, but "`const T &` -> JSON" in the other direction. + */ +template +void checkpointJson(CharacterizationTest & test, std::string_view testStem, const ref & got) +{ + using namespace nlohmann; + + auto file = test.goldenMaster(std::string{testStem} + ".json"); + + json gotJson = static_cast(*got); + + if (testAccept()) { + std::filesystem::create_directories(file.parent_path()); + writeFile(file, gotJson.dump(2) + "\n"); + ADD_FAILURE() << "Updating golden master " << file; + } else { + json expectedJson = json::parse(readFile(file)); + ASSERT_EQ(gotJson, expectedJson); + ref expected = adl_serializer>::from_json(expectedJson); + ASSERT_EQ(*got, *expected); + } +} + /** * Mixin class for writing characterization tests for `nlohmann::json` * conversions for a given type. @@ -96,7 +121,7 @@ struct JsonCharacterizationTest : virtual CharacterizationTest * @param test hook that takes the contents of the file and does the * actual work */ - void readJsonTest(PathView testStem, const T & expected, auto... args) + void readJsonTest(std::string_view testStem, const T & expected, auto... args) { nix::readJsonTest(*this, testStem, expected, args...); } @@ -107,12 +132,12 @@ struct JsonCharacterizationTest : virtual CharacterizationTest * @param test hook that produces contents of the file and does the * actual work */ - void writeJsonTest(PathView testStem, const T & value) + void writeJsonTest(std::string_view testStem, const T & value) { nix::writeJsonTest(*this, testStem, value); } - void checkpointJson(PathView testStem, const T & value) + void checkpointJson(std::string_view testStem, const T & value) { nix::checkpointJson(*this, testStem, value); } diff --git a/src/libutil-test-support/include/nix/util/tests/meson.build b/src/libutil-test-support/include/nix/util/tests/meson.build index 3be085892c97..9f09183f33f9 100644 --- a/src/libutil-test-support/include/nix/util/tests/meson.build +++ b/src/libutil-test-support/include/nix/util/tests/meson.build @@ -10,4 +10,5 @@ headers = files( 'json-characterization.hh', 'nix_api_util.hh', 'string_callback.hh', + 'test-data.hh', ) diff --git a/src/libutil-test-support/include/nix/util/tests/test-data.hh b/src/libutil-test-support/include/nix/util/tests/test-data.hh new file mode 100644 index 000000000000..6b965b848a91 --- /dev/null +++ b/src/libutil-test-support/include/nix/util/tests/test-data.hh @@ -0,0 +1,24 @@ +#pragma once +///@file + +#include +#include "nix/util/environment-variables.hh" +#include "nix/util/error.hh" + +namespace nix { + +/** + * The path to the unit test data directory. See the contributing guide + * in the manual for further details. + */ +static inline std::filesystem::path getUnitTestData() +{ + auto data = getEnv("_NIX_TEST_UNIT_DATA"); + if (!data) + throw Error( + "_NIX_TEST_UNIT_DATA environment variable is not set. " + "Recommendation: use meson, example: 'meson test -C build --gdb'"); + return std::filesystem::path(*data); +} + +} // namespace nix diff --git a/src/libutil-tests/canon-path.cc b/src/libutil-tests/canon-path.cc index aae9285c4e3b..ac40d6b0e61d 100644 --- a/src/libutil-tests/canon-path.cc +++ b/src/libutil-tests/canon-path.cc @@ -51,6 +51,19 @@ TEST(CanonPath, nullBytes) ASSERT_THROW(CanonPath(s, CanonPath::root), BadCanonPath); } +TEST(CanonPath, fromFilename) +{ + ASSERT_THROW(CanonPath::fromFilename("."), BadCanonPath); + ASSERT_THROW(CanonPath::fromFilename(".."), BadCanonPath); + ASSERT_THROW(CanonPath::fromFilename(""), BadCanonPath); + ASSERT_THROW(CanonPath::fromFilename("/.."), BadCanonPath); + ASSERT_THROW(CanonPath::fromFilename("/."), BadCanonPath); + ASSERT_THROW(CanonPath::fromFilename("/abc"), BadCanonPath); + ASSERT_THROW(CanonPath::fromFilename("abc/d"), BadCanonPath); + ASSERT_THROW(CanonPath::fromFilename("/abc/d"), BadCanonPath); + ASSERT_EQ(CanonPath::fromFilename("abc").rel(), "abc"); +} + TEST(CanonPath, from_existing) { CanonPath p0("foo//bar/"); diff --git a/src/libutil-tests/compression.cc b/src/libutil-tests/compression.cc index c6d57047118d..53d476fa8593 100644 --- a/src/libutil-tests/compression.cc +++ b/src/libutil-tests/compression.cc @@ -7,14 +7,9 @@ namespace nix { * compress / decompress * --------------------------------------------------------------------------*/ -TEST(compress, compressWithUnknownMethod) -{ - ASSERT_THROW(compress("invalid-method", "something-to-compress"), UnknownCompressionMethod); -} - TEST(compress, noneMethodDoesNothingToTheInput) { - auto o = compress("none", "this-is-a-test"); + auto o = compress(CompressionAlgo::none, "this-is-a-test"); ASSERT_EQ(o, "this-is-a-test"); } @@ -43,7 +38,7 @@ TEST(decompress, decompressXzCompressed) { auto method = "xz"; auto str = "slfja;sljfklsa;jfklsjfkl;sdjfkl;sadjfkl;sdjf;lsdfjsadlf"; - auto o = decompress(method, compress(method, str)); + auto o = decompress(method, compress(CompressionAlgo::xz, str)); ASSERT_EQ(o, str); } @@ -52,7 +47,7 @@ TEST(decompress, decompressBzip2Compressed) { auto method = "bzip2"; auto str = "slfja;sljfklsa;jfklsjfkl;sdjfkl;sadjfkl;sdjf;lsdfjsadlf"; - auto o = decompress(method, compress(method, str)); + auto o = decompress(method, compress(CompressionAlgo::bzip2, str)); ASSERT_EQ(o, str); } @@ -61,7 +56,7 @@ TEST(decompress, decompressBrCompressed) { auto method = "br"; auto str = "slfja;sljfklsa;jfklsjfkl;sdjfkl;sadjfkl;sdjf;lsdfjsadlf"; - auto o = decompress(method, compress(method, str)); + auto o = decompress(method, compress(CompressionAlgo::brotli, str)); ASSERT_EQ(o, str); } @@ -82,7 +77,7 @@ TEST(makeCompressionSink, noneSinkDoesNothingToInput) { StringSink strSink; auto inputString = "slfja;sljfklsa;jfklsjfkl;sdjfkl;sadjfkl;sdjf;lsdfjsadlf"; - auto sink = makeCompressionSink("none", strSink); + auto sink = makeCompressionSink(CompressionAlgo::none, strSink); (*sink)(inputString); sink->finish(); @@ -94,7 +89,7 @@ TEST(makeCompressionSink, compressAndDecompress) StringSink strSink; auto inputString = "slfja;sljfklsa;jfklsjfkl;sdjfkl;sadjfkl;sdjf;lsdfjsadlf"; auto decompressionSink = makeDecompressionSink("bzip2", strSink); - auto sink = makeCompressionSink("bzip2", *decompressionSink); + auto sink = makeCompressionSink(CompressionAlgo::bzip2, *decompressionSink); (*sink)(inputString); sink->finish(); diff --git a/src/libutil-tests/config.cc b/src/libutil-tests/config.cc index 87c1e556b736..11dcdea81cc8 100644 --- a/src/libutil-tests/config.cc +++ b/src/libutil-tests/config.cc @@ -183,7 +183,7 @@ TEST(Config, toJSONOnEmptyConfig) TEST(Config, toJSONOnNonEmptyConfig) { - using nlohmann::literals::operator"" _json; + using nlohmann::literals::operator""_json; Config config; Setting setting{ &config, @@ -209,7 +209,7 @@ TEST(Config, toJSONOnNonEmptyConfig) TEST(Config, toJSONOnNonEmptyConfigWithExperimentalSetting) { - using nlohmann::literals::operator"" _json; + using nlohmann::literals::operator""_json; Config config; Setting setting{ &config, diff --git a/src/libutil-tests/file-descriptor.cc b/src/libutil-tests/file-descriptor.cc new file mode 100644 index 000000000000..808bb31d8e99 --- /dev/null +++ b/src/libutil-tests/file-descriptor.cc @@ -0,0 +1,325 @@ +#include +#include + +#include "nix/util/file-descriptor.hh" +#include "nix/util/serialise.hh" +#include "nix/util/signals.hh" + +#include + +#ifndef _WIN32 +# include +# include +#endif + +namespace nix { + +// BufferedSource with configurable small buffer for precise boundary testing. +struct TestBufferedStringSource : BufferedSource +{ + std::string_view data; + size_t pos = 0; + + TestBufferedStringSource(std::string_view d, size_t bufSize) + : BufferedSource(bufSize) + , data(d) + { + } + +protected: + size_t readUnbuffered(char * buf, size_t len) override + { + if (pos >= data.size()) + throw EndOfFile("end of test data"); + size_t n = std::min(len, data.size() - pos); + std::memcpy(buf, data.data() + pos, n); + pos += n; + return n; + } +}; + +TEST(ReadLine, ReadsLinesFromPipe) +{ + Pipe pipe; + pipe.create(); + + writeFull(pipe.writeSide.get(), "hello\nworld\n", /*allowInterrupts=*/false); + pipe.writeSide.close(); + + EXPECT_EQ(readLine(pipe.readSide.get()), "hello"); + EXPECT_EQ(readLine(pipe.readSide.get()), "world"); + EXPECT_EQ(readLine(pipe.readSide.get(), /*eofOk=*/true), ""); +} + +TEST(ReadLine, ReturnsPartialLineOnEofWhenAllowed) +{ + Pipe pipe; + pipe.create(); + + writeFull(pipe.writeSide.get(), "partial", /*allowInterrupts=*/false); + pipe.writeSide.close(); + + EXPECT_EQ(readLine(pipe.readSide.get(), /*eofOk=*/true), "partial"); + EXPECT_EQ(readLine(pipe.readSide.get(), /*eofOk=*/true), ""); +} + +TEST(ReadLine, ThrowsOnEofWhenNotAllowed) +{ + Pipe pipe; + pipe.create(); + pipe.writeSide.close(); + + EXPECT_THROW(readLine(pipe.readSide.get()), EndOfFile); +} + +TEST(ReadLine, EmptyLine) +{ + Pipe pipe; + pipe.create(); + + writeFull(pipe.writeSide.get(), "\n", /*allowInterrupts=*/false); + pipe.writeSide.close(); + + EXPECT_EQ(readLine(pipe.readSide.get()), ""); +} + +TEST(ReadLine, ConsecutiveEmptyLines) +{ + Pipe pipe; + pipe.create(); + + writeFull(pipe.writeSide.get(), "\n\n\n", /*allowInterrupts=*/false); + pipe.writeSide.close(); + + EXPECT_EQ(readLine(pipe.readSide.get()), ""); + EXPECT_EQ(readLine(pipe.readSide.get()), ""); + EXPECT_EQ(readLine(pipe.readSide.get()), ""); + EXPECT_EQ(readLine(pipe.readSide.get(), /*eofOk=*/true), ""); +} + +TEST(ReadLine, LineWithNullBytes) +{ + Pipe pipe; + pipe.create(); + + std::string data( + "a\x00" + "b\n", + 4); + writeFull(pipe.writeSide.get(), data, /*allowInterrupts=*/false); + pipe.writeSide.close(); + + auto line = readLine(pipe.readSide.get()); + EXPECT_EQ(line.size(), 3); + EXPECT_EQ( + line, + std::string( + "a\x00" + "b", + 3)); +} + +#ifndef _WIN32 +TEST(ReadLine, TreatsEioAsEof) +{ + // Open a pty master. When the slave side is closed (or never opened), + // reading from the master returns EIO, which readLine should treat as EOF. + int master = posix_openpt(O_RDWR | O_NOCTTY); + ASSERT_NE(master, -1); + ASSERT_EQ(grantpt(master), 0); + ASSERT_EQ(unlockpt(master), 0); + + // Open and immediately close the slave to trigger EIO on the master. + int slave = open(ptsname(master), O_RDWR | O_NOCTTY); + ASSERT_NE(slave, -1); + close(slave); + + // With eofOk=true, readLine should return empty string (treating EIO as EOF). + EXPECT_EQ(readLine(master, /*eofOk=*/true), ""); + + // With eofOk=false, readLine should throw EndOfFile. + EXPECT_THROW(readLine(master), EndOfFile); + + close(master); +} + +// macOS (BSD) discards buffered pty data on slave close and returns normal +// EOF (0) instead of EIO, so partial data never reaches the master. +# ifdef __linux__ +TEST(ReadLine, PartialLineBeforeEio) +{ + int master = posix_openpt(O_RDWR | O_NOCTTY); + ASSERT_NE(master, -1); + ASSERT_EQ(grantpt(master), 0); + ASSERT_EQ(unlockpt(master), 0); + + int slave = open(ptsname(master), O_RDWR | O_NOCTTY); + ASSERT_NE(slave, -1); + + // Write a partial line (no terminator) from the slave, then close it. + ASSERT_EQ(::write(slave, "partial", 7), 7); + close(slave); + + // readLine should return the partial data when eofOk=true. + EXPECT_EQ(readLine(master, /*eofOk=*/true), "partial"); + + close(master); +} +# endif +#endif + +TEST(BufferedSourceReadLine, ReadsLinesFromPipe) +{ + Pipe pipe; + pipe.create(); + + writeFull(pipe.writeSide.get(), "hello\nworld\n", /*allowInterrupts=*/false); + pipe.writeSide.close(); + + FdSource source(pipe.readSide.get()); + + EXPECT_EQ(source.readLine(), "hello"); + EXPECT_EQ(source.readLine(), "world"); + EXPECT_EQ(source.readLine(/*eofOk=*/true), ""); +} + +TEST(BufferedSourceReadLine, ReturnsPartialLineOnEofWhenAllowed) +{ + Pipe pipe; + pipe.create(); + + writeFull(pipe.writeSide.get(), "partial", /*allowInterrupts=*/false); + pipe.writeSide.close(); + + FdSource source(pipe.readSide.get()); + + EXPECT_EQ(source.readLine(/*eofOk=*/true), "partial"); + EXPECT_EQ(source.readLine(/*eofOk=*/true), ""); +} + +TEST(BufferedSourceReadLine, ThrowsOnEofWhenNotAllowed) +{ + Pipe pipe; + pipe.create(); + pipe.writeSide.close(); + + FdSource source(pipe.readSide.get()); + + EXPECT_THROW(source.readLine(), EndOfFile); +} + +TEST(BufferedSourceReadLine, EmptyLine) +{ + Pipe pipe; + pipe.create(); + + writeFull(pipe.writeSide.get(), "\n", /*allowInterrupts=*/false); + pipe.writeSide.close(); + + FdSource source(pipe.readSide.get()); + + EXPECT_EQ(source.readLine(), ""); +} + +TEST(BufferedSourceReadLine, ConsecutiveEmptyLines) +{ + Pipe pipe; + pipe.create(); + + writeFull(pipe.writeSide.get(), "\n\n\n", /*allowInterrupts=*/false); + pipe.writeSide.close(); + + FdSource source(pipe.readSide.get()); + + EXPECT_EQ(source.readLine(), ""); + EXPECT_EQ(source.readLine(), ""); + EXPECT_EQ(source.readLine(), ""); + EXPECT_EQ(source.readLine(/*eofOk=*/true), ""); +} + +TEST(BufferedSourceReadLine, LineWithNullBytes) +{ + Pipe pipe; + pipe.create(); + + std::string data( + "a\x00" + "b\n", + 4); + writeFull(pipe.writeSide.get(), data, /*allowInterrupts=*/false); + pipe.writeSide.close(); + + FdSource source(pipe.readSide.get()); + + auto line = source.readLine(); + EXPECT_EQ(line.size(), 3); + EXPECT_EQ( + line, + std::string( + "a\x00" + "b", + 3)); +} + +TEST(BufferedSourceReadLine, NewlineAtBufferBoundary) +{ + // "abc\n" with buf=4: newline is last byte, triggers buffer reset. + TestBufferedStringSource source("abc\nxyz\n", 4); + + EXPECT_EQ(source.readLine(), "abc"); + EXPECT_FALSE(source.hasData()); + EXPECT_EQ(source.readLine(), "xyz"); +} + +TEST(BufferedSourceReadLine, DataRemainsAfterReadLine) +{ + // "ab\ncd\n" with buf=8: all fits in buffer, data remains after first read. + TestBufferedStringSource source("ab\ncd\n", 8); + + EXPECT_EQ(source.readLine(), "ab"); + EXPECT_TRUE(source.hasData()); + EXPECT_EQ(source.readLine(), "cd"); +} + +TEST(BufferedSourceReadLine, LineSpansMultipleRefills) +{ + // 10-char line with buf=4 requires 3 buffer refills. + TestBufferedStringSource source("0123456789\n", 4); + + EXPECT_EQ(source.readLine(), "0123456789"); +} + +TEST(BufferedSourceReadLine, BufferExhaustedThenEof) +{ + // 8 chars with buf=4: two refills, then EOF with partial line. + TestBufferedStringSource source("abcdefgh", 4); + + EXPECT_EQ(source.readLine(/*eofOk=*/true), "abcdefgh"); + EXPECT_EQ(source.readLine(/*eofOk=*/true), ""); +} + +TEST(WriteFull, RespectsAllowInterrupts) +{ +#ifdef _WIN32 + GTEST_SKIP() << "Broken on Windows"; +#endif + Pipe pipe; + pipe.create(); + + setInterrupted(true); + + // Must not throw Interrupted even though the interrupt flag is set. + EXPECT_NO_THROW(writeFull(pipe.writeSide.get(), "hello", /*allowInterrupts=*/false)); + + // Must throw Interrupted when allowInterrupts is true. + EXPECT_THROW(writeFull(pipe.writeSide.get(), "hello", /*allowInterrupts=*/true), Interrupted); + + setInterrupted(false); + pipe.writeSide.close(); + + // Verify the data from the first write was actually written. + FdSource source(pipe.readSide.get()); + EXPECT_EQ(source.readLine(/*eofOk=*/true), "hello"); +} + +} // namespace nix diff --git a/src/libutil-tests/file-system-at.cc b/src/libutil-tests/file-system-at.cc new file mode 100644 index 000000000000..5c6f93c35429 --- /dev/null +++ b/src/libutil-tests/file-system-at.cc @@ -0,0 +1,182 @@ +#include +#include + +#include "nix/util/file-system-at.hh" +#include "nix/util/file-system.hh" +#include "nix/util/fs-sink.hh" + +namespace nix { + +/* ---------------------------------------------------------------------------- + * readLinkAt + * --------------------------------------------------------------------------*/ + +TEST(readLinkAt, works) +{ +#ifdef _WIN32 + GTEST_SKIP() << "Broken on Windows"; +#endif + std::filesystem::path tmpDir = nix::createTempDir(); + nix::AutoDelete delTmpDir(tmpDir, /*recursive=*/true); + + constexpr size_t maxPathLength = +#ifdef _WIN32 + 260 +#else + PATH_MAX +#endif + ; + std::string mediumTarget(maxPathLength / 2, 'x'); + std::string longTarget(maxPathLength - 1, 'y'); + + { + RestoreSink sink(/*startFsync=*/false); + sink.dstPath = tmpDir; + sink.dirFd = openDirectory(tmpDir); + sink.createSymlink(CanonPath("link"), "target"); + sink.createSymlink(CanonPath("relative"), "../relative/path"); + sink.createSymlink(CanonPath("absolute"), "/absolute/path"); + sink.createSymlink(CanonPath("medium"), mediumTarget); + sink.createSymlink(CanonPath("long"), longTarget); + sink.createDirectory(CanonPath("a")); + sink.createDirectory(CanonPath("a/b")); + sink.createSymlink(CanonPath("a/b/link"), "nested_target"); + sink.createRegularFile(CanonPath("regular"), [](CreateRegularFileSink &) {}); + sink.createDirectory(CanonPath("dir")); + } + + auto dirFd = openDirectory(tmpDir); + + EXPECT_EQ(readLinkAt(dirFd.get(), CanonPath("link")), OS_STR("target")); + EXPECT_EQ(readLinkAt(dirFd.get(), CanonPath("relative")), OS_STR("../relative/path")); + EXPECT_EQ(readLinkAt(dirFd.get(), CanonPath("absolute")), OS_STR("/absolute/path")); + EXPECT_EQ(readLinkAt(dirFd.get(), CanonPath("medium")), string_to_os_string(mediumTarget)); + EXPECT_EQ(readLinkAt(dirFd.get(), CanonPath("long")), string_to_os_string(longTarget)); + EXPECT_EQ(readLinkAt(dirFd.get(), CanonPath("a/b/link")), OS_STR("nested_target")); + + auto subDirFd = openDirectory(tmpDir / "a"); + EXPECT_EQ(readLinkAt(subDirFd.get(), CanonPath("b/link")), OS_STR("nested_target")); + + // Test error cases - expect SystemError on both platforms + EXPECT_THROW(readLinkAt(dirFd.get(), CanonPath("regular")), SystemError); + EXPECT_THROW(readLinkAt(dirFd.get(), CanonPath("dir")), SystemError); + EXPECT_THROW(readLinkAt(dirFd.get(), CanonPath("nonexistent")), SystemError); +} + +/* ---------------------------------------------------------------------------- + * openFileEnsureBeneathNoSymlinks + * --------------------------------------------------------------------------*/ + +TEST(openFileEnsureBeneathNoSymlinks, works) +{ +#ifdef _WIN32 + GTEST_SKIP() << "Broken on Windows"; +#endif + std::filesystem::path tmpDir = nix::createTempDir(); + nix::AutoDelete delTmpDir(tmpDir, /*recursive=*/true); + + { + RestoreSink sink(/*startFsync=*/false); + sink.dstPath = tmpDir; + sink.dirFd = openDirectory(tmpDir); + sink.createDirectory(CanonPath("a")); + sink.createDirectory(CanonPath("c")); + sink.createDirectory(CanonPath("c/d")); + sink.createRegularFile(CanonPath("c/d/regular"), [](CreateRegularFileSink & crf) { crf("some contents"); }); + sink.createSymlink(CanonPath("a/absolute_symlink"), tmpDir.string()); + sink.createSymlink(CanonPath("a/relative_symlink"), "../."); + sink.createSymlink(CanonPath("a/broken_symlink"), "./nonexistent"); + sink.createDirectory(CanonPath("a/b"), [](FileSystemObjectSink & dirSink, const CanonPath & relPath) { + dirSink.createDirectory(CanonPath("d")); + dirSink.createSymlink(CanonPath("c"), "./d"); + }); +#ifdef _WIN32 + EXPECT_THROW(sink.createDirectory(CanonPath("a/b/c/e")), SymlinkNotAllowed); +#else + // FIXME: This still follows symlinks on Unix (incorrectly succeeds) + sink.createDirectory(CanonPath("a/b/c/e")); +#endif + // Test that symlinks in intermediate path are detected during nested operations + EXPECT_THROW( + sink.createDirectory( + CanonPath("a/b/c/f"), [](FileSystemObjectSink & dirSink, const CanonPath & relPath) {}), + SymlinkNotAllowed); + EXPECT_THROW( + sink.createRegularFile( + CanonPath("a/b/c/regular"), [](CreateRegularFileSink & crf) { crf("some contents"); }), + SymlinkNotAllowed); + } + + auto dirFd = openDirectory(tmpDir); + + // Helper to open files with platform-specific arguments + auto openRead = [&](std::string_view path) -> AutoCloseFD { + return openFileEnsureBeneathNoSymlinks( + dirFd.get(), + CanonPath(path), +#ifdef _WIN32 + FILE_READ_DATA | FILE_READ_ATTRIBUTES | SYNCHRONIZE, + 0 +#else + O_RDONLY, + 0 +#endif + ); + }; + + auto openReadDir = [&](std::string_view path) -> AutoCloseFD { + return openFileEnsureBeneathNoSymlinks( + dirFd.get(), + CanonPath(path), +#ifdef _WIN32 + FILE_READ_ATTRIBUTES | SYNCHRONIZE, + FILE_DIRECTORY_FILE +#else + O_RDONLY | O_DIRECTORY, + 0 +#endif + ); + }; + + auto openCreateExclusive = [&](std::string_view path) -> AutoCloseFD { + return openFileEnsureBeneathNoSymlinks( + dirFd.get(), + CanonPath(path), +#ifdef _WIN32 + FILE_WRITE_DATA | SYNCHRONIZE, + 0, + FILE_CREATE // Create new file, fail if exists (equivalent to O_CREAT | O_EXCL) +#else + O_CREAT | O_WRONLY | O_EXCL, + 0666 +#endif + ); + }; + + // Test that symlinks are detected and rejected + EXPECT_THROW(openRead("a/absolute_symlink"), SymlinkNotAllowed); + EXPECT_THROW(openRead("a/relative_symlink"), SymlinkNotAllowed); + EXPECT_THROW(openRead("a/absolute_symlink/a"), SymlinkNotAllowed); + EXPECT_THROW(openRead("a/absolute_symlink/c/d"), SymlinkNotAllowed); + EXPECT_THROW(openRead("a/relative_symlink/c"), SymlinkNotAllowed); + EXPECT_THROW(openRead("a/b/c/d"), SymlinkNotAllowed); + EXPECT_THROW(openRead("a/broken_symlink"), SymlinkNotAllowed); + +#if !defined(_WIN32) && !defined(__CYGWIN__) + // This returns ELOOP on cygwin when O_NOFOLLOW is used + EXPECT_FALSE(openCreateExclusive("a/broken_symlink")); + /* Sanity check, no symlink shenanigans and behaves the same as regular openat with O_EXCL | O_CREAT. */ + EXPECT_EQ(errno, EEXIST); +#endif + EXPECT_THROW(openCreateExclusive("a/absolute_symlink/broken_symlink"), SymlinkNotAllowed); + + // Test invalid paths + EXPECT_FALSE(openRead("c/d/regular/a")); + EXPECT_FALSE(openReadDir("c/d/regular")); + + // Test valid paths work + EXPECT_TRUE(openRead("c/d/regular")); + EXPECT_TRUE(openCreateExclusive("a/regular")); +} + +} // namespace nix diff --git a/src/libutil-tests/file-system.cc b/src/libutil-tests/file-system.cc index 1551227cbd90..95bfa16b140c 100644 --- a/src/libutil-tests/file-system.cc +++ b/src/libutil-tests/file-system.cc @@ -1,5 +1,5 @@ -#include "nix/util/fs-sink.hh" #include "nix/util/util.hh" +#include "nix/util/serialise.hh" #include "nix/util/types.hh" #include "nix/util/file-system.hh" #include "nix/util/processes.hh" @@ -12,12 +12,16 @@ #include +using namespace std::string_view_literals; + #ifdef _WIN32 # define FS_SEP L"\\" -# define FS_ROOT L"C:" FS_SEP // Need a mounted one, C drive is likely +# define FS_ROOT_NO_TRAILING_SLASH L"C:" // Need a mounted one, C drive is likely +# define FS_ROOT FS_ROOT_NO_TRAILING_SLASH FS_SEP #else # define FS_SEP "/" -# define FS_ROOT FS_SEP +# define FS_ROOT_NO_TRAILING_SLASH FS_SEP +# define FS_ROOT FS_ROOT_NO_TRAILING_SLASH #endif #ifndef PATH_MAX @@ -42,7 +46,7 @@ TEST(absPath, doesntChangeRoot) { auto p = absPath(std::filesystem::path{FS_ROOT}); - ASSERT_EQ(p, FS_ROOT); + ASSERT_EQ(p, FS_ROOT_NO_TRAILING_SLASH); } TEST(absPath, turnsEmptyPathIntoCWD) @@ -58,9 +62,10 @@ TEST(absPath, usesOptionalBasePathWhenGiven) OsChar _cwd[PATH_MAX + 1]; OsChar * cwd = GET_CWD((OsChar *) &_cwd, PATH_MAX); - auto p = absPath(std::filesystem::path{""}.string(), std::filesystem::path{cwd}.string()); + auto cwdPath = std::filesystem::path{cwd}; + auto p = absPath("", &cwdPath); - ASSERT_EQ(p, std::filesystem::path{cwd}.string()); + ASSERT_EQ(p, cwdPath); } TEST(absPath, isIdempotent) @@ -113,33 +118,10 @@ TEST(canonPath, removesDots2) TEST(canonPath, requiresAbsolutePath) { - ASSERT_ANY_THROW(canonPath(".")); - ASSERT_ANY_THROW(canonPath("..")); - ASSERT_ANY_THROW(canonPath("../")); - ASSERT_DEATH({ canonPath(""); }, "path != \"\""); -} - -/* ---------------------------------------------------------------------------- - * dirOf - * --------------------------------------------------------------------------*/ - -TEST(dirOf, returnsEmptyStringForRoot) -{ - auto p = dirOf("/"); - - ASSERT_EQ(p, "/"); -} - -TEST(dirOf, returnsFirstPathComponent) -{ - auto p1 = dirOf("/dir/"); - ASSERT_EQ(p1, "/dir"); - auto p2 = dirOf("/dir"); - ASSERT_EQ(p2, "/"); - auto p3 = dirOf("/dir/.."); - ASSERT_EQ(p3, "/dir"); - auto p4 = dirOf("/dir/../"); - ASSERT_EQ(p4, "/dir/.."); + ASSERT_ANY_THROW(canonPath("."sv)); + ASSERT_ANY_THROW(canonPath(".."sv)); + ASSERT_ANY_THROW(canonPath("../"sv)); + ASSERT_DEATH({ canonPath(""sv); }, "!path.empty\\(\\)"); } /* ---------------------------------------------------------------------------- @@ -194,20 +176,42 @@ TEST(baseNameOf, absoluteNothingSlashNothing) TEST(isInDir, trivialCase) { - auto p1 = isInDir("/foo/bar", "/foo"); - ASSERT_EQ(p1, true); + EXPECT_TRUE(isInDir(FS_ROOT "foo" FS_SEP "bar", FS_ROOT "foo")); } TEST(isInDir, notInDir) { - auto p1 = isInDir("/zes/foo/bar", "/foo"); - ASSERT_EQ(p1, false); + EXPECT_FALSE(isInDir(FS_ROOT "zes" FS_SEP "foo" FS_SEP "bar", FS_ROOT "foo")); } TEST(isInDir, emptyDir) { - auto p1 = isInDir("/zes/foo/bar", ""); - ASSERT_EQ(p1, false); + EXPECT_FALSE(isInDir(FS_ROOT "zes" FS_SEP "foo" FS_SEP "bar", "")); +} + +TEST(isInDir, hiddenSubdirectory) +{ + EXPECT_TRUE(isInDir(FS_ROOT "foo" FS_SEP ".ssh", FS_ROOT "foo")); +} + +TEST(isInDir, ellipsisEntry) +{ + EXPECT_TRUE(isInDir(FS_ROOT "foo" FS_SEP "...", FS_ROOT "foo")); +} + +TEST(isInDir, sameDir) +{ + EXPECT_FALSE(isInDir(FS_ROOT "foo", FS_ROOT "foo")); +} + +TEST(isInDir, sameDirDot) +{ + EXPECT_FALSE(isInDir(FS_ROOT "foo" FS_SEP ".", FS_ROOT "foo")); +} + +TEST(isInDir, dotDotPrefix) +{ + EXPECT_FALSE(isInDir(FS_ROOT "foo" FS_SEP ".." FS_SEP "bar", FS_ROOT "foo")); } /* ---------------------------------------------------------------------------- @@ -216,8 +220,8 @@ TEST(isInDir, emptyDir) TEST(isDirOrInDir, trueForSameDirectory) { - ASSERT_EQ(isDirOrInDir("/nix", "/nix"), true); - ASSERT_EQ(isDirOrInDir("/", "/"), true); + ASSERT_EQ(isDirOrInDir(FS_ROOT "nix", FS_ROOT "nix"), true); + ASSERT_EQ(isDirOrInDir(FS_ROOT, FS_ROOT), true); } TEST(isDirOrInDir, trueForEmptyPaths) @@ -227,17 +231,17 @@ TEST(isDirOrInDir, trueForEmptyPaths) TEST(isDirOrInDir, falseForDisjunctPaths) { - ASSERT_EQ(isDirOrInDir("/foo", "/bar"), false); + ASSERT_EQ(isDirOrInDir(FS_ROOT "foo", FS_ROOT "bar"), false); } TEST(isDirOrInDir, relativePaths) { - ASSERT_EQ(isDirOrInDir("/foo/..", "/foo"), false); + ASSERT_EQ(isDirOrInDir(FS_ROOT "foo" FS_SEP "..", FS_ROOT "foo"), false); } TEST(isDirOrInDir, relativePathsTwice) { - ASSERT_EQ(isDirOrInDir("/foo/..", "/foo/."), false); + ASSERT_EQ(isDirOrInDir(FS_ROOT "foo" FS_SEP "..", FS_ROOT "foo" FS_SEP "."), false); } /* ---------------------------------------------------------------------------- @@ -270,7 +274,7 @@ TEST(makeParentCanonical, noParent) TEST(makeParentCanonical, root) { - ASSERT_EQ(makeParentCanonical("/"), "/"); + ASSERT_EQ(makeParentCanonical(FS_ROOT), FS_ROOT_NO_TRAILING_SLASH); } /* ---------------------------------------------------------------------------- @@ -279,6 +283,11 @@ TEST(makeParentCanonical, root) TEST(chmodIfNeeded, works) { +#ifdef _WIN32 + // Windows doesn't support Unix-style permission bits - lstat always + // returns the same mode regardless of what chmod sets. + GTEST_SKIP() << "Broken on Windows"; +#endif auto [autoClose_, tmpFile] = nix::createTempFile(); auto deleteTmpFile = AutoDelete(tmpFile); @@ -295,7 +304,7 @@ TEST(chmodIfNeeded, works) TEST(chmodIfNeeded, nonexistent) { - ASSERT_THROW(chmodIfNeeded("/schnitzel/darmstadt/pommes", 0755), SysError); + ASSERT_THROW(chmodIfNeeded("/schnitzel/darmstadt/pommes", 0755), SystemError); } /* ---------------------------------------------------------------------------- @@ -316,71 +325,9 @@ TEST(DirectoryIterator, works) TEST(DirectoryIterator, nonexistent) { - ASSERT_THROW(DirectoryIterator("/schnitzel/darmstadt/pommes"), SysError); + ASSERT_THROW(DirectoryIterator("/schnitzel/darmstadt/pommes"), SystemError); } -/* ---------------------------------------------------------------------------- - * openFileEnsureBeneathNoSymlinks - * --------------------------------------------------------------------------*/ - -#ifndef _WIN32 - -TEST(openFileEnsureBeneathNoSymlinks, works) -{ - std::filesystem::path tmpDir = nix::createTempDir(); - nix::AutoDelete delTmpDir(tmpDir, /*recursive=*/true); - using namespace nix::unix; - - { - RestoreSink sink(/*startFsync=*/false); - sink.dstPath = tmpDir; - sink.dirFd = openDirectory(tmpDir); - sink.createDirectory(CanonPath("a")); - sink.createDirectory(CanonPath("c")); - sink.createDirectory(CanonPath("c/d")); - sink.createRegularFile(CanonPath("c/d/regular"), [](CreateRegularFileSink & crf) { crf("some contents"); }); - sink.createSymlink(CanonPath("a/absolute_symlink"), tmpDir.string()); - sink.createSymlink(CanonPath("a/relative_symlink"), "../."); - sink.createSymlink(CanonPath("a/broken_symlink"), "./nonexistent"); - sink.createDirectory(CanonPath("a/b"), [](FileSystemObjectSink & dirSink, const CanonPath & relPath) { - dirSink.createDirectory(CanonPath("d")); - dirSink.createSymlink(CanonPath("c"), "./d"); - }); - sink.createDirectory(CanonPath("a/b/c/e")); // FIXME: This still follows symlinks - ASSERT_THROW( - sink.createDirectory( - CanonPath("a/b/c/f"), [](FileSystemObjectSink & dirSink, const CanonPath & relPath) {}), - SymlinkNotAllowed); - ASSERT_THROW( - sink.createRegularFile( - CanonPath("a/b/c/regular"), [](CreateRegularFileSink & crf) { crf("some contents"); }), - SymlinkNotAllowed); - } - - AutoCloseFD dirFd = openDirectory(tmpDir); - - auto open = [&](std::string_view path, int flags, mode_t mode = 0) { - return openFileEnsureBeneathNoSymlinks(dirFd.get(), CanonPath(path), flags, mode); - }; - - EXPECT_THROW(open("a/absolute_symlink", O_RDONLY), SymlinkNotAllowed); - EXPECT_THROW(open("a/relative_symlink", O_RDONLY), SymlinkNotAllowed); - EXPECT_THROW(open("a/absolute_symlink/a", O_RDONLY), SymlinkNotAllowed); - EXPECT_THROW(open("a/absolute_symlink/c/d", O_RDONLY), SymlinkNotAllowed); - EXPECT_THROW(open("a/relative_symlink/c", O_RDONLY), SymlinkNotAllowed); - EXPECT_THROW(open("a/b/c/d", O_RDONLY), SymlinkNotAllowed); - EXPECT_EQ(open("a/broken_symlink", O_CREAT | O_WRONLY | O_EXCL, 0666), INVALID_DESCRIPTOR); - /* Sanity check, no symlink shenanigans and behaves the same as regular openat with O_EXCL | O_CREAT. */ - EXPECT_EQ(errno, EEXIST); - EXPECT_THROW(open("a/absolute_symlink/broken_symlink", O_CREAT | O_WRONLY | O_EXCL, 0666), SymlinkNotAllowed); - EXPECT_EQ(open("c/d/regular/a", O_RDONLY), INVALID_DESCRIPTOR); - EXPECT_EQ(open("c/d/regular", O_RDONLY | O_DIRECTORY), INVALID_DESCRIPTOR); - EXPECT_TRUE(AutoCloseFD{open("c/d/regular", O_RDONLY)}); - EXPECT_TRUE(AutoCloseFD{open("a/regular", O_CREAT | O_WRONLY | O_EXCL, 0666)}); -} - -#endif - /* ---------------------------------------------------------------------------- * createAnonymousTempFile * --------------------------------------------------------------------------*/ @@ -388,14 +335,13 @@ TEST(openFileEnsureBeneathNoSymlinks, works) TEST(createAnonymousTempFile, works) { auto fd = createAnonymousTempFile(); - auto fd_ = fromDescriptorReadOnly(fd.get()); writeFull(fd.get(), "test"); - lseek(fd_, 0, SEEK_SET); + lseek(fd.get(), 0, SEEK_SET); FdSource source{fd.get()}; EXPECT_EQ(source.drain(), "test"); - lseek(fd_, 0, SEEK_END); + lseek(fd.get(), 0, SEEK_END); writeFull(fd.get(), "test"); - lseek(fd_, 0, SEEK_SET); + lseek(fd.get(), 0, SEEK_SET); EXPECT_EQ(source.drain(), "testtest"); } @@ -406,9 +352,8 @@ TEST(createAnonymousTempFile, works) TEST(FdSource, restartWorks) { auto fd = createAnonymousTempFile(); - auto fd_ = fromDescriptorReadOnly(fd.get()); writeFull(fd.get(), "hello world"); - lseek(fd_, 0, SEEK_SET); + lseek(fd.get(), 0, SEEK_SET); FdSource source{fd.get()}; EXPECT_EQ(source.drain(), "hello world"); source.restart(); @@ -416,4 +361,11 @@ TEST(FdSource, restartWorks) EXPECT_EQ(source.drain(), ""); } +TEST(createTempDir, works) +{ + auto tmpDir = createTempDir(); + nix::AutoDelete delTmpDir(tmpDir, /*recursive=*/true); + ASSERT_TRUE(std::filesystem::is_directory(tmpDir)); +} + } // namespace nix diff --git a/src/libutil-tests/fun.cc b/src/libutil-tests/fun.cc new file mode 100644 index 000000000000..e763ac018e7f --- /dev/null +++ b/src/libutil-tests/fun.cc @@ -0,0 +1,180 @@ +#include +#include + +#include "nix/util/fun.hh" + +namespace nix { + +static int add(int a, int b) +{ + return a + b; +} + +TEST(fun, constructFromLambda) +{ + fun f = [](int x) { return x * 2; }; + EXPECT_EQ(f(3), 6); +} + +TEST(fun, constructFromFunctionPointer) +{ + fun f = add; + EXPECT_EQ(f(2, 3), 5); +} + +TEST(fun, constructFromStdFunction) +{ + std::function sf = [](int x) { return x + 1; }; + fun f(sf); + EXPECT_EQ(f(5), 6); +} + +TEST(fun, moveConstructFromStdFunction) +{ + std::function sf = [](int x) { return x + 1; }; + fun f(std::move(sf)); + EXPECT_EQ(f(5), 6); +} + +TEST(fun, rejectsEmptyStdFunction) +{ + std::function empty; + EXPECT_THROW((fun{empty}), std::invalid_argument); +} + +TEST(fun, rejectsEmptyStdFunctionMove) +{ + std::function empty; + EXPECT_THROW((fun{std::move(empty)}), std::invalid_argument); +} + +TEST(fun, rejectsNullFunctionPointer) +{ + int (*nullFp)(int) = nullptr; + EXPECT_THROW((fun{nullFp}), std::invalid_argument); +} + +TEST(fun, nullptrDeletedAtCompileTime) +{ + // fun(nullptr) is a deleted constructor + static_assert(!std::is_constructible_v, std::nullptr_t>); +} + +TEST(fun, notDefaultConstructible) +{ + static_assert(!std::is_default_constructible_v>); +} + +TEST(fun, voidReturn) +{ + int called = 0; + fun f = [&]() { called++; }; + f(); + EXPECT_EQ(called, 1); +} + +TEST(fun, referenceArgs) +{ + fun f = [](int & x) { x += 10; }; + int val = 5; + f(val); + EXPECT_EQ(val, 15); +} + +TEST(fun, convertsToStdFunction) +{ + fun f = [](int x) { return x * 3; }; + std::function sf = f.get_fn(); + EXPECT_EQ(sf(4), 12); +} + +TEST(fun, copyable) +{ + fun f = [](int x) { return x + 1; }; + auto g = f; + EXPECT_EQ(f(1), 2); + EXPECT_EQ(g(1), 2); +} + +TEST(fun, movable) +{ + fun f = [](int x) { return x + 1; }; + auto g = std::move(f); + EXPECT_EQ(g(1), 2); +} + +TEST(fun, capturesState) +{ + int offset = 100; + fun f = [offset](int x) { return x + offset; }; + EXPECT_EQ(f(5), 105); +} + +TEST(fun, getFn) +{ + fun f = [](int x) { return x; }; + const auto & sf = f.get_fn(); + EXPECT_EQ(sf(42), 42); +} + +TEST(fun, getFnMove) +{ + fun f = [](int x) { return x; }; + auto sf = std::move(f).get_fn(); + EXPECT_EQ(sf(42), 42); +} + +TEST(fun, forwardsMoveOnlyTypes) +{ + fun)> f = [](std::unique_ptr p) { return *p; }; + auto p = std::make_unique(42); + EXPECT_EQ(f(std::move(p)), 42); +} + +TEST(fun, perfectForwardingZeroCost) +{ + int copies = 0, moves = 0; + + struct Tracker + { + int & copies; + int & moves; + + Tracker(int & copies, int & moves) + : copies(copies) + , moves(moves) + { + } + + Tracker(const Tracker & o) + : copies(o.copies) + , moves(o.moves) + { + copies++; + } + + Tracker(Tracker && o) noexcept + : copies(o.copies) + , moves(o.moves) + { + moves++; + } + }; + + // Baseline: call std::function directly + std::function sf = [](Tracker, Tracker) {}; + Tracker t1{copies, moves}, t2{copies, moves}; + sf(t1, t2); + int baseline_copies = copies, baseline_moves = moves; + + copies = 0; + moves = 0; + + // Call through fun<> — should match baseline exactly + fun f = [](Tracker, Tracker) {}; + f(t1, t2); + EXPECT_EQ(copies, baseline_copies); + EXPECT_EQ(moves, baseline_moves); +} + +} // namespace nix diff --git a/src/libutil-tests/git.cc b/src/libutil-tests/git.cc index f761c4433502..dc7bd8fed4e3 100644 --- a/src/libutil-tests/git.cc +++ b/src/libutil-tests/git.cc @@ -237,8 +237,7 @@ TEST_F(GitTest, both_roundrip) for (const auto hashAlgo : {HashAlgorithm::SHA1, HashAlgorithm::SHA256}) { std::map cas; - std::function dumpHook; - dumpHook = [&](const SourcePath & path) { + fun dumpHook = [&](const SourcePath & path) { StringSink s; HashSink hashSink{hashAlgo}; TeeSink s2{s, hashSink}; @@ -251,7 +250,7 @@ TEST_F(GitTest, both_roundrip) }; }; - auto root = dumpHook({files}); + auto root = dumpHook(SourcePath{files}); auto files2 = make_ref(); diff --git a/src/libutil-tests/memory-source-accessor.cc b/src/libutil-tests/memory-source-accessor.cc index 6c7c9ce9e811..d9a48da0cebd 100644 --- a/src/libutil-tests/memory-source-accessor.cc +++ b/src/libutil-tests/memory-source-accessor.cc @@ -1,7 +1,11 @@ -#include - #include "nix/util/memory-source-accessor.hh" #include "nix/util/tests/json-characterization.hh" +#include "nix/util/tests/gmock-matchers.hh" + +#include +#include + +#include namespace nix { @@ -59,6 +63,163 @@ ref exampleComplex() } // namespace memory_source_accessor +/* ---------------------------------------------------------------------------- + * MemorySourceAccessor + * --------------------------------------------------------------------------*/ + +using ::nix::testing::HasSubstrIgnoreANSIMatcher; + +class MemorySourceAccessorTestErrors : public ::testing::Test +{ +protected: + ref accessor = make_ref(); + MemorySink sink{*accessor}; + + void SetUp() override + { + accessor->setPathDisplay("somepath"); + sink.createDirectory(CanonPath::root); + } +}; + +TEST_F(MemorySourceAccessorTestErrors, readFileNotFound) +{ + EXPECT_THAT( + [&] { accessor->readFile(CanonPath("nonexistent")); }, + ThrowsMessage( + AllOf(HasSubstrIgnoreANSIMatcher("somepath/nonexistent"), HasSubstrIgnoreANSIMatcher("does not exist")))); +} + +TEST_F(MemorySourceAccessorTestErrors, readFileNotARegularFile) +{ + sink.createDirectory(CanonPath("subdir")); + + EXPECT_THAT( + [&] { accessor->readFile(CanonPath("subdir")); }, + ThrowsMessage( + AllOf(HasSubstrIgnoreANSIMatcher("somepath/subdir"), HasSubstrIgnoreANSIMatcher("is not a regular file")))); +} + +TEST_F(MemorySourceAccessorTestErrors, readDirectoryNotFound) +{ + EXPECT_THAT( + [&] { accessor->readDirectory(CanonPath("nonexistent")); }, + ThrowsMessage( + AllOf(HasSubstrIgnoreANSIMatcher("somepath/nonexistent"), HasSubstrIgnoreANSIMatcher("does not exist")))); +} + +TEST_F(MemorySourceAccessorTestErrors, readDirectoryNotADirectory) +{ + sink.createRegularFile(CanonPath("file"), [](CreateRegularFileSink &) {}); + + EXPECT_THAT( + [&] { accessor->readDirectory(CanonPath("file")); }, + ThrowsMessage( + AllOf(HasSubstrIgnoreANSIMatcher("somepath/file"), HasSubstrIgnoreANSIMatcher("is not a directory")))); +} + +TEST_F(MemorySourceAccessorTestErrors, readLinkNotFound) +{ + EXPECT_THAT( + [&] { accessor->readLink(CanonPath("nonexistent")); }, + ThrowsMessage( + AllOf(HasSubstrIgnoreANSIMatcher("somepath/nonexistent"), HasSubstrIgnoreANSIMatcher("does not exist")))); +} + +TEST_F(MemorySourceAccessorTestErrors, readLinkNotASymlink) +{ + sink.createRegularFile(CanonPath("file"), [](CreateRegularFileSink &) {}); + + EXPECT_THAT( + [&] { accessor->readLink(CanonPath("file")); }, + ThrowsMessage( + AllOf(HasSubstrIgnoreANSIMatcher("somepath/file"), HasSubstrIgnoreANSIMatcher("is not a symbolic link")))); +} + +TEST_F(MemorySourceAccessorTestErrors, addFileParentNotDirectory) +{ + sink.createRegularFile(CanonPath("file"), [](CreateRegularFileSink &) {}); + + EXPECT_THAT( + [&] { accessor->addFile(CanonPath("file/child"), "contents"); }, + ThrowsMessage(AllOf( + HasSubstrIgnoreANSIMatcher("somepath/file/child"), + HasSubstrIgnoreANSIMatcher("cannot be created because some parent file is not a directory")))); +} + +TEST_F(MemorySourceAccessorTestErrors, addFileNotARegularFile) +{ + sink.createDirectory(CanonPath("subdir")); + + EXPECT_THAT( + [&] { accessor->addFile(CanonPath("subdir"), "contents"); }, + ThrowsMessage( + AllOf(HasSubstrIgnoreANSIMatcher("somepath/subdir"), HasSubstrIgnoreANSIMatcher("is not a regular file")))); +} + +TEST_F(MemorySourceAccessorTestErrors, createDirectoryParentNotDirectory) +{ + sink.createRegularFile(CanonPath("file"), [](CreateRegularFileSink &) {}); + + EXPECT_THAT( + [&] { sink.createDirectory(CanonPath("file/child")); }, + ThrowsMessage(AllOf( + HasSubstrIgnoreANSIMatcher("somepath/file/child"), + HasSubstrIgnoreANSIMatcher("cannot be created because some parent file is not a directory")))); +} + +TEST_F(MemorySourceAccessorTestErrors, createDirectoryNotADirectory) +{ + sink.createRegularFile(CanonPath("file"), [](CreateRegularFileSink &) {}); + + EXPECT_THAT( + [&] { sink.createDirectory(CanonPath("file")); }, + ThrowsMessage( + AllOf(HasSubstrIgnoreANSIMatcher("somepath/file"), HasSubstrIgnoreANSIMatcher("is not a directory")))); +} + +TEST_F(MemorySourceAccessorTestErrors, createRegularFileParentNotDirectory) +{ + sink.createRegularFile(CanonPath("file"), [](CreateRegularFileSink &) {}); + + EXPECT_THAT( + [&] { sink.createRegularFile(CanonPath("file/child"), [](CreateRegularFileSink &) {}); }, + ThrowsMessage(AllOf( + HasSubstrIgnoreANSIMatcher("file/child"), + HasSubstrIgnoreANSIMatcher("cannot be created because some parent file is not a directory")))); +} + +TEST_F(MemorySourceAccessorTestErrors, createRegularFileNotARegularFile) +{ + sink.createDirectory(CanonPath("subdir")); + + EXPECT_THAT( + [&] { sink.createRegularFile(CanonPath("subdir"), [](CreateRegularFileSink &) {}); }, + ThrowsMessage( + AllOf(HasSubstrIgnoreANSIMatcher("somepath/subdir"), HasSubstrIgnoreANSIMatcher("is not a regular file")))); +} + +TEST_F(MemorySourceAccessorTestErrors, createSymlinkParentNotDirectory) +{ + sink.createRegularFile(CanonPath("file"), [](CreateRegularFileSink &) {}); + + EXPECT_THAT( + [&] { sink.createSymlink(CanonPath("file/child"), "target"); }, + ThrowsMessage(AllOf( + HasSubstrIgnoreANSIMatcher("somepath/file/child"), + HasSubstrIgnoreANSIMatcher("cannot be created because some parent file is not a directory")))); +} + +TEST_F(MemorySourceAccessorTestErrors, createSymlinkNotASymlink) +{ + sink.createRegularFile(CanonPath("file"), [](CreateRegularFileSink &) {}); + + EXPECT_THAT( + [&] { sink.createSymlink(CanonPath("file"), "target"); }, + ThrowsMessage( + AllOf(HasSubstrIgnoreANSIMatcher("somepath/file"), HasSubstrIgnoreANSIMatcher("is not a symbolic link")))); +} + /* ---------------------------------------------------------------------------- * JSON * --------------------------------------------------------------------------*/ @@ -87,7 +248,7 @@ TEST_P(MemorySourceAccessorJsonTest, from_json) auto & [name, expected] = GetParam(); /* Cannot use `readJsonTest` because need to compare `root` field of the source accessors for equality. */ - readTest(Path{name} + ".json", [&](const auto & encodedRaw) { + readTest(std::string{name} + ".json", [&](const auto & encodedRaw) { auto encoded = json::parse(encodedRaw); auto decoded = static_cast(encoded); ASSERT_EQ(decoded.root, expected.root); diff --git a/src/libutil-tests/meson.build b/src/libutil-tests/meson.build index 019bdb6d2a34..bcc9a607a179 100644 --- a/src/libutil-tests/meson.build +++ b/src/libutil-tests/meson.build @@ -33,6 +33,9 @@ deps_private += rapidcheck gtest = dependency('gtest', main : true) deps_private += gtest +gmock = dependency('gmock') +deps_private += gmock + configdata = configuration_data() configdata.set_quoted('PACKAGE_VERSION', meson.project_version()) @@ -56,7 +59,10 @@ sources = files( 'config.cc', 'executable-path.cc', 'file-content-address.cc', + 'file-descriptor.cc', + 'file-system-at.cc', 'file-system.cc', + 'fun.cc', 'git.cc', 'hash.cc', 'hilite.cc', @@ -71,7 +77,10 @@ sources = files( 'pool.cc', 'position.cc', 'processes.cc', + 'ref.cc', + 'serialise.cc', 'sort.cc', + 'source-accessor.cc', 'spawn.cc', 'strings.cc', 'suggestions.cc', @@ -82,6 +91,10 @@ sources = files( 'xml-writer.cc', ) +if host_machine.system() != 'windows' + subdir('unix') +endif + include_dirs = [ include_directories('.') ] diff --git a/src/libutil-tests/package.nix b/src/libutil-tests/package.nix index c06de6894afe..f24d69243b86 100644 --- a/src/libutil-tests/package.nix +++ b/src/libutil-tests/package.nix @@ -32,6 +32,7 @@ mkMesonExecutable (finalAttrs: { ../../.version ./.version ./meson.build + ./unix/meson.build # ./meson.options (fileset.fileFilter (file: file.hasExt "cc") ./.) (fileset.fileFilter (file: file.hasExt "hh") ./.) diff --git a/src/libutil-tests/ref.cc b/src/libutil-tests/ref.cc new file mode 100644 index 000000000000..ed016d5cc0cf --- /dev/null +++ b/src/libutil-tests/ref.cc @@ -0,0 +1,88 @@ +#include +#include + +#include "nix/util/demangle.hh" +#include "nix/util/ref.hh" + +namespace nix { + +// Test hierarchy for ref covariance tests +struct Base +{ + virtual ~Base() = default; +}; + +struct Derived : Base +{}; + +TEST(ref, upcast_is_implicit) +{ + // ref should be implicitly convertible to ref + static_assert(std::is_convertible_v, ref>); + + // Runtime test + auto derived = make_ref(); + ref base = derived; // implicit upcast + EXPECT_NE(&*base, nullptr); +} + +TEST(ref, downcast_is_rejected) +{ + // ref should NOT be implicitly convertible to ref + static_assert(!std::is_convertible_v, ref>); + + // Uncomment to see error message: + // auto base = make_ref(); + // ref d = base; +} + +TEST(ref, same_type_conversion) +{ + // ref should be convertible to ref + static_assert(std::is_convertible_v, ref>); + static_assert(std::is_convertible_v, ref>); +} + +TEST(ref, explicit_downcast_with_cast) +{ + // .cast() should work for valid downcasts at runtime + auto derived = make_ref(); + ref base = derived; + + // Downcast back to Derived using .cast() + ref backToDerived = base.cast(); + EXPECT_NE(&*backToDerived, nullptr); +} + +TEST(ref, invalid_cast_throws) +{ + // .cast() throws bad_ref_cast (a std::bad_cast subclass) with type info on invalid downcast + // (unlike .dynamic_pointer_cast() which returns nullptr) + auto base = make_ref(); + try { + base.cast(); + FAIL() << "Expected bad_ref_cast"; + } catch (const bad_ref_cast & e) { + std::string expected = "ref<" + demangle(typeid(Base).name()) + "> cannot be cast to ref<" + + demangle(typeid(Derived).name()) + ">"; + EXPECT_EQ(e.what(), expected); + } +} + +TEST(ref, explicit_downcast_with_dynamic_pointer_cast) +{ + // .dynamic_pointer_cast() returns nullptr for invalid casts + auto base = make_ref(); + + // Invalid downcast returns nullptr + auto invalidCast = base.dynamic_pointer_cast(); + EXPECT_EQ(invalidCast, nullptr); + + // Valid downcast returns non-null + auto derived = make_ref(); + ref baseFromDerived = derived; + auto validCast = baseFromDerived.dynamic_pointer_cast(); + EXPECT_NE(validCast, nullptr); +} + +} // namespace nix diff --git a/src/libutil-tests/serialise.cc b/src/libutil-tests/serialise.cc new file mode 100644 index 000000000000..13b285e2b44d --- /dev/null +++ b/src/libutil-tests/serialise.cc @@ -0,0 +1,25 @@ +#include "nix/util/serialise.hh" + +#include + +namespace nix { + +template + requires std::is_integral_v +auto makeNumSource(T num) +{ + return sinkToSource([num](Sink & writer) { writer << num; }); +} + +TEST(readNum, negativeValuesSerialiseWellDefined) +{ + EXPECT_THROW(readNum(*makeNumSource(int64_t(-1))), SerialisationError); + EXPECT_THROW(readNum(*makeNumSource(int16_t(-1))), SerialisationError); + EXPECT_EQ(readNum(*makeNumSource(int64_t(-1))), std::numeric_limits::max()); + EXPECT_EQ(readNum(*makeNumSource(int64_t(-2))), std::numeric_limits::max() - 1); + /* The result doesn't depend on the source type - only the destination matters. */ + EXPECT_EQ(readNum(*makeNumSource(int32_t(-1))), std::numeric_limits::max()); + EXPECT_EQ(readNum(*makeNumSource(int16_t(-1))), std::numeric_limits::max()); +} + +} // namespace nix diff --git a/src/libutil-tests/source-accessor.cc b/src/libutil-tests/source-accessor.cc new file mode 100644 index 000000000000..b9da508659bb --- /dev/null +++ b/src/libutil-tests/source-accessor.cc @@ -0,0 +1,176 @@ +#include "nix/util/fs-sink.hh" +#include "nix/util/file-system.hh" +#include "nix/util/processes.hh" + +#include +#include +#include + +namespace nix { + +MATCHER_P2(HasContents, path, expected, "") +{ + auto stat = arg->maybeLstat(path); + if (!stat) { + *result_listener << arg->showPath(path) << " does not exist"; + return false; + } + if (stat->type != SourceAccessor::tRegular) { + *result_listener << arg->showPath(path) << " is not a regular file"; + return false; + } + auto actual = arg->readFile(path); + if (actual != expected) { + *result_listener << arg->showPath(path) << " has contents " << ::testing::PrintToString(actual); + return false; + } + return true; +} + +MATCHER_P2(HasSymlink, path, target, "") +{ + auto stat = arg->maybeLstat(path); + if (!stat) { + *result_listener << arg->showPath(path) << " does not exist"; + return false; + } + if (stat->type != SourceAccessor::tSymlink) { + *result_listener << arg->showPath(path) << " is not a symlink"; + return false; + } + auto actual = arg->readLink(path); + if (actual != target) { + *result_listener << arg->showPath(path) << " points to " << ::testing::PrintToString(actual); + return false; + } + return true; +} + +MATCHER_P2(HasDirectory, path, dirents, "") +{ + auto stat = arg->maybeLstat(path); + if (!stat) { + *result_listener << arg->showPath(path) << " does not exist"; + return false; + } + if (stat->type != SourceAccessor::tDirectory) { + *result_listener << arg->showPath(path) << " is not a directory"; + return false; + } + auto actual = arg->readDirectory(path); + std::set actualKeys, expectedKeys(dirents.begin(), dirents.end()); + for (auto & [k, _] : actual) + actualKeys.insert(k); + if (actualKeys != expectedKeys) { + *result_listener << arg->showPath(path) << " has entries " << ::testing::PrintToString(actualKeys); + return false; + } + return true; +} + +class FSSourceAccessorTest : public ::testing::Test +{ +protected: + std::filesystem::path tmpDir; + std::unique_ptr delTmpDir; + + void SetUp() override + { + tmpDir = nix::createTempDir(); + delTmpDir = std::make_unique(tmpDir, true); + } + + void TearDown() override + { + delTmpDir.reset(); + } +}; + +TEST_F(FSSourceAccessorTest, works) +{ +#ifdef _WIN32 + GTEST_SKIP() << "Broken on Windows"; +#endif + { + RestoreSink sink(false); + sink.dstPath = tmpDir; +#ifndef _WIN32 + sink.dirFd = openDirectory(tmpDir); +#endif + sink.createDirectory(CanonPath("subdir")); + sink.createRegularFile(CanonPath("file1"), [](CreateRegularFileSink & crf) { crf("content1"); }); + sink.createRegularFile(CanonPath("subdir/file2"), [](CreateRegularFileSink & crf) { crf("content2"); }); + sink.createSymlink(CanonPath("rootlink"), "target"); + sink.createDirectory(CanonPath("a")); + sink.createSymlink(CanonPath("a/dirlink"), "../subdir"); + } + + EXPECT_THAT(makeFSSourceAccessor(tmpDir / "file1"), HasContents(CanonPath::root, "content1")); + EXPECT_THAT(makeFSSourceAccessor(tmpDir / "rootlink"), HasSymlink(CanonPath::root, "target")); + EXPECT_THAT( + makeFSSourceAccessor(tmpDir), + HasDirectory(CanonPath::root, std::set{"file1", "subdir", "rootlink", "a"})); + EXPECT_THAT(makeFSSourceAccessor(tmpDir / "subdir"), HasDirectory(CanonPath::root, std::set{"file2"})); + + { + auto accessor = makeFSSourceAccessor(tmpDir); + EXPECT_THAT(accessor, HasContents(CanonPath("file1"), "content1")); + EXPECT_THAT(accessor, HasContents(CanonPath("subdir/file2"), "content2")); + + EXPECT_TRUE(accessor->pathExists(CanonPath("file1"))); + EXPECT_FALSE(accessor->pathExists(CanonPath("nonexistent"))); + + EXPECT_THROW(accessor->readFile(CanonPath("a/dirlink/file2")), SymlinkNotAllowed); + EXPECT_THROW(accessor->maybeLstat(CanonPath("a/dirlink/file2")), SymlinkNotAllowed); + EXPECT_THROW(accessor->readDirectory(CanonPath("a/dirlink")), SymlinkNotAllowed); + EXPECT_THROW(accessor->pathExists(CanonPath("a/dirlink/file2")), SymlinkNotAllowed); + } + + { + auto accessor = makeFSSourceAccessor(tmpDir / "nonexistent"); + EXPECT_FALSE(accessor->maybeLstat(CanonPath::root)); + EXPECT_THROW(accessor->readFile(CanonPath::root), SystemError); + } + + { + auto accessor = makeFSSourceAccessor(tmpDir, true); + EXPECT_EQ(accessor->getLastModified(), 0); + accessor->maybeLstat(CanonPath("file1")); + EXPECT_GT(accessor->getLastModified(), 0); + } +} + +/* ---------------------------------------------------------------------------- + * RestoreSink non-directory at root (no dirFd) + * --------------------------------------------------------------------------*/ + +TEST_F(FSSourceAccessorTest, RestoreSinkRegularFileAtRoot) +{ + auto filePath = tmpDir / "rootfile"; + { + RestoreSink sink(false); + sink.dstPath = filePath; + // No dirFd set - this tests the !dirFd path + sink.createRegularFile(CanonPath::root, [](CreateRegularFileSink & crf) { crf("root content"); }); + } + + EXPECT_THAT(makeFSSourceAccessor(filePath), HasContents(CanonPath::root, "root content")); +} + +TEST_F(FSSourceAccessorTest, RestoreSinkSymlinkAtRoot) +{ +#ifdef _WIN32 + GTEST_SKIP() << "symlinks have some problems under Wine"; +#endif + auto linkPath = tmpDir / "rootlink"; + { + RestoreSink sink(false); + sink.dstPath = linkPath; + // No dirFd set - this tests the !dirFd path + sink.createSymlink(CanonPath::root, "symlink_target"); + } + + EXPECT_THAT(makeFSSourceAccessor(linkPath), HasSymlink(CanonPath::root, "symlink_target")); +} + +} // namespace nix diff --git a/src/libutil-tests/spawn.cc b/src/libutil-tests/spawn.cc index cf3645260e12..715c0d17fb2f 100644 --- a/src/libutil-tests/spawn.cc +++ b/src/libutil-tests/spawn.cc @@ -1,5 +1,6 @@ #include +#include "nix/util/os-string.hh" #include "nix/util/processes.hh" namespace nix { @@ -7,30 +8,31 @@ namespace nix { #ifdef _WIN32 TEST(SpawnTest, spawnEcho) { - auto output = runProgram(RunOptions{.program = "cmd.exe", .args = {"/C", "echo", "hello world"}}); + auto output = + runProgram(RunOptions{.program = "cmd.exe", .args = {OS_STR("/C"), OS_STR("echo"), OS_STR("hello world")}}); ASSERT_EQ(output.first, 0); ASSERT_EQ(output.second, "\"hello world\"\r\n"); } -std::string windowsEscape(const std::string & str, bool cmd); +OsString windowsEscape(const OsString & str, bool cmd); TEST(SpawnTest, windowsEscape) { - auto empty = windowsEscape("", false); - ASSERT_EQ(empty, R"("")"); + auto empty = windowsEscape(L"", false); + ASSERT_EQ(empty, LR"("")"); // There's no quotes in this argument so the input should equal the output - auto backslashStr = R"(\\\\)"; + auto backslashStr = LR"(\\\\)"; auto backslashes = windowsEscape(backslashStr, false); ASSERT_EQ(backslashes, backslashStr); - auto nestedQuotes = windowsEscape(R"(he said: "hello there")", false); - ASSERT_EQ(nestedQuotes, R"("he said: \"hello there\"")"); + auto nestedQuotes = windowsEscape(LR"(he said: "hello there")", false); + ASSERT_EQ(nestedQuotes, LR"("he said: \"hello there\"")"); - auto middleQuote = windowsEscape(R"( \\\" )", false); - ASSERT_EQ(middleQuote, R"(" \\\\\\\" ")"); + auto middleQuote = windowsEscape(LR"( \\\" )", false); + ASSERT_EQ(middleQuote, LR"(" \\\\\\\" ")"); - auto space = windowsEscape("hello world", false); - ASSERT_EQ(space, R"("hello world")"); + auto space = windowsEscape(L"hello world", false); + ASSERT_EQ(space, LR"("hello world")"); } #endif } // namespace nix diff --git a/src/libutil-tests/unix/file-system-at.cc b/src/libutil-tests/unix/file-system-at.cc new file mode 100644 index 000000000000..609c4e2aa1d1 --- /dev/null +++ b/src/libutil-tests/unix/file-system-at.cc @@ -0,0 +1,137 @@ +#include + +#include "nix/util/file-system-at.hh" +#include "nix/util/file-system.hh" +#include "nix/util/fs-sink.hh" +#include "nix/util/processes.hh" + +#ifdef __linux__ +# include "nix/util/linux-namespaces.hh" + +# include +#endif + +#include + +namespace nix { + +using namespace nix::unix; + +/* ---------------------------------------------------------------------------- + * fchmodatTryNoFollow + * --------------------------------------------------------------------------*/ + +TEST(fchmodatTryNoFollow, works) +{ + std::filesystem::path tmpDir = nix::createTempDir(); + nix::AutoDelete delTmpDir(tmpDir, /*recursive=*/true); + + { + RestoreSink sink(/*startFsync=*/false); + sink.dstPath = tmpDir; + sink.dirFd = openDirectory(tmpDir); + sink.createRegularFile(CanonPath("file"), [](CreateRegularFileSink & crf) {}); + sink.createDirectory(CanonPath("dir")); + sink.createSymlink(CanonPath("filelink"), "file"); + sink.createSymlink(CanonPath("dirlink"), "dir"); + } + + ASSERT_NO_THROW(chmod(tmpDir / "file", 0644)); + ASSERT_NO_THROW(chmod(tmpDir / "dir", 0755)); + + auto dirFd = openDirectory(tmpDir); + ASSERT_TRUE(dirFd); + + struct ::stat st; + + /* Check that symlinks are not followed and targets are not changed. */ + + EXPECT_NO_THROW( + try { fchmodatTryNoFollow(dirFd.get(), CanonPath("filelink"), 0777); } catch (SysError & e) { + if (e.errNo != EOPNOTSUPP) + throw; + }); + ASSERT_EQ(stat((tmpDir / "file").c_str(), &st), 0); + EXPECT_EQ(st.st_mode & 0777, 0644); + + EXPECT_NO_THROW( + try { fchmodatTryNoFollow(dirFd.get(), CanonPath("dirlink"), 0777); } catch (SysError & e) { + if (e.errNo != EOPNOTSUPP) + throw; + }); + ASSERT_EQ(stat((tmpDir / "dir").c_str(), &st), 0); + EXPECT_EQ(st.st_mode & 0777, 0755); + + /* Check fchmodatTryNoFollow works on regular files and directories. */ + + EXPECT_NO_THROW(fchmodatTryNoFollow(dirFd.get(), CanonPath("file"), 0600)); + ASSERT_EQ(stat((tmpDir / "file").c_str(), &st), 0); + EXPECT_EQ(st.st_mode & 0777, 0600); + + EXPECT_NO_THROW((fchmodatTryNoFollow(dirFd.get(), CanonPath("dir"), 0700), 0)); + ASSERT_EQ(stat((tmpDir / "dir").c_str(), &st), 0); + EXPECT_EQ(st.st_mode & 0777, 0700); +} + +#ifdef __linux__ + +TEST(fchmodatTryNoFollow, fallbackWithoutProc) +{ + if (!userNamespacesSupported()) + GTEST_SKIP() << "User namespaces not supported"; + + std::filesystem::path tmpDir = nix::createTempDir(); + nix::AutoDelete delTmpDir(tmpDir, /*recursive=*/true); + + { + RestoreSink sink(/*startFsync=*/false); + sink.dstPath = tmpDir; + sink.dirFd = openDirectory(tmpDir); + sink.createRegularFile(CanonPath("file"), [](CreateRegularFileSink & crf) {}); + sink.createSymlink(CanonPath("link"), "file"); + } + + ASSERT_NO_THROW(chmod(tmpDir / "file", 0644)); + + Pid pid = startProcess( + [&] { + if (unshare(CLONE_NEWNS) == -1) + _exit(1); + + if (mount(0, "/", 0, MS_PRIVATE | MS_REC, 0) == -1) + _exit(1); + + if (mount("tmpfs", "/proc", "tmpfs", 0, 0) == -1) + _exit(1); + + auto dirFd = openDirectory(tmpDir); + if (!dirFd) + exit(1); + + try { + fchmodatTryNoFollow(dirFd.get(), CanonPath("file"), 0600); + } catch (SysError & e) { + _exit(1); + } + + try { + fchmodatTryNoFollow(dirFd.get(), CanonPath("link"), 0777); + } catch (SysError & e) { + if (e.errNo == EOPNOTSUPP) + _exit(0); /* Success. */ + } + + _exit(1); /* Didn't throw the expected exception. */ + }, + {.cloneFlags = CLONE_NEWUSER}); + + int status = pid.wait(); + ASSERT_TRUE(statusOk(status)); + + struct ::stat st; + ASSERT_EQ(stat((tmpDir / "file").c_str(), &st), 0); + EXPECT_EQ(st.st_mode & 0777, 0600); +} +#endif + +} // namespace nix diff --git a/src/libutil-tests/unix/meson.build b/src/libutil-tests/unix/meson.build new file mode 100644 index 000000000000..1baeeb3410c5 --- /dev/null +++ b/src/libutil-tests/unix/meson.build @@ -0,0 +1,3 @@ +sources += files( + 'file-system-at.cc', +) diff --git a/src/libutil-tests/url.cc b/src/libutil-tests/url.cc index 356a134dc9b3..efc322feb85a 100644 --- a/src/libutil-tests/url.cc +++ b/src/libutil-tests/url.cc @@ -112,19 +112,13 @@ TEST_P(FixGitURLTestSuite, parsesVariedGitUrls) EXPECT_EQ(actual.to_string(), p.expected); } -TEST(FixGitURLTestSuite, scpLikeNoUserParsesPoorly) +TEST(FixGitURLTestSuite, rejectScpLikeNoUser) { - // SCP-like URL (no user) - - // Cannot "to_string" this because has illegal path not starting - // with `/`. - EXPECT_EQ( - fixGitURL("github.com:owner/repo.git"), - (ParsedURL{ - .scheme = "file", - .authority = ParsedURL::Authority{}, - .path = {"github.com:owner", "repo.git"}, - })); + // SCP-like URL without user. Proper support can be implemented, but this is + // a deceptively deep feature - study existing implementations carefully. + EXPECT_THAT( + []() { fixGitURL("github.com:owner/repo.git"); }, + ::testing::ThrowsMessage(testing::HasSubstrIgnoreANSIMatcher("SCP-like URL"))); } TEST(FixGitURLTestSuite, properlyRejectFileURLWithAuthority) @@ -136,37 +130,24 @@ TEST(FixGitURLTestSuite, properlyRejectFileURLWithAuthority) testing::HasSubstrIgnoreANSIMatcher("file:// URL 'file://var/repos/x' has unexpected authority 'var'"))); } -TEST(FixGitURLTestSuite, scpLikePathLeadingSlashParsesPoorly) +TEST(FixGitURLTestSuite, rejectScpLikeNoUserLeadingSlash) { - // SCP-like URL (no user) - - // Cannot "to_string" this because has illegal path not starting - // with `/`. - EXPECT_EQ( - fixGitURL("github.com:/owner/repo.git"), - (ParsedURL{ - .scheme = "file", - .authority = ParsedURL::Authority{}, - .path = {"github.com:", "owner", "repo.git"}, - })); + EXPECT_THAT( + []() { fixGitURL("github.com:/owner/repo.git"); }, + ::testing::ThrowsMessage(testing::HasSubstrIgnoreANSIMatcher("SCP-like URL"))); } -TEST(FixGitURLTestSuite, relativePathParsesPoorly) +TEST(FixGitURLTestSuite, relativePath) { - // Relative path (becomes file:// absolute) - - // Cannot "to_string" this because has illegal path not starting - // with `/`. + // Relative path - parsed as file path without authority + auto parsed = fixGitURL("relative/repo"); EXPECT_EQ( - fixGitURL("relative/repo"), + parsed, (ParsedURL{ .scheme = "file", - .authority = - ParsedURL::Authority{ - .hostType = ParsedURL::Authority::HostType::Name, - .host = "", - }, - .path = {"relative", "repo"}})); + .path = {"relative", "repo"}, + })); + EXPECT_EQ(parsed.to_string(), "file:relative/repo"); } struct ParseURLSuccessCase @@ -975,4 +956,108 @@ TEST(nix, isValidSchemeName) ASSERT_FALSE(isValidSchemeName("http ")); } +/* ---------------------------------------------------------------------------- + * pathToUrlPath / urlPathToPath + * --------------------------------------------------------------------------*/ + +struct UrlPathTestCase +{ + std::string_view urlString; + ParsedURL urlParsed; + std::filesystem::path path; + std::string description; +}; + +class UrlPathTest : public ::testing::TestWithParam +{}; + +TEST_P(UrlPathTest, pathToUrlPath) +{ + const auto & testCase = GetParam(); + auto urlPath = pathToUrlPath(testCase.path); + EXPECT_EQ(urlPath, testCase.urlParsed.path); +} + +TEST_P(UrlPathTest, urlPathToPath) +{ + const auto & testCase = GetParam(); + auto path = urlPathToPath(testCase.urlParsed.path); + EXPECT_EQ(path, testCase.path); +} + +TEST_P(UrlPathTest, urlToString) +{ + const auto & testCase = GetParam(); + EXPECT_EQ(testCase.urlParsed.to_string(), testCase.urlString); +} + +TEST_P(UrlPathTest, stringToUrl) +{ + const auto & testCase = GetParam(); + auto parsed = parseURL(std::string{testCase.urlString}); + EXPECT_EQ(parsed, testCase.urlParsed); +} + +#ifndef _WIN32 + +INSTANTIATE_TEST_SUITE_P( + Unix, + UrlPathTest, + ::testing::Values( + UrlPathTestCase{ + .urlString = "file:///foo/bar/baz", + .urlParsed = + ParsedURL{ + .scheme = "file", + .authority = ParsedURL::Authority{}, + .path = {"", "foo", "bar", "baz"}, + }, + .path = "/foo/bar/baz", + .description = "absolute_path", + }, + UrlPathTestCase{ + .urlString = "file:///", + .urlParsed = + ParsedURL{ + .scheme = "file", + .authority = ParsedURL::Authority{}, + .path = {"", ""}, + }, + .path = "/", + .description = "root_path", + }), + [](const auto & info) { return info.param.description; }); + +#else // _WIN32 + +INSTANTIATE_TEST_SUITE_P( + Windows, + UrlPathTest, + ::testing::Values( + UrlPathTestCase{ + .urlString = "file:///C:/foo/bar/baz", + .urlParsed = + ParsedURL{ + .scheme = "file", + .authority = ParsedURL::Authority{}, + .path = {"", "C:", "foo", "bar", "baz"}, + }, + .path = L"C:\\foo\\bar\\baz", + .description = "absolute_path", + }, + UrlPathTestCase{ + .urlString = "file:///C:/", + .urlParsed = + ParsedURL{ + .scheme = "file", + .authority = ParsedURL::Authority{}, + .path = {"", "C:", ""}, + }, + .path = L"C:\\", + .description = "drive_root", + }), + [](const auto & info) { return info.param.description; }); + +#endif // _WIN32 + } // namespace nix diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index 0291d6827290..847e339b6d2d 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -32,7 +32,7 @@ static ArchiveSettings archiveSettings; static GlobalConfig::Register rArchiveSettings(&archiveSettings); -PathFilter defaultPathFilter = [](const Path &) { return true; }; +PathFilter defaultPathFilter = [](const std::string &) { return true; }; void SourceAccessor::dumpPath(const CanonPath & path, Sink & sink, PathFilter & filter) { @@ -101,14 +101,14 @@ void SourceAccessor::dumpPath(const CanonPath & path, Sink & sink, PathFilter & }(path); } -time_t dumpPathAndGetMtime(const Path & path, Sink & sink, PathFilter & filter) +time_t dumpPathAndGetMtime(const std::filesystem::path & path, Sink & sink, PathFilter & filter) { auto path2 = PosixSourceAccessor::createAtRoot(path, /*trackLastModified=*/true); path2.dumpPath(sink, filter); return path2.accessor->getLastModified().value(); } -void dumpPath(const Path & path, Sink & sink, PathFilter & filter) +void dumpPath(const std::filesystem::path & path, Sink & sink, PathFilter & filter) { dumpPathAndGetMtime(path, sink, filter); } @@ -201,7 +201,7 @@ static void parse(FileSystemObjectSink & sink, Source & source, const CanonPath else if (type == "directory") { sink.createDirectory(path, [&](FileSystemObjectSink & dirSink, const CanonPath & relDirPath) { - std::map names; + std::map names; std::string prevName; diff --git a/src/libutil/args.cc b/src/libutil/args.cc index bd3dc9c95dfa..0bef95b3331c 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -284,7 +284,7 @@ void RootArgs::parseCmdline(const Strings & _cmdline, bool allowShebang) // executable file, and it starts with "#!". Strings savedArgs; if (allowShebang) { - auto script = *cmdline.begin(); + std::filesystem::path script = *cmdline.begin(); try { std::ifstream stream(script); char shebang[3] = {0, 0, 0}; @@ -310,8 +310,8 @@ void RootArgs::parseCmdline(const Strings & _cmdline, bool allowShebang) for (const auto & word : parseShebangContent(shebangContent)) { cmdline.push_back(word); } - cmdline.push_back(script); - commandBaseDir = dirOf(script); + cmdline.push_back(script.string()); + commandBaseDir = script.parent_path(); for (auto pos = savedArgs.begin(); pos != savedArgs.end(); pos++) cmdline.push_back(*pos); } diff --git a/src/libutil/canon-path.cc b/src/libutil/canon-path.cc index 22ca3e066a92..e62abdf31964 100644 --- a/src/libutil/canon-path.cc +++ b/src/libutil/canon-path.cc @@ -23,6 +23,16 @@ static void ensureNoNullBytes(std::string_view s) } } +CanonPath CanonPath::fromFilename(std::string_view segment) +{ + auto res = CanonPath(segment); + /* Use existing canonicalisation logic for CanonPath to check that the segment + is already a valid filename. */ + if (segment != res.rel() || std::ranges::distance(res) != 1) + throw BadCanonPath("invalid filename '%s'", segment); + return res; +} + CanonPath::CanonPath(std::string_view raw) : path(absPathPure(concatStrings("/", raw))) { diff --git a/src/libutil/compression-algo.cc b/src/libutil/compression-algo.cc new file mode 100644 index 000000000000..dab53b124f13 --- /dev/null +++ b/src/libutil/compression-algo.cc @@ -0,0 +1,46 @@ +#include "nix/util/compression-algo.hh" +#include "nix/util/error.hh" +#include "nix/util/types.hh" + +#include + +namespace nix { + +CompressionAlgo parseCompressionAlgo(std::string_view method, bool suggestions) +{ +#define NIX_COMPRESSION_ALGO_FROM_STRING(name, value) {name, CompressionAlgo::value}, + static const std::unordered_map lookupTable = { + NIX_FOR_EACH_COMPRESSION_ALGO(NIX_COMPRESSION_ALGO_FROM_STRING)}; +#undef NIX_COMPRESSION_ALGO_FROM_STRING + + if (auto it = lookupTable.find(method); it != lookupTable.end()) + return it->second; + + ErrorInfo err = {.level = lvlError, .msg = HintFmt("unknown compression method '%s'", method)}; + + if (suggestions) { + static const StringSet allNames = [&]() { + StringSet res; + for (auto & [name, _] : lookupTable) + res.emplace(name); + return res; + }(); + err.suggestions = Suggestions::bestMatches(allNames, method); + } + + throw UnknownCompressionMethod(std::move(err)); +} + +std::string showCompressionAlgo(CompressionAlgo method) +{ + switch (method) { +#define NIX_COMPRESSION_ALGO_TO_STRING(name, value) \ + case CompressionAlgo::value: \ + return name; + NIX_FOR_EACH_COMPRESSION_ALGO(NIX_COMPRESSION_ALGO_TO_STRING); +#undef NIX_COMPRESSION_ALGO_TO_STRING + } + unreachable(); +} + +} // namespace nix diff --git a/src/libutil/compression-settings.cc b/src/libutil/compression-settings.cc new file mode 100644 index 000000000000..6e30811ef1b2 --- /dev/null +++ b/src/libutil/compression-settings.cc @@ -0,0 +1,68 @@ +#include "nix/util/configuration.hh" +#include "nix/util/compression-settings.hh" +#include "nix/util/json-impls.hh" +#include "nix/util/config-impl.hh" +#include "nix/util/abstract-setting-to-json.hh" + +#include + +namespace nix { + +template<> +CompressionAlgo BaseSetting::parse(const std::string & str) const +try { + return parseCompressionAlgo(str, /*suggestions=*/true); +} catch (UnknownCompressionMethod & e) { + throw UsageError(e.info().suggestions, "option '%s' has invalid value '%s'", name, str); +} + +template<> +std::optional BaseSetting>::parse(const std::string & str) const +try { + if (str.empty()) + return std::nullopt; + return parseCompressionAlgo(str, /*suggestions=*/true); +} catch (UnknownCompressionMethod & e) { + throw UsageError(e.info().suggestions, "option '%s' has invalid value '%s'", name, str); +} + +template<> +struct BaseSetting::trait +{ + static constexpr bool appendable = false; +}; + +template<> +struct BaseSetting>::trait +{ + static constexpr bool appendable = false; +}; + +template<> +std::string BaseSetting::to_string() const +{ + return std::string{showCompressionAlgo(value)}; +} + +template<> +std::string BaseSetting>::to_string() const +{ + if (value) + return std::string{showCompressionAlgo(*value)}; + return ""; +} + +/* Same as with all settings - empty string means std::nullopt. */ +template<> +struct json_avoids_null : std::true_type +{}; + +#define NIX_COMPRESSION_JSON(name, value) {CompressionAlgo::value, name}, +NLOHMANN_JSON_SERIALIZE_ENUM(CompressionAlgo, {NIX_FOR_EACH_COMPRESSION_ALGO(NIX_COMPRESSION_JSON)}); +#undef NIX_COMPRESSION_JSON + +/* Explicit instantiation of templates */ +template class BaseSetting; +template class BaseSetting>; + +} // namespace nix diff --git a/src/libutil/compression.cc b/src/libutil/compression.cc index 36b476e9ad5f..8b55c44a2db2 100644 --- a/src/libutil/compression.cc +++ b/src/libutil/compression.cc @@ -69,24 +69,53 @@ struct ArchiveDecompressionSource : Source } }; +/* Happens to match enum names. */ +#define NIX_FOR_EACH_LA_ALGO(MACRO) \ + MACRO(bzip2) \ + MACRO(compress) \ + MACRO(grzip) \ + MACRO(gzip) \ + MACRO(lrzip) \ + MACRO(lz4) \ + MACRO(lzip) \ + MACRO(lzma) \ + MACRO(lzop) \ + MACRO(xz) \ + MACRO(zstd) + struct ArchiveCompressionSink : CompressionSink { Sink & nextSink; struct archive * archive; - ArchiveCompressionSink(Sink & nextSink, std::string format, bool parallel, int level = COMPRESSION_LEVEL_DEFAULT) + ArchiveCompressionSink( + Sink & nextSink, CompressionAlgo method, bool parallel, int level = COMPRESSION_LEVEL_DEFAULT) : nextSink(nextSink) { archive = archive_write_new(); if (!archive) throw Error("failed to initialize libarchive"); - check(archive_write_add_filter_by_name(archive, format.c_str()), "couldn't initialize compression (%s)"); + + auto [addFilter, format] = [method]() -> std::pair { + switch (method) { + case CompressionAlgo::none: + case CompressionAlgo::brotli: + unreachable(); +#define NIX_DEF_LA_ALGO_CASE(algo) \ + case CompressionAlgo::algo: \ + return {archive_write_add_filter_##algo, #algo}; + NIX_FOR_EACH_LA_ALGO(NIX_DEF_LA_ALGO_CASE) +#undef NIX_DEF_LA_ALGO_CASE + } + unreachable(); + }(); + + check(addFilter(archive), "couldn't initialize compression (%s)"); check(archive_write_set_format_raw(archive)); if (parallel) - check(archive_write_set_filter_option(archive, format.c_str(), "threads", "0")); + check(archive_write_set_filter_option(archive, format, "threads", "0")); if (level != COMPRESSION_LEVEL_DEFAULT) - check(archive_write_set_filter_option( - archive, format.c_str(), "compression-level", std::to_string(level).c_str())); + check(archive_write_set_filter_option(archive, format, "compression-level", std::to_string(level).c_str())); // disable internal buffering check(archive_write_set_bytes_per_block(archive, 0)); // disable output padding @@ -289,22 +318,23 @@ struct BrotliCompressionSink : ChunkedCompressionSink } }; -ref makeCompressionSink(const std::string & method, Sink & nextSink, const bool parallel, int level) +ref makeCompressionSink(CompressionAlgo method, Sink & nextSink, const bool parallel, int level) { - std::vector la_supports = { - "bzip2", "compress", "grzip", "gzip", "lrzip", "lz4", "lzip", "lzma", "lzop", "xz", "zstd"}; - if (std::find(la_supports.begin(), la_supports.end(), method) != la_supports.end()) { - return make_ref(nextSink, method, parallel, level); - } - if (method == "none") + switch (method) { + case CompressionAlgo::none: return make_ref(nextSink); - else if (method == "br") + case CompressionAlgo::brotli: return make_ref(nextSink); - else - throw UnknownCompressionMethod("unknown compression method '%s'", method); + /* Everything else is supported via libarchive. */ +#define NIX_DEF_LA_ALGO_CASE(algo) case CompressionAlgo::algo: + NIX_FOR_EACH_LA_ALGO(NIX_DEF_LA_ALGO_CASE) + return make_ref(nextSink, method, parallel, level); +#undef NIX_DEF_LA_ALGO_CASE + } + unreachable(); } -std::string compress(const std::string & method, std::string_view in, const bool parallel, int level) +std::string compress(CompressionAlgo method, std::string_view in, const bool parallel, int level) { StringSink ssink; auto sink = makeCompressionSink(method, ssink, parallel, level); diff --git a/src/libutil/configuration.cc b/src/libutil/configuration.cc index 407320a6b51b..d99d362c4a55 100644 --- a/src/libutil/configuration.cc +++ b/src/libutil/configuration.cc @@ -103,7 +103,7 @@ void Config::getSettings(std::map & res, bool overridd */ static void parseConfigFiles( const std::string & contents, - const std::string & path, + const std::filesystem::path & path, std::vector> & parsedContents) { unsigned int pos = 0; @@ -122,7 +122,7 @@ static void parseConfigFiles( continue; if (tokens.size() < 2) - throw UsageError("syntax error in configuration line '%1%' in '%2%'", line, path); + throw UsageError("syntax error in configuration line '%s' in %s", line, PathFmt(path)); auto include = false; auto ignoreMissing = false; @@ -135,8 +135,9 @@ static void parseConfigFiles( if (include) { if (tokens.size() != 2) - throw UsageError("syntax error in configuration line '%1%' in '%2%'", line, path); - auto p = absPath(tokens[1], dirOf(path)); + throw UsageError("syntax error in configuration line '%1%' in %s", line, PathFmt(path)); + auto parent = path.parent_path(); + auto p = absPath(std::filesystem::path{tokens[1]}, &parent); if (pathExists(p)) { try { std::string includedContents = readFile(p); @@ -145,13 +146,13 @@ static void parseConfigFiles( // TODO: Do we actually want to ignore this? Or is it better to fail? } } else if (!ignoreMissing) { - throw Error("file '%1%' included from '%2%' not found", p, path); + throw Error("file %s included from %s not found", PathFmt(p), PathFmt(path)); } continue; } if (tokens[1] != "=") - throw UsageError("syntax error in configuration line '%1%' in '%2%'", line, path); + throw UsageError("syntax error in configuration line '%s' in %s", line, PathFmt(path)); std::string name = std::move(tokens[0]); @@ -393,6 +394,28 @@ std::string BaseSetting::to_string() const return concatStringsSep(" ", value); } +template<> +std::set BaseSetting>::parse(const std::string & str) const +{ + auto tokens = tokenizeString(str); + return {tokens.begin(), tokens.end()}; +} + +template<> +void BaseSetting>::appendOrSet(std::set newValue, bool append) +{ + if (!append) + value.clear(); + value.insert(std::make_move_iterator(newValue.begin()), std::make_move_iterator(newValue.end())); +} + +template<> +std::string BaseSetting>::to_string() const +{ + return concatStringsSep( + " ", value | std::views::transform([](const auto & p) { return p.string(); }) | std::ranges::to()); +} + template<> std::set BaseSetting>::parse(const std::string & str) const { @@ -402,6 +425,10 @@ std::set BaseSetting>::parse( res.insert(thisXpFeature.value()); else if (stabilizedFeatures.count(s)) debug("experimental feature '%s' is now stable", s); + else if (s == "no-url-literals") + warn( + "experimental feature '%s' has been stabilized and renamed; use 'lint-url-literals = fatal' setting instead", + s); else warn("unknown experimental feature '%s'", s); } @@ -456,7 +483,7 @@ std::string BaseSetting::to_string() const [](const auto & kvpair) { return kvpair.first + "=" + kvpair.second; }); } -static Path parsePath(const AbstractSetting & s, const std::string & str) +static AbsolutePath parseAbsolutePath(const AbstractSetting & s, const std::string & str) { if (str == "") throw UsageError("setting '%s' is a path and paths cannot be empty", s.name); @@ -467,7 +494,9 @@ static Path parsePath(const AbstractSetting & s, const std::string & str) template<> std::filesystem::path BaseSetting::parse(const std::string & str) const { - return parsePath(*this, str); + if (str == "") + throw UsageError("setting '%s' is a path and paths cannot be empty", name); + return str; } template<> @@ -477,17 +506,28 @@ std::string BaseSetting::to_string() const } template<> -std::optional -BaseSetting>::parse(const std::string & str) const +AbsolutePath BaseSetting::parse(const std::string & str) const +{ + return parseAbsolutePath(*this, str); +} + +template<> +std::string BaseSetting::to_string() const +{ + return value.string(); +} + +template<> +std::optional BaseSetting>::parse(const std::string & str) const { if (str == "") return std::nullopt; else - return parsePath(*this, str); + return parseAbsolutePath(*this, str); } template<> -std::string BaseSetting>::to_string() const +std::string BaseSetting>::to_string() const { return value ? value->string() : ""; } @@ -506,47 +546,9 @@ template class BaseSetting; template class BaseSetting; template class BaseSetting>; template class BaseSetting; -template class BaseSetting>; - -PathSetting::PathSetting( - Config * options, - const Path & def, - const std::string & name, - const std::string & description, - const StringSet & aliases) - : BaseSetting(def, true, name, description, aliases) -{ - options->addSetting(this); -} - -Path PathSetting::parse(const std::string & str) const -{ - return parsePath(*this, str); -} - -OptionalPathSetting::OptionalPathSetting( - Config * options, - const std::optional & def, - const std::string & name, - const std::string & description, - const StringSet & aliases) - : BaseSetting>(def, true, name, description, aliases) -{ - options->addSetting(this); -} - -std::optional OptionalPathSetting::parse(const std::string & str) const -{ - if (str == "") - return std::nullopt; - else - return parsePath(*this, str); -} - -void OptionalPathSetting::operator=(const std::optional & v) -{ - this->assign(v); -} +template class BaseSetting; +template class BaseSetting>; +template class BaseSetting>; bool ExperimentalFeatureSettings::isEnabled(const ExperimentalFeature & feature) const { diff --git a/src/libutil/current-process.cc b/src/libutil/current-process.cc index 5c48a4f77700..8f7150c97de6 100644 --- a/src/libutil/current-process.cc +++ b/src/libutil/current-process.cc @@ -35,7 +35,7 @@ unsigned int getMaxCPU() if (!cgroupFS) return 0; - auto cpuFile = *cgroupFS + "/" + getCurrentCgroup() + "/cpu.max"; + auto cpuFile = *cgroupFS / getCurrentCgroup().rel() / "cpu.max"; auto cpuMax = readFile(cpuFile); auto cpuMaxParts = tokenizeString>(cpuMax, " \n"); @@ -120,9 +120,9 @@ void restoreProcessContext(bool restoreMounts) ////////////////////////////////////////////////////////////////////// -std::optional getSelfExe() +std::optional getSelfExe() { - static auto cached = []() -> std::optional { + static auto cached = []() -> std::optional { #if defined(__linux__) || defined(__GNU__) return readLink(std::filesystem::path{"/proc/self/exe"}); #elif defined(__APPLE__) @@ -154,7 +154,7 @@ std::optional getSelfExe() // serialized to JSON and evaluated as a Nix string. path.pop_back(); - return Path(path.begin(), path.end()); + return std::string(path.begin(), path.end()); #else return std::nullopt; #endif diff --git a/src/libutil/environment-variables.cc b/src/libutil/environment-variables.cc index f2f24f7be10a..9d44ad65d8db 100644 --- a/src/libutil/environment-variables.cc +++ b/src/libutil/environment-variables.cc @@ -1,8 +1,6 @@ #include "nix/util/util.hh" #include "nix/util/environment-variables.hh" -extern char ** environ __attribute__((weak)); - namespace nix { std::optional getEnv(const std::string & key) @@ -21,24 +19,18 @@ std::optional getEnvNonEmpty(const std::string & key) return value; } -StringMap getEnv() +std::optional getEnvOsNonEmpty(const OsString & key) { - StringMap env; - for (size_t i = 0; environ[i]; ++i) { - auto s = environ[i]; - auto eq = strchr(s, '='); - if (!eq) - // invalid env, just keep going - continue; - env.emplace(std::string(s, eq), std::string(eq + 1)); - } - return env; + auto value = getEnvOs(key); + if (value == OS_STR("")) + return {}; + return value; } void clearEnv() { - for (auto & name : getEnv()) - unsetenv(name.first.c_str()); + for (auto & [name, value] : getEnvOs()) + unsetEnvOs(name.c_str()); } void replaceEnv(const StringMap & newEnv) diff --git a/src/libutil/error.cc b/src/libutil/error.cc index 35e42823ce63..8d6a1d9d9f5b 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -2,6 +2,7 @@ #include "nix/util/error.hh" #include "nix/util/environment-variables.hh" +#include "nix/util/exit.hh" #include "nix/util/signals.hh" #include "nix/util/terminal.hh" #include "nix/util/position.hh" @@ -26,18 +27,25 @@ void throwExceptionSelfCheck() "C++ exception handling is broken. This would appear to be a problem with the way Nix was compiled and/or linked and/or loaded."); } +void BaseError::recalcWhat() const +{ + std::ostringstream oss; + showErrorInfo(oss, err, loggerSettings.showTrace); + what_ = oss.str(); +} + // c++ std::exception descendants must have a 'const char* what()' function. // This stringifies the error and caches it for use by what(), or similarly by msg(). const std::string & BaseError::calcWhat() const { - if (what_.has_value()) - return *what_; - else { - std::ostringstream oss; - showErrorInfo(oss, err, loggerSettings.showTrace); - what_ = oss.str(); - return *what_; - } + if (!what_.has_value()) + recalcWhat(); + return *what_; +} + +bool BaseError::hasPos() const +{ + return err.pos.get() && *err.pos.get(); } std::optional ErrorInfo::programName = std::nullopt; @@ -421,13 +429,20 @@ std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool s */ static void writeErr(std::string_view buf) { + Descriptor fd = getStandardError(); while (!buf.empty()) { - auto n = write(STDERR_FILENO, buf.data(), buf.size()); +#ifdef _WIN32 + DWORD n; + if (!WriteFile(fd, buf.data(), buf.size(), &n, NULL)) + abort(); +#else + auto n = ::write(fd, buf.data(), buf.size()); if (n < 0) { if (errno == EINTR) continue; abort(); } +#endif buf = buf.substr(n); } } @@ -455,4 +470,41 @@ void unreachable(std::source_location loc) panic(std::string_view(buf, std::min(static_cast(sizeof(buf)), n))); } +int handleExceptions(const std::string & programName, fun body) +{ + ReceiveInterrupts receiveInterrupts; // FIXME: need better place for this + + ErrorInfo::programName = baseNameOf(programName); + + auto doLog = [&](BaseError & e) { + try { + logError(e.info()); + } catch (...) { + printError(ANSI_RED "error:" ANSI_NORMAL " Exception while printing an exception."); + } + }; + + std::string error = ANSI_RED "error:" ANSI_NORMAL " "; + try { + body(); + } catch (Exit & e) { + return e.status; + } catch (UsageError & e) { + doLog(e); + printError("\nTry '%1% --help' for more information.", programName); + return 1; + } catch (BaseError & e) { + doLog(e); + return e.info().status; + } catch (std::bad_alloc & e) { + printError(error + "out of memory"); + return 1; + } catch (std::exception & e) { + printError(error + e.what()); + return 1; + } + + return 0; +} + } // namespace nix diff --git a/src/libutil/executable-path.cc b/src/libutil/executable-path.cc index 75ab91f3a163..ec3520fd0ca6 100644 --- a/src/libutil/executable-path.cc +++ b/src/libutil/executable-path.cc @@ -20,17 +20,23 @@ ExecutablePath ExecutablePath::load() } ExecutablePath ExecutablePath::parse(const OsString & path) +{ + ExecutablePath ret; + ret.parseAppend(path); + return ret; +} + +void ExecutablePath::parseAppend(const OsString & path) { auto strings = path.empty() ? (std::list{}) : basicSplitString, OsChar>(path, path_var_separator); - std::vector ret; - ret.reserve(strings.size()); + directories.reserve(directories.size() + strings.size()); std::transform( std::make_move_iterator(strings.begin()), std::make_move_iterator(strings.end()), - std::back_inserter(ret), + std::back_inserter(directories), [](OsString && str) { return std::filesystem::path{ str.empty() @@ -45,13 +51,11 @@ ExecutablePath ExecutablePath::parse(const OsString & path) : std::move(str), }; }); - - return {ret}; } OsString ExecutablePath::render() const { - std::vector path2; + std::vector path2; path2.reserve(directories.size()); for (auto & p : directories) path2.push_back(p.native()); diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index ad692ff3d243..334a410bb863 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -139,48 +139,6 @@ constexpr std::array xpFeatureDetails )", .trackingUrl = "https://github.com/NixOS/nix/milestone/47", }, - { - .tag = Xp::NoUrlLiterals, - .name = "no-url-literals", - .description = R"( - Disallow unquoted URLs as part of the Nix language syntax. The Nix - language allows for URL literals, like so: - - ``` - $ nix repl - Welcome to Nix 2.15.0. Type :? for help. - - nix-repl> http://foo - "http://foo" - ``` - - But enabling this experimental feature causes the Nix parser to - throw an error when encountering a URL literal: - - ``` - $ nix repl --extra-experimental-features 'no-url-literals' - Welcome to Nix 2.15.0. Type :? for help. - - nix-repl> http://foo - error: URL literals are disabled - - at «string»:1:1: - - 1| http://foo - | ^ - - ``` - - While this is currently an experimental feature, unquoted URLs are - being deprecated and their usage is discouraged. - - The reason is that, as opposed to path literals, URLs have no - special properties that distinguish them from regular strings, URLs - containing parameters have to be quoted anyway, and unquoted URLs - may confuse external tooling. - )", - .trackingUrl = "https://github.com/NixOS/nix/milestone/44", - }, { .tag = Xp::FetchClosure, .name = "fetch-closure", @@ -294,7 +252,7 @@ constexpr std::array xpFeatureDetails .description = R"( Enables support for external builders / sandbox providers. )", - .trackingUrl = "", + .trackingUrl = "https://github.com/NixOS/nix/milestone/62", }, { .tag = Xp::BLAKE3Hashes, @@ -302,7 +260,7 @@ constexpr std::array xpFeatureDetails .description = R"( Enables support for BLAKE3 hashes. )", - .trackingUrl = "", + .trackingUrl = "https://github.com/NixOS/nix/milestone/60", }, { .tag = Xp::BuildTimeFetchTree, @@ -410,7 +368,7 @@ std::set parseFeatures(const StringSet & rawFeatures) } MissingExperimentalFeature::MissingExperimentalFeature(ExperimentalFeature feature, std::string reason) - : Error( + : CloneableError( "experimental Nix feature '%1%' is disabled%2%; add '--extra-experimental-features %1%' to enable it", showExperimentalFeature(feature), Uncolored(optionalBracket(" (", reason, ")"))) diff --git a/src/libutil/file-content-address.cc b/src/libutil/file-content-address.cc index df1b09f6e4bd..caf12f81e6c0 100644 --- a/src/libutil/file-content-address.cc +++ b/src/libutil/file-content-address.cc @@ -75,7 +75,7 @@ void dumpPath(const SourcePath & path, Sink & sink, FileSerialisationMethod meth } } -void restorePath(const Path & path, Source & source, FileSerialisationMethod method, bool startFsync) +void restorePath(const std::filesystem::path & path, Source & source, FileSerialisationMethod method, bool startFsync) { switch (method) { case FileSerialisationMethod::Flat: diff --git a/src/libutil/file-descriptor.cc b/src/libutil/file-descriptor.cc index 6e07e6e8818d..61c444d4fe78 100644 --- a/src/libutil/file-descriptor.cc +++ b/src/libutil/file-descriptor.cc @@ -1,37 +1,221 @@ #include "nix/util/serialise.hh" #include "nix/util/util.hh" +#include "nix/util/signals.hh" +#include #include #include #ifdef _WIN32 # include # include -# include "nix/util/windows-error.hh" +#else +# include #endif namespace nix { +namespace { + +enum class PollDirection { In, Out }; + +/** + * Retry an I/O operation if it fails with EAGAIN/EWOULDBLOCK. + * + * On Unix, polls the fd and retries. On Windows, just calls `f` once. + * + * This retry logic is needed to handle non-blocking reads/writes. This + * is needed in the buildhook, because somehow the json logger file + * descriptor ends up being non-blocking and breaks remote-building. + * + * @todo Get rid of buildhook and remove this logic again + * (https://github.com/NixOS/nix/issues/12688) + */ +template +auto retryOnBlock([[maybe_unused]] Descriptor fd, [[maybe_unused]] PollDirection dir, F && f) -> decltype(f()) +{ +#ifndef _WIN32 + while (true) { + try { + return std::forward(f)(); + } catch (SystemError & e) { + if (e.is(std::errc::resource_unavailable_try_again) || e.is(std::errc::operation_would_block)) { + struct pollfd pfd; + pfd.fd = fd; + pfd.events = dir == PollDirection::In ? POLLIN : POLLOUT; + if (poll(&pfd, 1, -1) == -1) + throw SysError("poll on file descriptor failed"); + continue; + } + throw; + } + } +#else + return std::forward(f)(); +#endif +} + +} // namespace + +void readFull(Descriptor fd, char * buf, size_t count) +{ + while (count) { + checkInterrupt(); + auto res = retryOnBlock( + fd, PollDirection::In, [&]() { return read(fd, {reinterpret_cast(buf), count}); }); + if (res == 0) + throw EndOfFile("unexpected end-of-file"); + count -= res; + buf += res; + } +} + +std::string readLine(Descriptor fd, bool eofOk, char terminator) +{ + std::string s; + while (1) { + checkInterrupt(); + char ch; + // FIXME: inefficient + auto rd = retryOnBlock(fd, PollDirection::In, [&]() -> size_t { + try { + return read(fd, {reinterpret_cast(&ch), 1}); + } catch (SystemError & e) { + // On pty masters, EIO signals that the slave side closed, + // which is semantically EOF. Map it to a zero-length read + // so the existing EOF path handles it. + if (e.is(std::errc::io_error)) + return 0; + throw; + } + }); + if (rd == 0) { + if (eofOk) + return s; + else + throw EndOfFile("unexpected EOF reading a line"); + } else { + if (ch == terminator) + return s; + s += ch; + } + } +} + +void writeFull(Descriptor fd, std::string_view s, bool allowInterrupts) +{ + while (!s.empty()) { + if (allowInterrupts) + checkInterrupt(); + auto res = retryOnBlock(fd, PollDirection::Out, [&]() { + return write(fd, {reinterpret_cast(s.data()), s.size()}, allowInterrupts); + }); + if (res > 0) + s.remove_prefix(res); + } +} + void writeLine(Descriptor fd, std::string s) { s += '\n'; writeFull(fd, s); } -std::string drainFD(Descriptor fd, bool block, const size_t reserveSize) +std::string readFile(Descriptor fd) +{ + auto size = getFileSize(fd); + // We can't rely on size being correct, most files in /proc have a nominal size of 0 + return drainFD(fd, {.size = size, .expected = false}); +} + +void drainFD(Descriptor fd, Sink & sink, DrainFdSinkOpts opts) +{ +#ifndef _WIN32 + // silence GCC maybe-uninitialized warning in finally + int saved = 0; + + if (!opts.block) { + saved = fcntl(fd, F_GETFL); + if (fcntl(fd, F_SETFL, saved | O_NONBLOCK) == -1) + throw SysError("making file descriptor non-blocking"); + } + + Finally finally([&]() { + if (!opts.block) { + if (fcntl(fd, F_SETFL, saved) == -1) + throw SysError("making file descriptor blocking"); + } + }); +#endif + + size_t bytesRead = 0; + std::array buf; + while (1) { + checkInterrupt(); + + size_t toRead = buf.size(); + if (opts.expectedSize) { + size_t remaining = *opts.expectedSize - bytesRead; + if (remaining == 0) + break; + toRead = std::min(toRead, remaining); + } + + size_t n; + try { + n = read(fd, std::span(buf.data(), toRead)); + } catch (SystemError & e) { +#ifndef _WIN32 + if (!opts.block + && (e.is(std::errc::resource_unavailable_try_again) || e.is(std::errc::operation_would_block))) + break; +#endif + throw; + } + + if (n == 0) { + if (opts.expectedSize && bytesRead < *opts.expectedSize) + throw EndOfFile("unexpected end-of-file"); + break; + } + + bytesRead += n; + sink(std::string_view(reinterpret_cast(buf.data()), n)); + } +} + +std::string drainFD(Descriptor fd, DrainFdOpts opts) { // the parser needs two extra bytes to append terminating characters, other users will // not care very much about the extra memory. + size_t reserveSize = opts.expected ? 0 : opts.size; StringSink sink(reserveSize + 2); -#ifdef _WIN32 - // non-blocking is not supported this way on Windows - assert(block); - drainFD(fd, sink); -#else - drainFD(fd, sink, block); + DrainFdSinkOpts sinkOpts{ + .expectedSize = opts.expected ? std::optional(opts.size) : std::nullopt, +#ifndef _WIN32 + .block = opts.block, #endif + }; + drainFD(fd, sink, sinkOpts); return std::move(sink.s); } +void copyFdRange(Descriptor fd, off_t offset, size_t nbytes, Sink & sink) +{ + auto left = nbytes; + std::array buf; + + while (left) { + auto limit = std::min(left, buf.size()); + auto n = readOffset(fd, offset, std::span(buf.data(), limit)); + if (n == 0) + throw EndOfFile("unexpected end-of-file"); + assert(n <= left); + sink(std::string_view(reinterpret_cast(buf.data()), n)); + offset += n; + left -= n; + } +} + ////////////////////////////////////////////////////////////////////// AutoCloseFD::AutoCloseFD() @@ -90,24 +274,6 @@ void AutoCloseFD::close() } } -void AutoCloseFD::fsync() const -{ - if (fd != INVALID_DESCRIPTOR) { - int result; - result = -#ifdef _WIN32 - ::FlushFileBuffers(fd) -#elif defined(__APPLE__) - ::fcntl(fd, F_FULLFSYNC) -#else - ::fsync(fd) -#endif - ; - if (result == -1) - throw NativeSysError("fsync file descriptor %1%", fd); - } -} - void AutoCloseFD::startFsync() const { #ifdef __linux__ diff --git a/src/libutil/file-system.cc b/src/libutil/file-system.cc index ead940760a38..e1efca0ddf7c 100644 --- a/src/libutil/file-system.cc +++ b/src/libutil/file-system.cc @@ -22,6 +22,7 @@ #include #include +#include #ifdef __FreeBSD__ # include @@ -40,9 +41,9 @@ DirectoryIterator::DirectoryIterator(const std::filesystem::path & p) // **Attempt to create the underlying directory_iterator** it_ = std::filesystem::directory_iterator(p); } catch (const std::filesystem::filesystem_error & e) { - // **Catch filesystem_error and throw SysError** - // Adapt the error message as needed for SysError - throw SysError("cannot read directory %s", p); + // **Catch filesystem_error and throw SystemError** + // Adapt the error message as needed for SystemError + throw SystemError(e.code(), "cannot read directory %s", PathFmt(p)); } } @@ -55,7 +56,7 @@ DirectoryIterator & DirectoryIterator::operator++() // Try to get path info if possible, might fail if iterator is bad try { if (it_ != std::filesystem::directory_iterator{}) { - throw SysError("cannot read directory past %s: %s", it_->path(), ec.message()); + throw SysError("cannot read directory past %s: %s", PathFmt(it_->path()), ec.message()); } } catch (...) { throw SysError("cannot read directory"); @@ -64,21 +65,14 @@ DirectoryIterator & DirectoryIterator::operator++() return *this; } -bool isAbsolute(PathView path) -{ - return std::filesystem::path{path}.is_absolute(); -} - -Path absPath(PathView path, std::optional dir, bool resolveSymlinks) +std::filesystem::path +absPath(const std::filesystem::path & path0, const std::filesystem::path * dir, bool resolveSymlinks) { - std::string scratch; + std::filesystem::path path = path0; - if (!isAbsolute(path)) { + if (!path.is_absolute()) { // In this case we need to call `canonPath` on a newly-created - // string. We set `scratch` to that string first, and then set - // `path` to `scratch`. This ensures the newly-created string - // lives long enough for the call to `canonPath`, and allows us - // to just accept a `std::string_view`. + // string. if (!dir) { #ifdef __GNU__ /* GNU (aka. GNU/Hurd) doesn't have any limitation on path @@ -90,33 +84,22 @@ Path absPath(PathView path, std::optional dir, bool resolveSymlinks) if (!getcwd(buf, sizeof(buf))) #endif throw SysError("cannot get cwd"); - scratch = concatStrings(buf, "/", path); + path = std::filesystem::path{buf} / path; #ifdef __GNU__ free(buf); #endif } else - scratch = concatStrings(*dir, "/", path); - path = scratch; + path = *dir / path; } return canonPath(path, resolveSymlinks); } -std::filesystem::path -absPath(const std::filesystem::path & path, const std::filesystem::path * dir_, bool resolveSymlinks) -{ - std::optional dir = dir_ ? std::optional{dir_->string()} : std::nullopt; - return absPath(PathView{path.string()}, dir.transform([](auto & p) { return PathView(p); }), resolveSymlinks); -} - -Path canonPath(PathView path, bool resolveSymlinks) +std::filesystem::path canonPath(const std::filesystem::path & path, bool resolveSymlinks) { - assert(path != ""); + assert(!path.empty()); - if (!isAbsolute(path)) - throw Error("not an absolute path: '%1%'", path); - - // For Windows - auto rootName = std::filesystem::path{path}.root_name(); + if (!path.is_absolute()) + throw Error("not an absolute path: %s", PathFmt(path)); /* This just exists because we cannot set the target of `remaining` (the callback parameter) directly to a newly-constructed string, @@ -128,16 +111,17 @@ Path canonPath(PathView path, bool resolveSymlinks) unsigned int followCount = 0, maxFollow = 1024; auto ret = canonPathInner>( - path, [&followCount, &temp, maxFollow, resolveSymlinks](std::string & result, std::string_view & remaining) { + path.string(), + [&followCount, &temp, maxFollow, resolveSymlinks](std::string & result, std::string_view & remaining) { if (resolveSymlinks && std::filesystem::is_symlink(result)) { if (++followCount >= maxFollow) throw Error("infinite symlink recursion in path '%1%'", remaining); - remaining = (temp = concatStrings(readLink(result), remaining)); - if (isAbsolute(remaining)) { + remaining = (temp = concatStrings(readLink(result).string(), remaining)); + if (std::filesystem::path(remaining).is_absolute()) { /* restart for symlinks pointing to absolute path */ result.clear(); } else { - result = dirOf(result); + result = std::filesystem::path(result).parent_path().string(); if (result == "/") { /* we don’t want trailing slashes here, which `dirOf` only produces if `result = /` */ @@ -147,19 +131,9 @@ Path canonPath(PathView path, bool resolveSymlinks) } }); - if (!rootName.empty()) - ret = rootName.string() + std::move(ret); return ret; } -Path dirOf(const PathView path) -{ - Path::size_type pos = OsPathTrait::rfindPathSep(path); - if (pos == path.npos) - return "."; - return std::filesystem::path{path}.parent_path().string(); -} - std::string_view baseNameOf(std::string_view path) { if (path.empty()) @@ -183,9 +157,11 @@ bool isInDir(const std::filesystem::path & path, const std::filesystem::path & d /* Note that while the standard doesn't guarantee this, the `lexically_*` functions should do no IO and not throw. */ auto rel = path.lexically_relative(dir); - /* Method from - https://stackoverflow.com/questions/62503197/check-if-path-contains-another-in-c++ */ - return !rel.empty() && rel.native()[0] != OS_STR('.'); + if (rel.empty()) + return false; + + auto first = *rel.begin(); + return first != "." && first != ".."; } bool isDirOrInDir(const std::filesystem::path & path, const std::filesystem::path & dir) @@ -193,52 +169,83 @@ bool isDirOrInDir(const std::filesystem::path & path, const std::filesystem::pat return path == dir || isInDir(path, dir); } -struct stat stat(const Path & path) +#ifdef _WIN32 +# define STAT _wstat64 +# define LSTAT _wstat64 +#else +# define STAT stat +# define LSTAT lstat +#endif + +PosixStat stat(const std::filesystem::path & path) +{ + PosixStat st; + if (STAT(path.c_str(), &st)) + throw SysError("getting status of %s", PathFmt(path)); + return st; +} + +PosixStat lstat(const std::filesystem::path & path) { - struct stat st; - if (stat(path.c_str(), &st)) - throw SysError("getting status of '%1%'", path); + PosixStat st; + if (LSTAT(path.c_str(), &st)) + throw SysError("getting status of %s", PathFmt(path)); return st; } +PosixStat fstat(int fd) +{ + PosixStat st; + if ( #ifdef _WIN32 -# define STAT stat + _fstat64 #else -# define STAT lstat + ::fstat #endif + (fd, &st)) + throw SysError("getting status of fd %d", fd); + return st; +} -struct stat lstat(const Path & path) +std::optional maybeStat(const std::filesystem::path & path) { - struct stat st; - if (STAT(path.c_str(), &st)) - throw SysError("getting status of '%1%'", path); + std::optional st{std::in_place}; + if (STAT(path.c_str(), &*st)) { + if (errno == ENOENT || errno == ENOTDIR) + st.reset(); + else + throw SysError("getting status of %s", PathFmt(path)); + } return st; } -std::optional maybeLstat(const Path & path) +std::optional maybeLstat(const std::filesystem::path & path) { - std::optional st{std::in_place}; - if (STAT(path.c_str(), &*st)) { + std::optional st{std::in_place}; + if (LSTAT(path.c_str(), &*st)) { if (errno == ENOENT || errno == ENOTDIR) st.reset(); else - throw SysError("getting status of '%s'", path); + throw SysError("getting status of %s", PathFmt(path)); } return st; } +#undef STAT +#undef LSTAT + bool pathExists(const std::filesystem::path & path) { - return maybeLstat(path.string()).has_value(); + return maybeLstat(path).has_value(); } bool pathAccessible(const std::filesystem::path & path) { try { - return pathExists(path.string()); - } catch (SysError & e) { + return pathExists(path); + } catch (SystemError & e) { // swallow EPERM - if (e.errNo == EPERM) + if (e.is(std::errc::operation_not_permitted)) return false; throw; } @@ -247,107 +254,88 @@ bool pathAccessible(const std::filesystem::path & path) std::filesystem::path readLink(const std::filesystem::path & path) { checkInterrupt(); - return std::filesystem::read_symlink(path); -} - -Path readLink(const Path & path) -{ - return readLink(std::filesystem::path{path}).string(); + try { + return std::filesystem::read_symlink(path); + } catch (std::filesystem::filesystem_error & e) { + throw SystemError(e.code(), "reading symbolic link %s", PathFmt(path)); + } } -std::string readFile(const Path & path) +std::string readFile(const std::filesystem::path & path) { - AutoCloseFD fd = toDescriptor(open( - path.c_str(), - O_RDONLY -#ifdef O_CLOEXEC - | O_CLOEXEC -#endif - )); + auto fd = openFileReadonly(path); if (!fd) - throw SysError("opening file '%1%'", path); + throw NativeSysError("opening file %1%", PathFmt(path)); return readFile(fd.get()); } -std::string readFile(const std::filesystem::path & path) -{ - return readFile(os_string_to_string(PathViewNG{path})); -} - -void readFile(const Path & path, Sink & sink, bool memory_map) +void readFile(const std::filesystem::path & path, Sink & sink, bool memory_map) { // Memory-map the file for faster processing where possible. if (memory_map) { try { - boost::iostreams::mapped_file_source mmap(path); + /* mapped_file_source can't be constructed from a std::filesystem::path. */ + boost::iostreams::mapped_file_source mmap(boost::filesystem::path(path.native())); if (mmap.is_open()) { sink({mmap.data(), mmap.size()}); return; } } catch (const boost::exception & e) { } - debug("memory-mapping failed for path: %s", path); + debug("memory-mapping failed for path: %s", PathFmt(path)); } // Stream the file instead if memory-mapping fails or is disabled. - AutoCloseFD fd = toDescriptor(open( - path.c_str(), - O_RDONLY -#ifdef O_CLOEXEC - | O_CLOEXEC -#endif - )); + auto fd = openFileReadonly(std::filesystem::path(path)); if (!fd) - throw SysError("opening file '%s'", path); + throw NativeSysError("opening file %s", PathFmt(path)); drainFD(fd.get(), sink); } -void writeFile(const Path & path, std::string_view s, mode_t mode, FsSync sync) +void writeFile(const std::filesystem::path & path, std::string_view s, mode_t mode, FsSync sync) { - AutoCloseFD fd = toDescriptor(open( - path.c_str(), - O_WRONLY | O_TRUNC | O_CREAT -#ifdef O_CLOEXEC - | O_CLOEXEC -#endif - , - mode)); + AutoCloseFD fd = openNewFileForWrite( + path, + mode, + { + .truncateExisting = true, + .followSymlinksOnTruncate = true, /* FIXME: Do we want this? */ + }); if (!fd) - throw SysError("opening file '%1%'", path); + throw NativeSysError("opening file %s", PathFmt(path)); - writeFile(fd, path, s, mode, sync); + writeFile(fd.get(), s, sync, &path); /* Close explicitly to propagate the exceptions. */ fd.close(); } -void writeFile(AutoCloseFD & fd, const Path & origPath, std::string_view s, mode_t mode, FsSync sync) +void writeFile(Descriptor fd, std::string_view s, FsSync sync, const std::filesystem::path * origPath) { - assert(fd); + assert(fd != INVALID_DESCRIPTOR); try { - writeFull(fd.get(), s); + writeFull(fd, s); if (sync == FsSync::Yes) - fd.fsync(); + syncDescriptor(fd); } catch (Error & e) { - e.addTrace({}, "writing file '%1%'", origPath); + e.addTrace({}, "writing file %1%", origPath ? PathFmt(*origPath) : PathFmt(descriptorToPath(fd))); throw; } } -void writeFile(const Path & path, Source & source, mode_t mode, FsSync sync) +void writeFile(const std::filesystem::path & path, Source & source, mode_t mode, FsSync sync) { - AutoCloseFD fd = toDescriptor(open( - path.c_str(), - O_WRONLY | O_TRUNC | O_CREAT -#ifdef O_CLOEXEC - | O_CLOEXEC -#endif - , - mode)); + AutoCloseFD fd = openNewFileForWrite( + path, + mode, + { + .truncateExisting = true, + .followSymlinksOnTruncate = true, /* FIXME: Do we want this? */ + }); if (!fd) - throw SysError("opening file '%1%'", path); + throw NativeSysError("opening file %s", PathFmt(path)); std::array buf; @@ -361,7 +349,7 @@ void writeFile(const Path & path, Source & source, mode_t mode, FsSync sync) } } } catch (Error & e) { - e.addTrace({}, "writing file '%1%'", path); + e.addTrace({}, "writing file %s", PathFmt(path)); throw; } if (sync == FsSync::Yes) @@ -372,30 +360,25 @@ void writeFile(const Path & path, Source & source, mode_t mode, FsSync sync) syncParent(path); } -void syncParent(const Path & path) +void syncParent(const std::filesystem::path & path) { - AutoCloseFD fd = toDescriptor(open(dirOf(path).c_str(), O_RDONLY, 0)); + assert(path.has_parent_path()); + AutoCloseFD fd = openDirectory(path.parent_path()); if (!fd) - throw SysError("opening file '%1%'", path); + throw NativeSysError("opening file %s", PathFmt(path)); + /* TODO: Fix on windows, FlushFileBuffers requires GENERIC_WRITE. */ fd.fsync(); } -#ifdef __FreeBSD__ -# define MOUNTEDPATHS_PARAM , std::set & mountedPaths -# define MOUNTEDPATHS_ARG , mountedPaths -#else -# define MOUNTEDPATHS_PARAM -# define MOUNTEDPATHS_ARG -#endif - -void recursiveSync(const Path & path) +void recursiveSync(const std::filesystem::path & path) { + /* TODO: Fix on windows, FlushFileBuffers requires GENERIC_WRITE. */ /* If it's a file or symlink, just fsync and return. */ auto st = lstat(path); if (S_ISREG(st.st_mode)) { - AutoCloseFD fd = toDescriptor(open(path.c_str(), O_RDONLY, 0)); + AutoCloseFD fd = openFileReadonly(path); /* TODO: O_NOFOLLOW? */ if (!fd) - throw SysError("opening file '%1%'", path); + throw NativeSysError("opening file %s", PathFmt(path)); fd.fsync(); return; } else if (S_ISLNK(st.st_mode)) @@ -414,9 +397,9 @@ void recursiveSync(const Path & path) if (std::filesystem::is_directory(st)) { dirsToEnumerate.emplace_back(entry.path()); } else if (std::filesystem::is_regular_file(st)) { - AutoCloseFD fd = toDescriptor(open(entry.path().string().c_str(), O_RDONLY, 0)); + AutoCloseFD fd = openFileReadonly(entry.path()); /* TODO: O_NOFOLLOW? */ if (!fd) - throw SysError("opening file '%1%'", entry.path()); + throw NativeSysError("opening file %1%", PathFmt(entry.path())); fd.fsync(); } } @@ -425,147 +408,24 @@ void recursiveSync(const Path & path) /* Fsync all the directories. */ for (auto dir = dirsToFsync.rbegin(); dir != dirsToFsync.rend(); ++dir) { - AutoCloseFD fd = toDescriptor(open(dir->string().c_str(), O_RDONLY, 0)); + AutoCloseFD fd = openDirectory(*dir); /* TODO: O_NOFOLLOW? */ if (!fd) - throw SysError("opening directory '%1%'", *dir); + throw NativeSysError("opening directory %1%", PathFmt(*dir)); fd.fsync(); } } -static void _deletePath( - Descriptor parentfd, - const std::filesystem::path & path, - uint64_t & bytesFreed, - std::exception_ptr & ex MOUNTEDPATHS_PARAM) -{ -#ifndef _WIN32 - checkInterrupt(); - -# ifdef __FreeBSD__ - // In case of emergency (unmount fails for some reason) not recurse into mountpoints. - // This prevents us from tearing up the nullfs-mounted nix store. - if (mountedPaths.find(path) != mountedPaths.end()) { - return; - } -# endif - - std::string name(path.filename()); - assert(name != "." && name != ".." && !name.empty()); - - struct stat st; - if (fstatat(parentfd, name.c_str(), &st, AT_SYMLINK_NOFOLLOW) == -1) { - if (errno == ENOENT) - return; - throw SysError("getting status of %1%", path); - } - - if (!S_ISDIR(st.st_mode)) { - /* We are about to delete a file. Will it likely free space? */ - - switch (st.st_nlink) { - /* Yes: last link. */ - case 1: - bytesFreed += st.st_size; - break; - /* Maybe: yes, if 'auto-optimise-store' or manual optimisation - was performed. Instead of checking for real let's assume - it's an optimised file and space will be freed. - - In worst case we will double count on freed space for files - with exactly two hardlinks for unoptimised packages. - */ - case 2: - bytesFreed += st.st_size; - break; - /* No: 3+ links. */ - default: - break; - } - } - - if (S_ISDIR(st.st_mode)) { - /* Make the directory accessible. */ - const auto PERM_MASK = S_IRUSR | S_IWUSR | S_IXUSR; - if ((st.st_mode & PERM_MASK) != PERM_MASK) { - if (fchmodat(parentfd, name.c_str(), st.st_mode | PERM_MASK, 0) == -1) - throw SysError("chmod %1%", path); - } - - int fd = openat(parentfd, name.c_str(), O_RDONLY | O_DIRECTORY | O_NOFOLLOW); - if (fd == -1) - throw SysError("opening directory %1%", path); - AutoCloseDir dir(fdopendir(fd)); - if (!dir) - throw SysError("opening directory %1%", path); - - struct dirent * dirent; - while (errno = 0, dirent = readdir(dir.get())) { /* sic */ - checkInterrupt(); - std::string childName = dirent->d_name; - if (childName == "." || childName == "..") - continue; - _deletePath(dirfd(dir.get()), path / childName, bytesFreed, ex MOUNTEDPATHS_ARG); - } - if (errno) - throw SysError("reading directory %1%", path); - } - - int flags = S_ISDIR(st.st_mode) ? AT_REMOVEDIR : 0; - if (unlinkat(parentfd, name.c_str(), flags) == -1) { - if (errno == ENOENT) - return; - try { - throw SysError("cannot unlink %1%", path); - } catch (...) { - if (!ex) - ex = std::current_exception(); - else - ignoreExceptionExceptInterrupt(); - } - } -#else - // TODO implement - throw UnimplementedError("_deletePath"); -#endif -} - -static void _deletePath(const std::filesystem::path & path, uint64_t & bytesFreed MOUNTEDPATHS_PARAM) -{ - assert(path.is_absolute()); - assert(path.parent_path() != path); - - AutoCloseFD dirfd = toDescriptor(open(path.parent_path().string().c_str(), O_RDONLY)); - if (!dirfd) { - if (errno == ENOENT) - return; - throw SysError("opening directory %s", path.parent_path()); - } - - std::exception_ptr ex; - - _deletePath(dirfd.get(), path, bytesFreed, ex MOUNTEDPATHS_ARG); - - if (ex) - std::rethrow_exception(ex); -} - -void deletePath(const std::filesystem::path & path) -{ - uint64_t dummy; - deletePath(path, dummy); -} - -void createDir(const Path & path, mode_t mode) +void createDir(const std::filesystem::path & path, mode_t mode) { if (mkdir( - path.c_str() + path.string().c_str() #ifndef _WIN32 , mode #endif ) == -1) - throw SysError("creating directory '%1%'", path); + throw SysError("creating directory %s", PathFmt(path)); } void createDirs(const std::filesystem::path & path) @@ -573,70 +433,50 @@ void createDirs(const std::filesystem::path & path) try { std::filesystem::create_directories(path); } catch (std::filesystem::filesystem_error & e) { - throw SysError("creating directory '%1%'", path.string()); - } -} - -void deletePath(const std::filesystem::path & path, uint64_t & bytesFreed) -{ - // Activity act(*logger, lvlDebug, "recursively deleting path '%1%'", path); -#ifdef __FreeBSD__ - std::set mountedPaths; - struct statfs * mntbuf; - int count; - if ((count = getmntinfo(&mntbuf, MNT_WAIT)) < 0) { - throw SysError("getmntinfo"); + throw SystemError(e.code(), "creating directory %1%", PathFmt(path)); } - - for (int i = 0; i < count; i++) { - mountedPaths.emplace(mntbuf[i].f_mntonname); - } -#endif - bytesFreed = 0; - _deletePath(path, bytesFreed MOUNTEDPATHS_ARG); } ////////////////////////////////////////////////////////////////////// AutoDelete::AutoDelete() : del{false} + , recursive(false) { } AutoDelete::AutoDelete(const std::filesystem::path & p, bool recursive) : _path(p) + , del(true) + , recursive(recursive) { - del = true; - this->recursive = recursive; +} + +void AutoDelete::deletePath() +{ + if (del) { + if (recursive) + nix::deletePath(_path); + else + std::filesystem::remove(_path); + cancel(); + } } AutoDelete::~AutoDelete() { try { - if (del) { - if (recursive) - deletePath(_path); - else { - std::filesystem::remove(_path); - } - } + deletePath(); } catch (...) { ignoreExceptionInDestructor(); } } -void AutoDelete::cancel() +void AutoDelete::cancel() noexcept { del = false; } -void AutoDelete::reset(const std::filesystem::path & p, bool recursive) -{ - _path = p; - this->recursive = recursive; - del = true; -} - ////////////////////////////////////////////////////////////////////// #ifdef __FreeBSD__ @@ -645,7 +485,7 @@ AutoUnmount::AutoUnmount() { } -AutoUnmount::AutoUnmount(Path & p) +AutoUnmount::AutoUnmount(const std::filesystem::path & p) : path(p) , del(true) { @@ -654,28 +494,29 @@ AutoUnmount::AutoUnmount(Path & p) AutoUnmount::~AutoUnmount() { try { - if (del) { - if (unmount(path.c_str(), 0) < 0) { - throw SysError("Failed to unmount path %1%", path); - } - } + unmount(); } catch (...) { ignoreExceptionInDestructor(); } } -void AutoUnmount::cancel() +void AutoUnmount::cancel() noexcept { del = false; } -#endif - -////////////////////////////////////////////////////////////////////// -std::filesystem::path defaultTempDir() +void AutoUnmount::unmount() { - return getEnvNonEmpty("TMPDIR").value_or("/tmp"); + if (del) { + if (::unmount(path.c_str(), 0) < 0) { + throw SysError("Failed to unmount path %1%", PathFmt(path)); + } + } + cancel(); } +#endif + +////////////////////////////////////////////////////////////////////// std::filesystem::path createTempDir(const std::filesystem::path & tmpRoot, const std::string & prefix, mode_t mode) { @@ -700,19 +541,33 @@ std::filesystem::path createTempDir(const std::filesystem::path & tmpRoot, const "wheel", then "tar" will fail to unpack archives that have the setgid bit set on directories. */ if (chown(tmpDir.c_str(), (uid_t) -1, getegid()) != 0) - throw SysError("setting group of directory '%1%'", tmpDir); + throw SysError("setting group of directory %1%", PathFmt(tmpDir)); #endif return tmpDir; } if (errno != EEXIST) - throw SysError("creating directory '%1%'", tmpDir); + throw SysError("creating directory %1%", PathFmt(tmpDir)); } } AutoCloseFD createAnonymousTempFile() { AutoCloseFD fd; -#ifdef O_TMPFILE + +#ifdef _WIN32 + auto path = makeTempPath(defaultTempDir(), "nix-anonymous"); + fd = CreateFileW( + path.c_str(), + GENERIC_READ | GENERIC_WRITE, + /*dwShareMode=*/0, + /*lpSecurityAttributes=*/nullptr, + CREATE_NEW, + FILE_ATTRIBUTE_TEMPORARY | FILE_FLAG_DELETE_ON_CLOSE, + /*hTemplateFile=*/nullptr); + if (!fd) + throw windows::WinError("creating temporary file %1%", PathFmt(path)); +#else +# ifdef O_TMPFILE static std::atomic_flag tmpfileUnsupported{}; if (!tmpfileUnsupported.test()) /* Try with O_TMPFILE first. */ { /* Use O_EXCL, because the file is never supposed to be linked into filesystem. */ @@ -727,46 +582,48 @@ AutoCloseFD createAnonymousTempFile() return fd; /* Successfully created. */ } } -#endif +# endif auto [fd2, path] = createTempFile("nix-anonymous"); if (!fd2) - throw SysError("creating temporary file '%s'", path); + throw SysError("creating temporary file %s", PathFmt(path)); fd = std::move(fd2); -#ifndef _WIN32 - unlink(requireCString(path)); /* We only care about the file descriptor. */ + tryUnlink(path); /* We only care about the file descriptor. */ #endif + return fd; } -std::pair createTempFile(const Path & prefix) +std::pair createTempFile(const std::filesystem::path & prefix) { - Path tmpl(defaultTempDir().string() + "/" + prefix + ".XXXXXX"); - // Strictly speaking, this is UB, but who cares... + assert(!prefix.is_absolute()); + auto tmpl = (defaultTempDir() / (prefix.string() + ".XXXXXX")).string(); // FIXME: use O_TMPFILE. - // FIXME: Windows should use FILE_ATTRIBUTE_TEMPORARY | FILE_FLAG_DELETE_ON_CLOSE - AutoCloseFD fd = toDescriptor(mkstemp((char *) tmpl.c_str())); + // `mkstemp` modifies the string to contain the actual filename. + AutoCloseFD fd = toDescriptor(mkstemp(tmpl.data())); + if (!fd) throw SysError("creating temporary file '%s'", tmpl); #ifndef _WIN32 unix::closeOnExec(fd.get()); #endif - return {std::move(fd), tmpl}; + return {std::move(fd), std::filesystem::path(std::move(tmpl))}; } std::filesystem::path makeTempPath(const std::filesystem::path & root, const std::string & suffix) { // start the counter at a random value to minimize issues with preexisting temp paths static std::atomic counter(std::random_device{}()); - auto tmpRoot = canonPath(root.empty() ? defaultTempDir().string() : root.string(), true); - return fmt("%1%/%2%-%3%-%4%", tmpRoot, suffix, getpid(), counter.fetch_add(1, std::memory_order_relaxed)); + assert(!std::filesystem::path(suffix).is_absolute()); + auto tmpRoot = canonPath(root.empty() ? defaultTempDir() : root, true); + return tmpRoot / fmt("%s-%s-%s", suffix, getpid(), counter.fetch_add(1, std::memory_order_relaxed)); } -void createSymlink(const Path & target, const Path & link) +void createSymlink(const std::filesystem::path & target, const std::filesystem::path & link) { std::error_code ec; std::filesystem::create_symlink(target, link, ec); if (ec) - throw SysError(ec.value(), "creating symlink '%1%' -> '%2%'", link, target); + throw SysError(ec.value(), "creating symlink %s -> %s", PathFmt(link), PathFmt(target)); } void replaceSymlink(const std::filesystem::path & target, const std::filesystem::path & link) @@ -780,7 +637,7 @@ void replaceSymlink(const std::filesystem::path & target, const std::filesystem: } catch (std::filesystem::filesystem_error & e) { if (e.code() == std::errc::file_exists) continue; - throw SysError("creating symlink %1% -> %2%", tmp, target); + throw SystemError(e.code(), "creating symlink %1% -> %2%", PathFmt(tmp), PathFmt(target)); } try { @@ -788,14 +645,14 @@ void replaceSymlink(const std::filesystem::path & target, const std::filesystem: } catch (std::filesystem::filesystem_error & e) { if (e.code() == std::errc::file_exists) continue; - throw SysError("renaming %1% to %2%", tmp, link); + throw SystemError(e.code(), "renaming %1% to %2%", PathFmt(tmp), PathFmt(link)); } break; } } -void setWriteTime(const std::filesystem::path & path, const struct stat & st) +void setWriteTime(const std::filesystem::path & path, const PosixStat & st) { setWriteTime(path, st.st_atime, st.st_mtime, S_ISLNK(st.st_mode)); } @@ -821,10 +678,10 @@ void copyFile(const std::filesystem::path & from, const std::filesystem::path & copyFile(entry, to / entry.path().filename(), andDelete); } } else { - throw Error("file %s has an unsupported type", from); + throw Error("file %s has an unsupported type", PathFmt(from)); } - setWriteTime(to, lstat(from.string().c_str())); + setWriteTime(to, lstat(from)); if (andDelete) { if (!std::filesystem::is_symlink(fromStatus)) std::filesystem::permissions( @@ -835,25 +692,24 @@ void copyFile(const std::filesystem::path & from, const std::filesystem::path & } } -void moveFile(const Path & oldName, const Path & newName) +void moveFile(const std::filesystem::path & oldName, const std::filesystem::path & newName) { try { std::filesystem::rename(oldName, newName); } catch (std::filesystem::filesystem_error & e) { - auto oldPath = std::filesystem::path(oldName); - auto newPath = std::filesystem::path(newName); + auto oldPath = oldName; + auto newPath = newName; // For the move to be as atomic as possible, copy to a temporary // directory - std::filesystem::path temp = - createTempDir(os_string_to_string(PathViewNG{newPath.parent_path()}), "rename-tmp"); + std::filesystem::path temp = createTempDir(os_string_to_string(PathView{newPath.parent_path()}), "rename-tmp"); Finally removeTemp = [&]() { std::filesystem::remove(temp); }; auto tempCopyTarget = temp / "copy-target"; if (e.code().value() == EXDEV) { std::filesystem::remove(newPath); - warn("can’t rename %s as %s, copying instead", oldName, newName); + warn("can’t rename %s as %s, copying instead", PathFmt(oldName), PathFmt(newName)); copyFile(oldPath, tempCopyTarget, true); std::filesystem::rename( - os_string_to_string(PathViewNG{tempCopyTarget}), os_string_to_string(PathViewNG{newPath})); + os_string_to_string(PathView{tempCopyTarget}), os_string_to_string(PathView{newPath})); } } } @@ -880,7 +736,6 @@ bool isExecutableFileAmbient(const std::filesystem::path & exe) std::filesystem::path makeParentCanonical(const std::filesystem::path & rawPath) { std::filesystem::path path(absPath(rawPath)); - ; try { auto parent = path.parent_path(); if (parent == path) { @@ -889,21 +744,50 @@ std::filesystem::path makeParentCanonical(const std::filesystem::path & rawPath) } return std::filesystem::canonical(parent) / path.filename(); } catch (std::filesystem::filesystem_error & e) { - throw SysError("canonicalising parent path of '%1%'", path); + throw SystemError(e.code(), "canonicalising parent path of %1%", PathFmt(path)); } } +void chmod(const std::filesystem::path & path, mode_t mode) +{ + if ( +#ifdef _WIN32 + ::_wchmod +#else + ::chmod +#endif + (path.c_str(), mode) + == -1) + throw SysError("setting permissions on %s", PathFmt(path)); +} + +#ifdef _WIN32 +# define UNLINK_PROC ::_wunlink +#else +# define UNLINK_PROC ::unlink +#endif + +void unlink(const std::filesystem::path & path) +{ + if (UNLINK_PROC(path.c_str()) == -1) + throw SysError("removing %s", PathFmt(path)); +} + +void tryUnlink(const std::filesystem::path & path) +{ + UNLINK_PROC(path.c_str()); +} + +#undef UNLINK_PROC + bool chmodIfNeeded(const std::filesystem::path & path, mode_t mode, mode_t mask) { - auto pathString = path.string(); - auto prevMode = lstat(pathString).st_mode; + auto prevMode = lstat(path).st_mode; if (((prevMode ^ mode) & mask) == 0) return false; - if (chmod(pathString.c_str(), mode) != 0) - throw SysError("could not set permissions on '%s' to %o", pathString, mode); - + chmod(path, mode); return true; } diff --git a/src/libutil/freebsd/freebsd-jail.cc b/src/libutil/freebsd/freebsd-jail.cc index 90fbe0cd62e5..1961c9a13f50 100644 --- a/src/libutil/freebsd/freebsd-jail.cc +++ b/src/libutil/freebsd/freebsd-jail.cc @@ -11,39 +11,33 @@ namespace nix { -AutoRemoveJail::AutoRemoveJail() - : del{false} +AutoRemoveJail::AutoRemoveJail(int jid) + : jid(jid) { } -AutoRemoveJail::AutoRemoveJail(int jid) - : jid(jid) - , del(true) +void AutoRemoveJail::remove() { + if (jid != INVALID_JAIL) { + if (jail_remove(jid) < 0) { + throw SysError("Failed to remove jail %1%", jid); + } + } + cancel(); } AutoRemoveJail::~AutoRemoveJail() { try { - if (del) { - if (jail_remove(jid) < 0) { - throw SysError("Failed to remove jail %1%", jid); - } - } + remove(); } catch (...) { ignoreExceptionInDestructor(); } } -void AutoRemoveJail::cancel() -{ - del = false; -} - -void AutoRemoveJail::reset(int j) +void AutoRemoveJail::cancel() noexcept { - del = true; - jid = j; + jid = INVALID_JAIL; } ////////////////////////////////////////////////////////////////////// diff --git a/src/libutil/freebsd/include/nix/util/freebsd-jail.hh b/src/libutil/freebsd/include/nix/util/freebsd-jail.hh index 33a86a3986ed..b4b1cc75a067 100644 --- a/src/libutil/freebsd/include/nix/util/freebsd-jail.hh +++ b/src/libutil/freebsd/include/nix/util/freebsd-jail.hh @@ -7,14 +7,51 @@ namespace nix { class AutoRemoveJail { - int jid; - bool del; + static constexpr int INVALID_JAIL = -1; + int jid = INVALID_JAIL; public: + AutoRemoveJail() = default; AutoRemoveJail(int jid); - AutoRemoveJail(); + AutoRemoveJail(const AutoRemoveJail &) = delete; + AutoRemoveJail & operator=(const AutoRemoveJail &) = delete; + + AutoRemoveJail(AutoRemoveJail && other) noexcept + : jid(other.jid) + { + other.cancel(); + } + + AutoRemoveJail & operator=(AutoRemoveJail && other) noexcept + { + swap(*this, other); + return *this; + } + + friend void swap(AutoRemoveJail & lhs, AutoRemoveJail & rhs) noexcept + { + using std::swap; + swap(lhs.jid, rhs.jid); + } + + operator int() const + { + return jid; + } + ~AutoRemoveJail(); - void cancel(); - void reset(int j); + + /** + * Remove the jail and cancel this `AutoRemoveJail`, so jail removal is not + * attempted a second time by the destructor. + * + * The destructor calls this ignoring any exception. + */ + void remove(); + + /** + * Cancel the jail removal. + */ + void cancel() noexcept; }; } // namespace nix diff --git a/src/libutil/fs-sink.cc b/src/libutil/fs-sink.cc index 521a10c9a777..6aebb3f5ea94 100644 --- a/src/libutil/fs-sink.cc +++ b/src/libutil/fs-sink.cc @@ -2,12 +2,12 @@ #include "nix/util/error.hh" #include "nix/util/config-global.hh" +#include "nix/util/file-system-at.hh" #include "nix/util/fs-sink.hh" #ifdef _WIN32 # include # include "nix/util/file-path.hh" -# include "nix/util/windows-error.hh" #endif #include "util-config-private.hh" @@ -70,7 +70,6 @@ static std::filesystem::path append(const std::filesystem::path & src, const Can return dst; } -#ifndef _WIN32 void RestoreSink::createDirectory(const CanonPath & path, DirectoryCreatedCallback callback) { if (path.isRoot()) { @@ -84,15 +83,23 @@ void RestoreSink::createDirectory(const CanonPath & path, DirectoryCreatedCallba RestoreSink dirSink{startFsync}; dirSink.dstPath = append(dstPath, path); - dirSink.dirFd = - unix::openFileEnsureBeneathNoSymlinks(dirFd.get(), path, O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC); + dirSink.dirFd = openFileEnsureBeneathNoSymlinks( + dirFd.get(), + path, +#ifdef _WIN32 + FILE_READ_ATTRIBUTES | SYNCHRONIZE, + FILE_DIRECTORY_FILE +#else + O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC, + 0 +#endif + ); if (!dirSink.dirFd) - throw SysError("opening directory '%s'", dirSink.dstPath.string()); + throw SysError("opening directory %s", PathFmt(dirSink.dstPath)); callback(dirSink, CanonPath::root); } -#endif void RestoreSink::createDirectory(const CanonPath & path) { @@ -102,10 +109,10 @@ void RestoreSink::createDirectory(const CanonPath & path) if (dirFd) { if (path.isRoot()) /* Trying to create a directory that we already have a file descriptor for. */ - throw Error("path '%s' already exists", p.string()); + throw Error("path %s already exists", PathFmt(p)); if (::mkdirat(dirFd.get(), path.rel_c_str(), 0777) == -1) - throw SysError("creating directory '%s'", p.string()); + throw SysError("creating directory %s", PathFmt(p)); return; } @@ -122,18 +129,35 @@ void RestoreSink::createDirectory(const CanonPath & path) directory. */ dirFd = open(p.c_str(), O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC); if (!dirFd) - throw SysError("creating directory '%1%'", p.string()); + throw SysError("creating directory %1%", PathFmt(p)); } #endif }; -struct RestoreRegularFile : CreateRegularFileSink +struct RestoreRegularFile : CreateRegularFileSink, FdSink { AutoCloseFD fd; bool startFsync = false; + RestoreRegularFile(bool startFSync_, AutoCloseFD fd_) + : FdSink(fd_.get()) + , fd(std::move(fd_)) + , startFsync(startFSync_) + { + } + ~RestoreRegularFile() { + /* Flush the sink before FdSink destructor has a chance to run and we've + closed the file descriptor. */ + if (fd) { + try { + FdSink::flush(); + } catch (...) { + ignoreExceptionInDestructor(); + } + } + /* Initiate an fsync operation without waiting for the result. The real fsync should be run before registering a store path, but this is a performance optimization to allow @@ -142,18 +166,16 @@ struct RestoreRegularFile : CreateRegularFileSink fd.startFsync(); } - void operator()(std::string_view data) override; void isExecutable() override; void preallocateContents(uint64_t size) override; }; -void RestoreSink::createRegularFile(const CanonPath & path, std::function func) +void RestoreSink::createRegularFile(const CanonPath & path, fun func) { auto p = append(dstPath, path); - RestoreRegularFile crf; - crf.startFsync = startFsync; - crf.fd = + auto crf = RestoreRegularFile( + startFsync, #ifdef _WIN32 CreateFileW( p.c_str(), @@ -166,17 +188,18 @@ void RestoreSink::createRegularFile(const CanonPath & path, std::function '%2%'", p.string(), target); + throw SysError("creating symlink from %1% -> '%2%'", PathFmt(p), target); return; } #endif nix::createSymlink(target, p.string()); } -void RegularFileSink::createRegularFile(const CanonPath & path, std::function func) +void RegularFileSink::createRegularFile(const CanonPath & path, fun func) { struct CRF : CreateRegularFileSink { @@ -250,8 +266,7 @@ void RegularFileSink::createRegularFile(const CanonPath & path, std::function func) +void NullFileSystemObjectSink::createRegularFile(const CanonPath & path, fun func) { struct : CreateRegularFileSink { diff --git a/src/libutil/git.cc b/src/libutil/git.cc index b17fdf145cc9..1b6c65c2e01f 100644 --- a/src/libutil/git.cc +++ b/src/libutil/git.cc @@ -115,7 +115,7 @@ void parseTree( const CanonPath & sinkPath, Source & source, HashAlgorithm hashAlgo, - std::function hook, + fun hook, const ExperimentalFeatureSettings & xpSettings) { const unsigned long long size = std::stoi(getStringUntil(source, 0)); @@ -178,7 +178,7 @@ void parse( Source & source, BlobMode rootModeIfBlob, HashAlgorithm hashAlgo, - std::function hook, + fun hook, const ExperimentalFeatureSettings & xpSettings) { xpSettings.require(Xp::GitHashing); @@ -217,7 +217,7 @@ std::optional convertMode(SourceAccessor::Type type) } } -void restore(FileSystemObjectSink & sink, Source & source, HashAlgorithm hashAlgo, std::function hook) +void restore(FileSystemObjectSink & sink, Source & source, HashAlgorithm hashAlgo, fun hook) { parse(sink, CanonPath::root, source, BlobMode::Regular, hashAlgo, [&](CanonPath name, TreeEntry entry) { auto [accessor, from] = hook(entry.hash); @@ -275,7 +275,7 @@ void dumpTree(const Tree & entries, Sink & sink, const ExperimentalFeatureSettin Mode dump( const SourcePath & path, Sink & sink, - std::function hook, + fun hook, PathFilter & filter, const ExperimentalFeatureSettings & xpSettings) { @@ -325,8 +325,7 @@ Mode dump( TreeEntry dumpHash(HashAlgorithm ha, const SourcePath & path, PathFilter & filter) { - std::function hook; - hook = [&](const SourcePath & path) -> TreeEntry { + fun hook = [&](const SourcePath & path) -> TreeEntry { auto hashSink = HashSink(ha); auto mode = dump(path, hashSink, hook, filter); auto hash = hashSink.finish().hash; diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index ea71ffff5dc0..1f89114c66be 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -369,7 +369,7 @@ Hash hashString(HashAlgorithm ha, std::string_view s, const ExperimentalFeatureS return hash; } -Hash hashFile(HashAlgorithm ha, const Path & path) +Hash hashFile(HashAlgorithm ha, const std::filesystem::path & path) { HashSink sink(ha); readFile(path, sink); diff --git a/src/libutil/include/nix/util/archive.hh b/src/libutil/include/nix/util/archive.hh index b88e1fa2d093..0b7b64e62b74 100644 --- a/src/libutil/include/nix/util/archive.hh +++ b/src/libutil/include/nix/util/archive.hh @@ -55,12 +55,12 @@ namespace nix { * `+` denotes string concatenation. * ``` */ -void dumpPath(const Path & path, Sink & sink, PathFilter & filter = defaultPathFilter); +void dumpPath(const std::filesystem::path & path, Sink & sink, PathFilter & filter = defaultPathFilter); /** * Same as dumpPath(), but returns the last modified date of the path. */ -time_t dumpPathAndGetMtime(const Path & path, Sink & sink, PathFilter & filter = defaultPathFilter); +time_t dumpPathAndGetMtime(const std::filesystem::path & path, Sink & sink, PathFilter & filter = defaultPathFilter); /** * Dump an archive with a single file with these contents. diff --git a/src/libutil/include/nix/util/args.hh b/src/libutil/include/nix/util/args.hh index d793411247b8..b4337590a39f 100644 --- a/src/libutil/include/nix/util/args.hh +++ b/src/libutil/include/nix/util/args.hh @@ -11,6 +11,7 @@ #include "nix/util/types.hh" #include "nix/util/experimental-features.hh" +#include "nix/util/fun.hh" #include "nix/util/ref.hh" namespace nix { @@ -61,6 +62,8 @@ public: */ virtual std::filesystem::path getCommandBaseDir() const; + virtual ~Args() = default; + protected: /** @@ -383,7 +386,7 @@ struct Command : virtual public Args } }; -using Commands = std::map()>>; +using Commands = std::map()>>; /** * An argument parser that supports multiple subcommands, @@ -481,6 +484,8 @@ public: * Add a single completion to the collection */ virtual void add(std::string completion, std::string description = "") = 0; + + virtual ~AddCompletions() = default; }; Strings parseShebangContent(std::string_view s); diff --git a/src/libutil/include/nix/util/callback.hh b/src/libutil/include/nix/util/callback.hh index 2ed48c7a3d06..bec980ec91a3 100644 --- a/src/libutil/include/nix/util/callback.hh +++ b/src/libutil/include/nix/util/callback.hh @@ -5,6 +5,8 @@ #include #include +#include "nix/util/fun.hh" + namespace nix { /** @@ -15,12 +17,12 @@ namespace nix { template class Callback { - std::function)> fun; + nix::fun)> fun; std::atomic_flag done = ATOMIC_FLAG_INIT; public: - Callback(std::function)> fun) + Callback(nix::fun)> fun) : fun(fun) { } diff --git a/src/libutil/include/nix/util/canon-path.hh b/src/libutil/include/nix/util/canon-path.hh index 2156b02fc411..56aa4d038038 100644 --- a/src/libutil/include/nix/util/canon-path.hh +++ b/src/libutil/include/nix/util/canon-path.hh @@ -62,7 +62,7 @@ public: struct unchecked_t {}; - CanonPath(unchecked_t _, std::string path) + constexpr CanonPath(unchecked_t _, std::string path) : path(std::move(path)) { } @@ -74,6 +74,12 @@ public: static const CanonPath root; + /** + * Construct a CanonPath from a single path segment. The segment + * must not contain any slashes and must not be `.` or `..`. + */ + static CanonPath fromFilename(std::string_view segment); + /** * If `raw` starts with a slash, return * `CanonPath(raw)`. Otherwise return a `CanonPath` representing diff --git a/src/libutil/include/nix/util/closure.hh b/src/libutil/include/nix/util/closure.hh index 9e37b4cfb025..b423eb172758 100644 --- a/src/libutil/include/nix/util/closure.hh +++ b/src/libutil/include/nix/util/closure.hh @@ -3,6 +3,7 @@ #include #include +#include "nix/util/fun.hh" #include "nix/util/sync.hh" using std::set; @@ -10,7 +11,7 @@ using std::set; namespace nix { template -using GetEdgesAsync = std::function> &)>)>; +using GetEdgesAsync = fun> &)>)>; template void computeClosure(const set startElts, set & res, GetEdgesAsync getEdgesAsync) diff --git a/src/libutil/include/nix/util/compression-algo.hh b/src/libutil/include/nix/util/compression-algo.hh new file mode 100644 index 000000000000..fe1fc57596c3 --- /dev/null +++ b/src/libutil/include/nix/util/compression-algo.hh @@ -0,0 +1,42 @@ +#pragma once +///@file + +#include "nix/util/error.hh" + +#include + +namespace nix { + +#define NIX_FOR_EACH_COMPRESSION_ALGO(MACRO) \ + MACRO("none", none) \ + MACRO("br", brotli) \ + MACRO("bzip2", bzip2) \ + MACRO("compress", compress) \ + MACRO("grzip", grzip) \ + MACRO("gzip", gzip) \ + MACRO("lrzip", lrzip) \ + MACRO("lz4", lz4) \ + MACRO("lzip", lzip) \ + MACRO("lzma", lzma) \ + MACRO("lzop", lzop) \ + MACRO("xz", xz) \ + MACRO("zstd", zstd) + +#define NIX_DEFINE_COMPRESSION_ALGO(name, value) value, +enum class CompressionAlgo { NIX_FOR_EACH_COMPRESSION_ALGO(NIX_DEFINE_COMPRESSION_ALGO) }; +#undef NIX_DEFINE_COMPRESSION_ALGO + +/** + * Parses a *compression* method into the corresponding enum. This is only used + * in the *compression* case and user interface. Content-Encoding should not use + * these. + * + * @param suggestions Whether to throw an exception with suggestions. + */ +CompressionAlgo parseCompressionAlgo(std::string_view method, bool suggestions = false); + +std::string showCompressionAlgo(CompressionAlgo method); + +MakeError(UnknownCompressionMethod, Error); + +} // namespace nix diff --git a/src/libutil/include/nix/util/compression-settings.hh b/src/libutil/include/nix/util/compression-settings.hh new file mode 100644 index 000000000000..f9d42c298c6d --- /dev/null +++ b/src/libutil/include/nix/util/compression-settings.hh @@ -0,0 +1,21 @@ +#pragma once +///@file + +#include "nix/util/configuration.hh" +#include "nix/util/compression-algo.hh" + +namespace nix { + +template<> +CompressionAlgo BaseSetting::parse(const std::string & str) const; + +template<> +std::string BaseSetting::to_string() const; + +template<> +std::optional BaseSetting>::parse(const std::string & str) const; + +template<> +std::string BaseSetting>::to_string() const; + +} // namespace nix diff --git a/src/libutil/include/nix/util/compression.hh b/src/libutil/include/nix/util/compression.hh index 3518268567bd..db49c8dfd6c1 100644 --- a/src/libutil/include/nix/util/compression.hh +++ b/src/libutil/include/nix/util/compression.hh @@ -4,6 +4,7 @@ #include "nix/util/ref.hh" #include "nix/util/types.hh" #include "nix/util/serialise.hh" +#include "nix/util/compression-algo.hh" #include @@ -20,12 +21,10 @@ std::string decompress(const std::string & method, std::string_view in); std::unique_ptr makeDecompressionSink(const std::string & method, Sink & nextSink); -std::string compress(const std::string & method, std::string_view in, const bool parallel = false, int level = -1); +std::string compress(CompressionAlgo method, std::string_view in, const bool parallel = false, int level = -1); ref -makeCompressionSink(const std::string & method, Sink & nextSink, const bool parallel = false, int level = -1); - -MakeError(UnknownCompressionMethod, Error); +makeCompressionSink(CompressionAlgo method, Sink & nextSink, const bool parallel = false, int level = -1); MakeError(CompressionError, Error); diff --git a/src/libutil/include/nix/util/config-impl.hh b/src/libutil/include/nix/util/config-impl.hh index 8f6f9a358a44..88d82394f312 100644 --- a/src/libutil/include/nix/util/config-impl.hh +++ b/src/libutil/include/nix/util/config-impl.hh @@ -136,7 +136,9 @@ DECLARE_CONFIG_SERIALISER(StringSet) DECLARE_CONFIG_SERIALISER(StringMap) DECLARE_CONFIG_SERIALISER(std::set) DECLARE_CONFIG_SERIALISER(std::filesystem::path) -DECLARE_CONFIG_SERIALISER(std::optional) +DECLARE_CONFIG_SERIALISER(AbsolutePath) +DECLARE_CONFIG_SERIALISER(std::set) +DECLARE_CONFIG_SERIALISER(std::optional) template T BaseSetting::parse(const std::string & str) const diff --git a/src/libutil/include/nix/util/configuration.hh b/src/libutil/include/nix/util/configuration.hh index 6b9f2d6f5d02..b89e17d652f2 100644 --- a/src/libutil/include/nix/util/configuration.hh +++ b/src/libutil/include/nix/util/configuration.hh @@ -2,12 +2,15 @@ ///@file #include +#include #include #include #include +#include "nix/util/json-non-null.hh" #include "nix/util/types.hh" +#include "nix/util/fmt.hh" #include "nix/util/experimental-features.hh" namespace nix { @@ -216,6 +219,30 @@ protected: bool isOverridden() const; }; +/** + * For `Setting`. `parse()` calls `canonPath`, + * rejecting empty and relative paths. + */ +struct AbsolutePath : std::filesystem::path +{ + using path::path; + using path::operator=; + + AbsolutePath(const std::filesystem::path & p) + : path(p) + { + } + + AbsolutePath(std::filesystem::path && p) + : path(std::move(p)) + { + } +}; + +template<> +struct json_avoids_null : std::true_type +{}; + /** * A setting of type T. */ @@ -380,56 +407,35 @@ public: }; /** - * A special setting for Paths. These are automatically canonicalised - * (e.g. "/foo//bar/" becomes "/foo/bar"). - * - * It is mandatory to specify a path; i.e. the empty string is not - * permitted. + * A setting whose value is represented as JSON. The type `T` must be supported by `nlohmann::json`'s `get()`. */ -class PathSetting : public BaseSetting +template +class JSONSetting : public Setting { public: + using Setting::Setting; - PathSetting( - Config * options, - const Path & def, - const std::string & name, - const std::string & description, - const StringSet & aliases = {}); + T parse(const std::string & str) const override; - Path parse(const std::string & str) const override; + std::string to_string() const override; +}; - Path operator+(const char * p) const - { - return value + p; - } +/* Delete these overloads to avoid footguns with implicit quoting of Setting in fmt(). */ - void operator=(const Path & v) - { - this->assign(v); - } -}; +template +inline void formatHelper(F & f, const AbsolutePath & x, const Args &... args) = delete; -/** - * Like `PathSetting`, but the absence of a path is also allowed. - * - * `std::optional` is used instead of the empty string for clarity. - */ -class OptionalPathSetting : public BaseSetting> -{ -public: +template +inline void formatHelper(F & f, const AbsolutePath & x) = delete; - OptionalPathSetting( - Config * options, - const std::optional & def, - const std::string & name, - const std::string & description, - const StringSet & aliases = {}); +template +inline void formatHelper(F & f, const Setting & x, const Args &... args) = delete; - std::optional parse(const std::string & str) const override; +template +inline void formatHelper(F & f, const Setting & x) = delete; - void operator=(const std::optional & v); -}; +template<> +void BaseSetting>::appendOrSet(std::set newValue, bool append); struct ExperimentalFeatureSettings : Config { diff --git a/src/libutil/include/nix/util/current-process.hh b/src/libutil/include/nix/util/current-process.hh index c4a95258174e..657ff0b44e9c 100644 --- a/src/libutil/include/nix/util/current-process.hh +++ b/src/libutil/include/nix/util/current-process.hh @@ -1,6 +1,7 @@ #pragma once ///@file +#include #include #include @@ -42,6 +43,6 @@ void restoreProcessContext(bool restoreMounts = true); /** * @return the path of the current executable. */ -std::optional getSelfExe(); +std::optional getSelfExe(); } // namespace nix diff --git a/src/libutil/include/nix/util/demangle.hh b/src/libutil/include/nix/util/demangle.hh new file mode 100644 index 000000000000..997a535469e6 --- /dev/null +++ b/src/libutil/include/nix/util/demangle.hh @@ -0,0 +1,26 @@ +#pragma once +///@file + +#include +#include +#include + +namespace nix { + +/** + * Demangle a C++ type name. + * Returns the demangled name, or the original if demangling fails. + */ +inline std::string demangle(const char * name) +{ + int status; + char * demangled = abi::__cxa_demangle(name, nullptr, nullptr, &status); + if (demangled) { + std::string result(demangled); + std::free(demangled); + return result; + } + return name; +} + +} // namespace nix diff --git a/src/libutil/include/nix/util/environment-variables.hh b/src/libutil/include/nix/util/environment-variables.hh index f8c3b7ad0289..5fccf53c8413 100644 --- a/src/libutil/include/nix/util/environment-variables.hh +++ b/src/libutil/include/nix/util/environment-variables.hh @@ -13,8 +13,6 @@ namespace nix { -static constexpr auto environmentVariablesCategory = "Options that change environment variables"; - /** * @return an environment variable. */ @@ -25,12 +23,23 @@ std::optional getEnv(const std::string & key); */ std::optional getEnvOs(const OsString & key); +/** + * Like `getEnv`, but using `OsString` to avoid coercions. + */ +OsStringMap getEnvOs(); + /** * @return a non empty environment variable. Returns nullopt if the env * variable is set to "" */ std::optional getEnvNonEmpty(const std::string & key); +/** + * Like `getEnvNonEmpty`, but using `OsString` to avoid coercions. + * Returns nullopt if the env variable is not set or set to "". + */ +std::optional getEnvOsNonEmpty(const OsString & key); + /** * Get the entire environment. */ @@ -56,6 +65,11 @@ int setEnv(const char * name, const char * value); */ int setEnvOs(const OsString & name, const OsString & value); +/** + * Like `unsetenv`, but using `OsChar` to avoid coercions. + */ +int unsetEnvOs(const OsChar * name); + /** * Clear the environment. */ diff --git a/src/libutil/include/nix/util/error.hh b/src/libutil/include/nix/util/error.hh index cc8460592a2b..63131ec5b642 100644 --- a/src/libutil/include/nix/util/error.hh +++ b/src/libutil/include/nix/util/error.hh @@ -17,7 +17,10 @@ #include "nix/util/suggestions.hh" #include "nix/util/fmt.hh" +#include "nix/util/fun.hh" +#include "nix/util/config.hh" +#include #include #include #include @@ -27,6 +30,9 @@ #include #include #include +#ifdef _WIN32 +# include +#endif namespace nix { @@ -123,25 +129,25 @@ public: BaseError & operator=(BaseError &&) = default; template - BaseError(unsigned int status, const Args &... args) - : err{.level = lvlError, .msg = HintFmt(args...), .status = status} + BaseError(unsigned int status, Args &&... args) + : err{.level = lvlError, .msg = HintFmt(std::forward(args)...), .pos = {}, .status = status} { } template - explicit BaseError(const std::string & fs, const Args &... args) - : err{.level = lvlError, .msg = HintFmt(fs, args...)} + explicit BaseError(const std::string & fs, Args &&... args) + : err{.level = lvlError, .msg = HintFmt(fs, std::forward(args)...), .pos = {}} { } template - BaseError(const Suggestions & sug, const Args &... args) - : err{.level = lvlError, .msg = HintFmt(args...), .suggestions = sug} + BaseError(const Suggestions & sug, Args &&... args) + : err{.level = lvlError, .msg = HintFmt(std::forward(args)...), .pos = {}, .suggestions = sug} { } BaseError(HintFmt hint) - : err{.level = lvlError, .msg = hint} + : err{.level = lvlError, .msg = hint, .pos = {}} { } @@ -156,7 +162,7 @@ public: } /** The error message without "error: " prefixed to it. */ - std::string message() + std::string message() const { return err.msg.str(); } @@ -187,6 +193,8 @@ public: err.pos = pos; } + bool hasPos() const; + void pushTrace(Trace trace) { err.traces.push_front(trace); @@ -200,9 +208,9 @@ public: * @param args... Format string arguments. */ template - void addTrace(std::shared_ptr && pos, std::string_view fs, const Args &... args) + void addTrace(std::shared_ptr && pos, std::string_view fs, Args &&... args) { - addTrace(std::move(pos), HintFmt(std::string(fs), args...)); + addTrace(std::move(pos), HintFmt(std::string(fs), std::forward(args)...)); } /** @@ -219,17 +227,47 @@ public: return !err.traces.empty(); } - const ErrorInfo & info() + /** + * Returns a mutable reference to the error info. + * + * @warning After modifying the returned ErrorInfo, you must call + * recalcWhat() to update the cached formatted message. + */ + ErrorInfo & unsafeInfo() { return err; - }; + } + + /** + * Recalculate the cached formatted error message. + * Must be called after modifying the error info via unsafeInfo(). + */ + void recalcWhat() const; + + [[noreturn]] virtual void throwClone() const = 0; }; -#define MakeError(newClass, superClass) \ - class newClass : public superClass \ - { \ - public: \ - using superClass::superClass; \ +template +class CloneableError : public Base +{ +public: + using Base::Base; + + /** + * Rethrow a copy of this exception. Useful when the exception can get + * modified when appending traces. + */ + [[noreturn]] void throwClone() const override + { + throw Derived(static_cast(*this)); + } +}; + +#define MakeError(newClass, superClass) \ + class newClass : public CloneableError \ + { \ + public: \ + using CloneableError::CloneableError; \ } MakeError(Error, BaseError); @@ -237,9 +275,87 @@ MakeError(UsageError, Error); MakeError(UnimplementedError, Error); /** - * To use in catch-blocks. + * To use in catch-blocks. Provides a convenience method to get the portable + * std::error_code. Use when you want to catch and check an error condition like + * no_such_file_or_directory (ENOENT) without ifdefs. */ -MakeError(SystemError, Error); +class SystemError : public CloneableError +{ + std::error_code errorCode; + std::string errorDetails; + +protected: + + /** + * Just here to allow derived classes to use the right constructor + * (the protected one). + * + * This one indicates the prebuilt `HintFmt` one with the explicit `errorDetails` + */ + struct DisambigHintFmt + {}; + + /** + * Just here to allow derived classes to use the right constructor + * (the protected one). + * + * This one indicates the varargs one to build the `HintFmt` with the explicit `errorDetails` + */ + struct DisambigVarArgs + {}; + + /** + * Protected constructor that takes a pre-built HintFmt. + * Use this when the error message needs to be constructed before + * capturing errno/GetLastError(). + */ + SystemError(DisambigHintFmt, std::error_code errorCode, std::string_view errorDetails, const HintFmt & hf) + : CloneableError(HintFmt{"%s: %s", Uncolored(hf.str()), errorDetails}) + , errorCode(errorCode) + , errorDetails(errorDetails) + { + } + + /** + * Protected constructor for subclasses that provide their own error message. + * The error message is appended to the formatted hint. + */ + template + SystemError(DisambigVarArgs, std::error_code errorCode, std::string_view errorDetails, Args &&... args) + : SystemError(DisambigHintFmt{}, errorCode, errorDetails, HintFmt{std::forward(args)...}) + { + } + +public: + /** + * Construct with an error code. The error code's message is automatically + * appended to the error message. + */ + SystemError(std::error_code errorCode, const HintFmt & hf) + : SystemError(DisambigHintFmt{}, errorCode, errorCode.message(), hf) + { + } + + /** + * Construct with an error code. The error code's message is automatically + * appended to the error message. + */ + template + SystemError(std::error_code errorCode, Args &&... args) + : SystemError(DisambigVarArgs{}, errorCode, errorCode.message(), std::forward(args)...) + { + } + + const std::error_code ec() const & + { + return errorCode; + } + + bool is(std::errc e) const + { + return errorCode == e; + } +}; /** * POSIX system error, created using `errno`, `strerror` friends. @@ -257,7 +373,7 @@ MakeError(SystemError, Error); * support is too WIP to justify the code churn, but if it is finished * then a better identifier becomes moe worth it. */ -class SysError : public SystemError +class SysError final : public CloneableError { public: int errNo; @@ -267,12 +383,27 @@ public: * will be used to try to add additional information to the message. */ template - SysError(int errNo, const Args &... args) - : SystemError("") + SysError(int errNo, Args &&... args) + : CloneableError( + DisambigVarArgs{}, + std::make_error_code(static_cast(errNo)), + strerror(errNo), + std::forward(args)...) + , errNo(errNo) + { + } + + /** + * Construct using the explicitly-provided error number. `strerror` + * will be used to try to add additional information to the message. + * + * Unlike above, the `HintFmt` already exists rather than being made on + * the spot. + */ + SysError(int errNo, const HintFmt & hf) + : CloneableError(DisambigHintFmt{}, std::make_error_code(static_cast(errNo)), strerror(errNo), hf) , errNo(errNo) { - auto hf = HintFmt(args...); - err.msg = HintFmt("%1%: %2%", Uncolored(hf.str()), strerror(errNo)); } /** @@ -282,30 +413,39 @@ public: * calling this constructor! */ template - SysError(const Args &... args) - : SysError(errno, args...) + SysError(Args &&... args) + : SysError(errno, std::forward(args)...) { } -}; -#ifdef _WIN32 -namespace windows { -class WinError; -} -#endif + /** + * Construct using the ambient `errno` and a function that produces + * a `HintFmt`. errno is read first, then the function is called, so + * the function is safe to modify `errno`. + */ + SysError(auto && mkHintFmt) + requires std::invocable && std::same_as, HintFmt> + : SysError(captureErrno(std::forward(mkHintFmt))) + { + } -/** - * Convenience alias for when we use a `errno`-based error handling - * function on Unix, and `GetLastError()`-based error handling on on - * Windows. - */ -using NativeSysError = -#ifdef _WIN32 - windows::WinError -#else - SysError -#endif - ; +private: + /** + * Helper to ensure errno is captured before mkHintFmt is called. + * C++ argument evaluation order is unspecified, so we can't rely on + * `SysError(errno, mkHintFmt())` evaluating errno first. + */ + static std::pair captureErrno(auto && mkHintFmt) + { + int e = errno; + return {e, mkHintFmt()}; + } + + SysError(std::pair && p) + : SysError(p.first, std::move(p.second)) + { + } +}; /** * Throw an exception for the purpose of checking that exception @@ -319,6 +459,16 @@ void throwExceptionSelfCheck(); [[noreturn]] void panic(std::string_view msg); +/** + * Run a function, printing an error and returning on exception. + * Useful for wrapping a `main` function that may throw + * + * @param programName Name of program, usually argv[0] + * @param body Function to run inside the try block + * @return exit code: 0 if success, 1 if exception does not specify. + */ +int handleExceptions(const std::string & programName, fun body); + /** * Print a basic error message with source position and std::terminate(). * @@ -326,4 +476,119 @@ void panic(std::string_view msg); */ [[gnu::noinline, gnu::cold, noreturn]] void unreachable(std::source_location loc = std::source_location::current()); +#if NIX_UBSAN_ENABLED +/* When building with sanitizers, also enable expensive unreachable checks. In + optimised builds this explicitly invokes UB with std::unreachable for better + optimisations. */ +# define nixUnreachableWhenHardened ::nix::unreachable +#else +# define nixUnreachableWhenHardened std::unreachable +#endif + +#ifdef _WIN32 + +namespace windows { + +/** + * Windows Error type. + * + * Unless you need to catch a specific error number, don't catch this in + * portable code. Catch `SystemError` instead. + */ +class WinError : public CloneableError +{ +public: + DWORD lastError; + + /** + * Construct using the explicitly-provided error number. + * `FormatMessageA` will be used to try to add additional + * information to the message. + */ + template + WinError(DWORD lastError, Args &&... args) + : CloneableError( + DisambigVarArgs{}, + std::error_code(lastError, std::system_category()), + renderError(lastError), + std::forward(args)...) + , lastError(lastError) + { + } + + /** + * Construct using the explicitly-provided error number. + * `FormatMessageA` will be used to try to add additional + * information to the message. + * + * Unlike above, the `HintFmt` already exists rather than being made on + * the spot. + */ + WinError(DWORD lastError, const HintFmt & hf) + : CloneableError( + DisambigHintFmt{}, std::error_code(lastError, std::system_category()), renderError(lastError), hf) + , lastError(lastError) + { + } + + /** + * Construct using `GetLastError()` and the ambient "last error". + * + * Be sure to not perform another last-error-modifying operation + * before calling this constructor! + */ + template + WinError(Args &&... args) + : WinError(GetLastError(), std::forward(args)...) + { + } + + /** + * Construct using `GetLastError()` and a function that produces a + * `HintFmt`. `GetLastError()` is called first, then the function is + * called, so the function is safe to modify the last error. + */ + WinError(auto && mkHintFmt) + requires std::invocable && std::same_as, HintFmt> + : WinError(captureLastError(std::forward(mkHintFmt))) + { + } + +private: + /** + * Helper to ensure GetLastError() is captured before mkHintFmt is called. + * C++ argument evaluation order is unspecified, so we can't rely on + * `WinError(GetLastError(), mkHintFmt())` evaluating GetLastError() first. + */ + static std::pair captureLastError(auto && mkHintFmt) + { + DWORD e = GetLastError(); + return {e, mkHintFmt()}; + } + + WinError(std::pair && p) + : WinError(p.first, std::move(p.second)) + { + } + + static std::string renderError(DWORD lastError); +}; + +} // namespace windows + +#endif + +/** + * Convenience alias for when we use a `errno`-based error handling + * function on Unix, and `GetLastError()`-based error handling on on + * Windows. + */ +using NativeSysError = +#ifdef _WIN32 + windows::WinError +#else + SysError +#endif + ; + } // namespace nix diff --git a/src/libutil/include/nix/util/executable-path.hh b/src/libutil/include/nix/util/executable-path.hh index cf6f3b252008..4e9fe39a5eab 100644 --- a/src/libutil/include/nix/util/executable-path.hh +++ b/src/libutil/include/nix/util/executable-path.hh @@ -33,6 +33,12 @@ struct ExecutablePath */ static ExecutablePath parse(const OsString & path); + /** + * Like `parse` but appends new entries to the end of an existing + * `ExecutablePath`. + */ + void parseAppend(const OsString & path); + /** * Load the `PATH` environment variable and `parse` it. */ diff --git a/src/libutil/include/nix/util/experimental-features.hh b/src/libutil/include/nix/util/experimental-features.hh index f8955ec8ca9f..85ca58e23da1 100644 --- a/src/libutil/include/nix/util/experimental-features.hh +++ b/src/libutil/include/nix/util/experimental-features.hh @@ -22,7 +22,6 @@ enum struct ExperimentalFeature { FetchTree, GitHashing, RecursiveNix, - NoUrlLiterals, FetchClosure, AutoAllocateUids, Cgroups, @@ -85,7 +84,7 @@ std::set parseFeatures(const StringSet &); * An experimental feature was required for some (experimental) * operation, but was not enabled. */ -class MissingExperimentalFeature : public Error +class MissingExperimentalFeature final : public CloneableError { public: /** diff --git a/src/libutil/include/nix/util/file-content-address.hh b/src/libutil/include/nix/util/file-content-address.hh index def1232023cb..650c864916ca 100644 --- a/src/libutil/include/nix/util/file-content-address.hh +++ b/src/libutil/include/nix/util/file-content-address.hh @@ -64,7 +64,8 @@ void dumpPath( * * \todo use an arbitrary `FileSystemObjectSink`. */ -void restorePath(const Path & path, Source & source, FileSerialisationMethod method, bool startFsync = false); +void restorePath( + const std::filesystem::path & path, Source & source, FileSerialisationMethod method, bool startFsync = false); /** * Compute the hash of the given file system object according to the diff --git a/src/libutil/include/nix/util/file-descriptor.hh b/src/libutil/include/nix/util/file-descriptor.hh index d049845883c1..1e0d726adbad 100644 --- a/src/libutil/include/nix/util/file-descriptor.hh +++ b/src/libutil/include/nix/util/file-descriptor.hh @@ -1,9 +1,17 @@ #pragma once -///@file +/** + * @file + * + * @brief File descriptor operations for almost arbitrary file + * descriptors. + * + * More specialized file-system-specific operations are in + * @ref file-system-at.hh. + */ #include "nix/util/canon-path.hh" -#include "nix/util/types.hh" #include "nix/util/error.hh" +#include "nix/util/os-string.hh" #ifdef _WIN32 # define WIN32_LEAN_AND_MEAN @@ -49,24 +57,69 @@ static inline Descriptor toDescriptor(int fd) } /** - * Convert a POSIX file descriptor to a native `Descriptor` in read-only - * mode. + * Read the contents of a resource into a string. + */ +std::string readFile(Descriptor fd); + +/** + * Platform-specific read into a buffer. * - * This is a no-op except on Windows. + * Thin wrapper around ::read (Unix) or ReadFile (Windows). + * Handles EINTR on Unix. Treats ERROR_BROKEN_PIPE as EOF on Windows. + * + * @param fd The file descriptor to read from + * @param buffer The buffer to read into + * @return The number of bytes actually read (0 indicates EOF) + * @throws SystemError on failure */ -static inline int fromDescriptorReadOnly(Descriptor fd) -{ -#ifdef _WIN32 - return _open_osfhandle(reinterpret_cast(fd), _O_RDONLY); -#else - return fd; -#endif -} +size_t read(Descriptor fd, std::span buffer); /** - * Read the contents of a resource into a string. + * Platform-specific write from a buffer. + * + * Thin wrapper around ::write (Unix) or WriteFile (Windows). + * Handles EINTR on Unix. + * + * @param fd The file descriptor to write to + * @param buffer The buffer to write from + * @return The number of bytes actually written + * @throws SystemError on failure */ -std::string readFile(Descriptor fd); +size_t write(Descriptor fd, std::span buffer, bool allowInterrupts); + +/** + * Get the size of a file. + * + * Thin wrapper around fstat (Unix) or GetFileSizeEx (Windows). + * + * @param fd The file descriptor + * @return The file size + * @throws SystemError on failure + */ +std::make_unsigned_t getFileSize(Descriptor fd); + +/** + * Platform-specific positioned read into a buffer. + * + * Thin wrapper around pread (Unix) or ReadFile with OVERLAPPED (Windows). + * Does NOT handle EINTR on Unix - caller must catch and retry if needed. + * + * @param fd The file descriptor to read from (must be seekable) + * @param offset The offset to read from + * @param buffer The buffer to read into + * @return The number of bytes actually read (0 indicates EOF) + * @throws SystemError on failure + */ +size_t readOffset(Descriptor fd, off_t offset, std::span buffer); + +/** + * Read \ref nbytes starting at \ref offset from a seekable file into a sink. + * + * @throws SystemError if fd is not seekable or any operation fails + * @throws Interrupted if the operation was interrupted + * @throws EndOfFile if an EOF was reached before reading \ref nbytes + */ +void copyFdRange(Descriptor fd, off_t offset, size_t nbytes, Sink & sink); /** * Wrappers around read()/write() that read/write exactly the @@ -77,14 +130,16 @@ void readFull(Descriptor fd, char * buf, size_t count); void writeFull(Descriptor fd, std::string_view s, bool allowInterrupts = true); /** - * Read a line from a file descriptor. + * Read a line from an unbuffered file descriptor. + * See BufferedSource::readLine for a buffered variant. * * @param fd The file descriptor to read from * @param eofOk If true, return an unterminated line if EOF is reached. (e.g. the empty string) + * @param terminator The chartacter that ends the line * * @return A line of text ending in `\n`, or a string without `\n` if `eofOk` is true and EOF is reached. */ -std::string readLine(Descriptor fd, bool eofOk = false); +std::string readLine(Descriptor fd, bool eofOk = false, char terminator = '\n'); /** * Write a line to a file descriptor. @@ -92,21 +147,68 @@ std::string readLine(Descriptor fd, bool eofOk = false); void writeLine(Descriptor fd, std::string s); /** - * Read a file descriptor until EOF occurs. + * Perform a blocking fsync operation on a file descriptor. */ -std::string drainFD(Descriptor fd, bool block = true, const size_t reserveSize = 0); +void syncDescriptor(Descriptor fd); /** - * The Windows version is always blocking. + * Options for draining a file descriptor to a sink. */ -void drainFD( - Descriptor fd, - Sink & sink +struct DrainFdSinkOpts +{ + /** + * If provided, read exactly this many bytes (throws EndOfFile if EOF occurs before reading all bytes). + */ + std::optional> expectedSize = {}; + #ifndef _WIN32 - , - bool block = true + /** + * Whether to block on read. + */ + bool block = true; #endif -); +}; + +/** + * Options for draining a file descriptor to a string. + */ +struct DrainFdOpts +{ + /** + * If expected=true: read exactly this many bytes (throws EndOfFile if EOF occurs before reading all bytes). + * If expected=false: size hint for string allocation. + */ + std::make_unsigned_t size = 0; + + /** + * If true, size is exact expected size. If false, size is just a reservation hint. + */ + bool expected = false; + +#ifndef _WIN32 + /** + * Whether to block on read. + */ + bool block = true; +#endif +}; + +/** + * Read a file descriptor until EOF occurs. + * + * @param fd The file descriptor to drain + * @param opts Options for the drain operation + */ +std::string drainFD(Descriptor fd, DrainFdOpts opts = {}); + +/** + * Read a file descriptor until EOF occurs, writing to a sink. + * + * @param fd The file descriptor to drain + * @param sink The sink to write data to + * @param opts Options for the drain operation + */ +void drainFD(Descriptor fd, Sink & sink, DrainFdSinkOpts opts = {}); /** * Get [Standard Input](https://en.wikipedia.org/wiki/Standard_streams#Standard_input_(stdin)) @@ -169,7 +271,11 @@ public: /** * Perform a blocking fsync operation. */ - void fsync() const; + void fsync() const + { + if (fd != INVALID_DESCRIPTOR) + nix::syncDescriptor(fd); + } /** * Asynchronously flush to disk without blocking, if available on @@ -204,74 +310,16 @@ void closeOnExec(Descriptor fd); } // namespace unix #endif -#ifdef __linux__ -namespace linux { - -/** - * Wrapper around Linux's openat2 syscall introduced in Linux 5.6. - * - * @see https://man7.org/linux/man-pages/man2/openat2.2.html - * @see https://man7.org/linux/man-pages/man2/open_how.2type.html -v* - * @param flags O_* flags - * @param mode Mode for O_{CREAT,TMPFILE} - * @param resolve RESOLVE_* flags - * - * @return nullopt if openat2 is not supported by the kernel. - */ -std::optional openat2(Descriptor dirFd, const char * path, uint64_t flags, uint64_t mode, uint64_t resolve); - -} // namespace linux -#endif - -#if defined(_WIN32) && _WIN32_WINNT >= 0x0600 -namespace windows { - -Path handleToPath(Descriptor handle); -std::wstring handleToFileName(Descriptor handle); - -} // namespace windows -#endif - -#ifndef _WIN32 -namespace unix { - -struct SymlinkNotAllowed : public Error -{ - CanonPath path; +MakeError(EndOfFile, Error); - SymlinkNotAllowed(CanonPath path) - /* Can't provide better error message, since the parent directory is only known to the caller. */ - : Error("relative path '%s' points to a symlink, which is not allowed", path.rel()) - , path(std::move(path)) - { - } -}; +#ifdef _WIN32 /** - * Safe(r) function to open \param path file relative to \param dirFd, while - * disallowing escaping from a directory and resolving any symlinks in the - * process. - * - * @note When not on Linux or when openat2 is not available this is implemented - * via openat single path component traversal. Uses RESOLVE_BENEATH with openat2 - * or O_RESOLVE_BENEATH. - * - * @note Since this is Unix-only path is specified as CanonPath, which models - * Unix-style paths and ensures that there are no .. or . components. - * - * @param flags O_* flags - * @param mode Mode for O_{CREAT,TMPFILE} - * - * @pre path.isRoot() is false - * - * @throws SymlinkNotAllowed if any path components + * Windows specific replacement for POSIX `lseek` that operates on a `HANDLE` and not + * a file descriptor. */ -Descriptor openFileEnsureBeneathNoSymlinks(Descriptor dirFd, const CanonPath & path, int flags, mode_t mode = 0); +off_t lseek(Descriptor fd, off_t offset, int whence); -} // namespace unix #endif -MakeError(EndOfFile, Error); - } // namespace nix diff --git a/src/libutil/include/nix/util/file-path-impl.hh b/src/libutil/include/nix/util/file-path-impl.hh index 91c1a58cd0b2..174eeb1be9aa 100644 --- a/src/libutil/include/nix/util/file-path-impl.hh +++ b/src/libutil/include/nix/util/file-path-impl.hh @@ -40,6 +40,11 @@ struct UnixPathTrait { return path.rfind('/', from); } + + static size_t rootNameLen(StringView) + { + return 0; + } }; /** @@ -83,6 +88,18 @@ struct WindowsPathTrait size_t p2 = path.rfind(preferredSep, from); return p1 == String::npos ? p2 : p2 == String::npos ? p1 : std::max(p1, p2); } + + static size_t rootNameLen(StringView path) + { + if (path.size() >= 2 && path[1] == ':') { + char driveLetter = path[0]; + if ((driveLetter >= 'A' && driveLetter <= 'Z') || (driveLetter >= 'a' && driveLetter <= 'z')) + return 2; + } + /* TODO: This needs to also handle UNC paths. + * https://learn.microsoft.com/en-us/dotnet/standard/io/file-path-formats#unc-paths */ + return 0; + } }; template @@ -116,6 +133,11 @@ typename PathDict::String canonPathInner(typename PathDict::StringView remaining typename PathDict::String result; result.reserve(256); + if (auto rootNameLength = PathDict::rootNameLen(remaining)) { + result += remaining.substr(0, rootNameLength); /* Copy drive letter verbatim. */ + remaining.remove_prefix(rootNameLength); + } + while (true) { /* Skip slashes. */ diff --git a/src/libutil/include/nix/util/file-path.hh b/src/libutil/include/nix/util/file-path.hh index 52dae32ff358..f570dcec3417 100644 --- a/src/libutil/include/nix/util/file-path.hh +++ b/src/libutil/include/nix/util/file-path.hh @@ -11,30 +11,26 @@ namespace nix { /** * Paths are just `std::filesystem::path`s. - * - * @todo drop `NG` suffix and replace the ones in `types.hh`. */ -typedef std::list PathsNG; -typedef std::set PathSetNG; +typedef std::list Paths; +typedef std::set PathSet; /** * Stop gap until `std::filesystem::path_view` from P1030R6 exists in a * future C++ standard. - * - * @todo drop `NG` suffix and replace the one in `types.hh`. */ -struct PathViewNG : OsStringView +struct PathView : OsStringView { using string_view = OsStringView; using string_view::string_view; - PathViewNG(const std::filesystem::path & path) + PathView(const std::filesystem::path & path) : OsStringView{path.native()} { } - PathViewNG(const OsString & path) + PathView(const OsString & path) : OsStringView{path} { } @@ -52,7 +48,7 @@ struct PathViewNG : OsStringView std::optional maybePath(PathView path); -std::filesystem::path pathNG(PathView path); +std::filesystem::path toOwnedPath(PathView path); template<> struct json_avoids_null : std::true_type diff --git a/src/libutil/include/nix/util/file-system-at.hh b/src/libutil/include/nix/util/file-system-at.hh new file mode 100644 index 000000000000..4558f0e0ef88 --- /dev/null +++ b/src/libutil/include/nix/util/file-system-at.hh @@ -0,0 +1,107 @@ +#pragma once +/** + * @file + * + * @brief File system operations relative to directory file descriptors. + * + * This header provides cross-platform wrappers for POSIX `*at` functions + * (e.g., `symlinkat`, `mkdirat`, `readlinkat`) that operate relative to + * a directory file descriptor. + * + * Prefer this to @ref file-system.hh because file descriptor-based file + * system operations are necessary to avoid + * [TOCTOU](https://en.wikipedia.org/wiki/Time-of-check_to_time-of-use) + * issues. + */ + +#include "nix/util/file-descriptor.hh" + +#include + +#ifdef _WIN32 +# define WIN32_LEAN_AND_MEAN +# include +#endif + +namespace nix { + +/** + * Read a symlink relative to a directory file descriptor. + * + * @throws SystemError on any I/O errors. + * @throws Interrupted if interrupted. + */ +OsString readLinkAt(Descriptor dirFd, const CanonPath & path); + +/** + * Safe(r) function to open a file relative to dirFd, while + * disallowing escaping from a directory and any symlinks in the process. + * + * @note On Windows, implemented via NtCreateFile single path component traversal + * with FILE_OPEN_REPARSE_POINT. On Unix, uses RESOLVE_BENEATH with openat2 when + * available, or falls back to openat single path component traversal. + * + * @param dirFd Directory handle to open relative to + * @param path Relative path (no .. or . components) + * @param desiredAccess (Windows) Windows ACCESS_MASK (e.g., GENERIC_READ, FILE_WRITE_DATA) + * @param createOptions (Windows) Windows create options (e.g., FILE_NON_DIRECTORY_FILE) + * @param createDisposition (Windows) FILE_OPEN, FILE_CREATE, etc. + * @param flags (Unix) O_* flags + * @param mode (Unix) Mode for O_{CREAT,TMPFILE} + * + * @pre path.isRoot() is false + * + * @throws SymlinkNotAllowed if any path components are symlinks + * @throws SystemError on other errors + */ +AutoCloseFD openFileEnsureBeneathNoSymlinks( + Descriptor dirFd, + const CanonPath & path, +#ifdef _WIN32 + ACCESS_MASK desiredAccess, + ULONG createOptions, + ULONG createDisposition = FILE_OPEN +#else + int flags, + mode_t mode = 0 +#endif +); + +#ifdef __linux__ +namespace linux { + +/** + * Wrapper around Linux's openat2 syscall introduced in Linux 5.6. + * + * @see https://man7.org/linux/man-pages/man2/openat2.2.html + * @see https://man7.org/linux/man-pages/man2/open_how.2type.html + * + * @param flags O_* flags + * @param mode Mode for O_{CREAT,TMPFILE} + * @param resolve RESOLVE_* flags + * + * @return nullopt if openat2 is not supported by the kernel. + */ +std::optional openat2(Descriptor dirFd, const char * path, uint64_t flags, uint64_t mode, uint64_t resolve); + +} // namespace linux +#endif + +#ifndef _WIN32 +namespace unix { + +/** + * Try to change the mode of file named by \ref path relative to the parent directory denoted by \ref dirFd. + * + * @note When on linux without fchmodat2 support and without procfs mounted falls back to fchmodat without + * AT_SYMLINK_NOFOLLOW, since it's the best we can do without failing. + * + * @pre path.isRoot() is false + * @throws SysError if any operation fails + */ +void fchmodatTryNoFollow(Descriptor dirFd, const CanonPath & path, mode_t mode); + +} // namespace unix +#endif + +} // namespace nix diff --git a/src/libutil/include/nix/util/file-system.hh b/src/libutil/include/nix/util/file-system.hh index 7d88939e32ec..067240812f9a 100644 --- a/src/libutil/include/nix/util/file-system.hh +++ b/src/libutil/include/nix/util/file-system.hh @@ -2,19 +2,25 @@ /** * @file * - * Utilities for working with the file system and file paths. + * @brief Utilities for working with the file system and file paths. + * + * Please try to use @ref file-system-at.hh instead of this where + * possible, for the reasons given in the documentation of that header. */ +#include "nix/util/fun.hh" #include "nix/util/types.hh" #include "nix/util/file-descriptor.hh" #include "nix/util/file-path.hh" +#include #include #include #include #include #ifdef _WIN32 # include +# include #endif #include @@ -36,11 +42,6 @@ namespace nix { struct Sink; struct Source; -/** - * Return whether the path denotes an absolute path. - */ -bool isAbsolute(PathView path); - /** * @return An absolutized path, resolving paths relative to the * specified directory, or the current directory otherwise. The path @@ -48,13 +49,6 @@ bool isAbsolute(PathView path); * * In the process of being deprecated for `std::filesystem::absolute`. */ -Path absPath(PathView path, std::optional dir = {}, bool resolveSymlinks = false); - -inline Path absPath(const Path & path, std::optional dir = {}, bool resolveSymlinks = false) -{ - return absPath(PathView{path}, dir, resolveSymlinks); -} - std::filesystem::path absPath(const std::filesystem::path & path, const std::filesystem::path * dir = nullptr, bool resolveSymlinks = false); @@ -69,18 +63,7 @@ absPath(const std::filesystem::path & path, const std::filesystem::path * dir = * false` case), and `std::filesystem::weakly_canonical` (for the * `resolveSymlinks = true` case). */ -Path canonPath(PathView path, bool resolveSymlinks = false); - -/** - * @return The directory part of the given canonical path, i.e., - * everything before the final `/`. If the path is the root or an - * immediate child thereof (e.g., `/foo`), this means `/` - * is returned. - * - * In the process of being deprecated for - * `std::filesystem::path::parent_path`. - */ -Path dirOf(const PathView path); +std::filesystem::path canonPath(const std::filesystem::path & path, bool resolveSymlinks = false); /** * @return the base name of the given canonical path, i.e., everything @@ -103,16 +86,35 @@ bool isInDir(const std::filesystem::path & path, const std::filesystem::path & d */ bool isDirOrInDir(const std::filesystem::path & path, const std::filesystem::path & dir); +/** + * `struct stat` is not 64-bit everywhere on Windows. + */ +using PosixStat = +#ifdef _WIN32 + struct ::__stat64 +#else + struct ::stat +#endif + ; + /** * Get status of `path`. */ -struct stat stat(const Path & path); -struct stat lstat(const Path & path); +PosixStat lstat(const std::filesystem::path & path); +/** + * Get status of `path` following symlinks. + */ +PosixStat stat(const std::filesystem::path & path); +/** + * Get status of an open file descriptor. + */ +PosixStat fstat(int fd); /** * `lstat` the given path if it exists. * @return std::nullopt if the path doesn't exist, or an optional containing the result of `lstat` otherwise */ -std::optional maybeLstat(const Path & path); +std::optional maybeLstat(const std::filesystem::path & path); +std::optional maybeStat(const std::filesystem::path & path); /** * @return true iff the given path exists. @@ -147,63 +149,85 @@ bool pathAccessible(const std::filesystem::path & path); /** * Read the contents (target) of a symbolic link. The result is not * in any way canonicalised. - * - * In the process of being deprecated for - * `std::filesystem::read_symlink`. */ -Path readLink(const Path & path); +std::filesystem::path readLink(const std::filesystem::path & path); /** - * Read the contents (target) of a symbolic link. The result is not - * in any way canonicalised. + * Get the path associated with a file descriptor. + * + * @note One MUST only use this for error handling, because it creates + * TOCTOU issues. We don't mind if error messages point to out of date + * paths (that is a rather trivial TOCTOU --- the error message is best + * effort) but for anything else we do. + * + * @note this function will clobber `errno` (Unix) / "last error" + * (Windows), so care must be used to get those error codes, then call + * this, then build a `SysError` / `WinError` with the saved error code. */ -std::filesystem::path readLink(const std::filesystem::path & path); +std::filesystem::path descriptorToPath(Descriptor fd); /** * Open a `Descriptor` with read-only access to the given directory. */ -Descriptor openDirectory(const std::filesystem::path & path); +AutoCloseFD openDirectory(const std::filesystem::path & path); + +/** + * Open a `Descriptor` with read-only access to the given file. + * + * @note For directories use @ref openDirectory. + */ +AutoCloseFD openFileReadonly(const std::filesystem::path & path); + +struct OpenNewFileForWriteParams +{ + /** + * Whether to truncate an existing file. + */ + bool truncateExisting:1 = false; + /** + * Whether to follow symlinks if @ref truncateExisting is true. + */ + bool followSymlinksOnTruncate:1 = false; +}; + +/** + * Open a `Descriptor` for write access or create it if it doesn't exist or truncate existing depending on @ref + * truncateExisting. + * + * @param mode POSIX permission bits. Ignored on Windows. + * @throws Nothing. + * + * @todo Reparse points on Windows. + */ +AutoCloseFD openNewFileForWrite(const std::filesystem::path & path, mode_t mode, OpenNewFileForWriteParams params); /** * Read the contents of a file into a string. */ -std::string readFile(const Path & path); std::string readFile(const std::filesystem::path & path); -void readFile(const Path & path, Sink & sink, bool memory_map = true); +void readFile(const std::filesystem::path & path, Sink & sink, bool memory_map = true); enum struct FsSync { Yes, No }; /** * Write a string to a file. */ -void writeFile(const Path & path, std::string_view s, mode_t mode = 0666, FsSync sync = FsSync::No); +void writeFile(const std::filesystem::path & path, std::string_view s, mode_t mode = 0666, FsSync sync = FsSync::No); -static inline void -writeFile(const std::filesystem::path & path, std::string_view s, mode_t mode = 0666, FsSync sync = FsSync::No) -{ - return writeFile(path.string(), s, mode, sync); -} - -void writeFile(const Path & path, Source & source, mode_t mode = 0666, FsSync sync = FsSync::No); - -static inline void -writeFile(const std::filesystem::path & path, Source & source, mode_t mode = 0666, FsSync sync = FsSync::No) -{ - return writeFile(path.string(), source, mode, sync); -} +void writeFile(const std::filesystem::path & path, Source & source, mode_t mode = 0666, FsSync sync = FsSync::No); void writeFile( - AutoCloseFD & fd, const Path & origPath, std::string_view s, mode_t mode = 0666, FsSync sync = FsSync::No); + Descriptor fd, std::string_view s, FsSync sync = FsSync::No, const std::filesystem::path * origPath = nullptr); /** * Flush a path's parent directory to disk. */ -void syncParent(const Path & path); +void syncParent(const std::filesystem::path & path); /** * Flush a file or entire directory tree to disk. */ -void recursiveSync(const Path & path); +void recursiveSync(const std::filesystem::path & path); /** * Delete a path; i.e., in the case of a directory, it is deleted @@ -224,7 +248,7 @@ void createDirs(const std::filesystem::path & path); /** * Create a single directory. */ -void createDir(const Path & path, mode_t mode = 0755); +void createDir(const std::filesystem::path & path, mode_t mode = 0755); /** * Set the access and modification times of the given path, not @@ -246,26 +270,21 @@ void setWriteTime( std::optional isSymlink = std::nullopt); /** - * Convenience wrapper that takes all arguments from the `struct stat`. + * Convenience wrapper that takes all arguments from the `PosixStat`. */ -void setWriteTime(const std::filesystem::path & path, const struct stat & st); +void setWriteTime(const std::filesystem::path & path, const PosixStat & st); /** * Create a symlink. * */ -void createSymlink(const Path & target, const Path & link); +void createSymlink(const std::filesystem::path & target, const std::filesystem::path & link); /** * Atomically create or replace a symlink. */ void replaceSymlink(const std::filesystem::path & target, const std::filesystem::path & link); -inline void replaceSymlink(const Path & target, const Path & link) -{ - return replaceSymlink(std::filesystem::path{target}, std::filesystem::path{link}); -} - /** * Similar to 'renameFile', but fallback to a copy+remove if `src` and `dst` * are on a different filesystem. @@ -273,7 +292,7 @@ inline void replaceSymlink(const Path & target, const Path & link) * Beware that this might not be atomic because of the copy that happens behind * the scenes */ -void moveFile(const Path & src, const Path & dst); +void moveFile(const std::filesystem::path & src, const std::filesystem::path & dst); /** * Recursively copy the content of `oldPath` to `newPath`. If `andDelete` is @@ -302,22 +321,44 @@ public: x.del = false; } + AutoDelete & operator=(AutoDelete && x) noexcept + { + swap(*this, x); + return *this; + } + + friend void swap(AutoDelete & lhs, AutoDelete & rhs) noexcept + { + using std::swap; + swap(lhs._path, rhs._path); + swap(lhs.del, rhs.del); + swap(lhs.recursive, rhs.recursive); + } + AutoDelete(const std::filesystem::path & p, bool recursive = true); AutoDelete(const AutoDelete &) = delete; - AutoDelete & operator=(AutoDelete &&) = delete; AutoDelete & operator=(const AutoDelete &) = delete; ~AutoDelete(); - void cancel(); + /** + * Delete the file the path points to, and cancel this `AutoDelete`, + * so deletion is not attempted a second time by the destructor. + * + * The destructor calls this, but ignoring any exception. + */ + void deletePath(); - void reset(const std::filesystem::path & p, bool recursive = true); + /** + * Cancel the pending deletion + */ + void cancel() noexcept; const std::filesystem::path & path() const { return _path; } - PathViewNG view() const + PathView view() const { return _path; } @@ -327,7 +368,7 @@ public: return _path; } - operator PathViewNG() const + operator PathView() const { return _path; } @@ -358,10 +399,12 @@ AutoCloseFD createAnonymousTempFile(); /** * Create a temporary file, returning a file handle and its path. */ -std::pair createTempFile(const Path & prefix = "nix"); +std::pair createTempFile(const std::filesystem::path & prefix = "nix"); /** * Return `TMPDIR`, or the default temporary directory if unset or empty. + * Uses GetTempPathW on windows which respects TMP, TEMP, USERPROFILE env variables. + * Does not resolve symlinks and the returned path might not be directory or exist at all. */ std::filesystem::path defaultTempDir(); @@ -381,8 +424,10 @@ std::filesystem::path makeTempPath(const std::filesystem::path & root, const std /** * Used in various places. + * + * @todo type */ -typedef std::function PathFilter; +typedef fun PathFilter; extern PathFilter defaultPathFilter; @@ -402,6 +447,31 @@ extern PathFilter defaultPathFilter; */ bool chmodIfNeeded(const std::filesystem::path & path, mode_t mode, mode_t mask = S_IRWXU | S_IRWXG | S_IRWXO); +/** + * Set permissions on a path, throwing an exception on error. + * + * @param path Path to the file to change the permissions for. + * @param mode New file mode. + * + * @todo stop using this and start using `fchmodatTryNoFollow` (or a different + * wrapper) to avoid TOCTOU issues. + */ +void chmod(const std::filesystem::path & path, mode_t mode); + +/** + * Remove a file, throwing an exception on error. + * + * @param path Path to the file to remove. + */ +void unlink(const std::filesystem::path & path); + +/** + * Try to remove a file, ignoring errors. + * + * @param path Path to the file to try to remove. + */ +void tryUnlink(const std::filesystem::path & path); + /** * @brief A directory iterator that can be used to iterate over the * contents of a directory. It is similar to std::filesystem::directory_iterator @@ -477,13 +547,46 @@ private: #ifdef __FreeBSD__ class AutoUnmount { - Path path; + std::filesystem::path path; bool del; public: - AutoUnmount(Path &); AutoUnmount(); + AutoUnmount(const std::filesystem::path &); + AutoUnmount(const AutoUnmount &) = delete; + + AutoUnmount(AutoUnmount && other) noexcept + : path(std::move(other.path)) + , del(std::exchange(other.del, false)) + { + } + + AutoUnmount & operator=(AutoUnmount && other) noexcept + { + swap(*this, other); + return *this; + } + + friend void swap(AutoUnmount & lhs, AutoUnmount & rhs) noexcept + { + using std::swap; + swap(lhs.path, rhs.path); + swap(lhs.del, rhs.del); + } + ~AutoUnmount(); - void cancel(); + + /** + * Cancel the unmounting + */ + void cancel() noexcept; + + /** + * Unmount the mountpoint right away (if it exists), resetting the + * `AutoUnmount` + * + * The destructor calls this, but ignoring any exception. + */ + void unmount(); }; #endif diff --git a/src/libutil/include/nix/util/fmt.hh b/src/libutil/include/nix/util/fmt.hh index f32a0b62b50a..97fff2cf62a2 100644 --- a/src/libutil/include/nix/util/fmt.hh +++ b/src/libutil/include/nix/util/fmt.hh @@ -3,6 +3,7 @@ #include #include +#include #include "nix/util/ansicolor.hh" namespace nix { @@ -27,6 +28,8 @@ inline void formatHelper(F & f) template inline void formatHelper(F & f, const T & x, const Args &... args) { + static_assert(!std::is_same_v, std::filesystem::path>); + static_assert(!(std::is_same_v, std::filesystem::path> || ...)); // Interpolate one argument and then recurse. formatHelper(f % x, args...); } @@ -103,12 +106,37 @@ struct Magenta const T & value; }; +/** + * All std::filesystem::path values must be wrapped in this class when formatting via HintFmt + * or fmt(). This avoids accidentail double-quoting due to the standard operator<< implementation + * for std::filesystem::path. + */ +struct PathFmt +{ + explicit PathFmt(const std::filesystem::path & p) + { +#ifdef _WIN32 + auto s = p.u8string(); + value = std::string(reinterpret_cast(s.data()), s.size()); +#else + value = p.string(); +#endif + } + + std::string value; +}; + template std::ostream & operator<<(std::ostream & out, const Magenta & y) { return out << ANSI_WARNING << y.value << ANSI_NORMAL; } +inline std::ostream & operator<<(std::ostream & out, const PathFmt & y) +{ + return out << "\"" << y.value << "\""; +} + /** * Values wrapped in this class are printed without coloring. * @@ -175,10 +203,13 @@ public: HintFmt(boost::format && fmt, const Args &... args) : fmt(std::move(fmt)) { + static_assert(!(std::is_same_v, std::filesystem::path> || ...)); setExceptions(fmt); formatHelper(*this, args...); } + HintFmt & operator%(const std::filesystem::path & value) = delete; + template HintFmt & operator%(const T & value) { diff --git a/src/libutil/include/nix/util/forwarding-source-accessor.hh b/src/libutil/include/nix/util/forwarding-source-accessor.hh index 02474a3a7f31..c9693b9e5f92 100644 --- a/src/libutil/include/nix/util/forwarding-source-accessor.hh +++ b/src/libutil/include/nix/util/forwarding-source-accessor.hh @@ -18,12 +18,7 @@ struct ForwardingSourceAccessor : SourceAccessor { } - std::string readFile(const CanonPath & path) override - { - return next->readFile(path); - } - - void readFile(const CanonPath & path, Sink & sink, std::function sizeCallback) override + void readFile(const CanonPath & path, Sink & sink, fun sizeCallback) override { next->readFile(path, sink, sizeCallback); } diff --git a/src/libutil/include/nix/util/fs-sink.hh b/src/libutil/include/nix/util/fs-sink.hh index bd9c7205fa8b..495a4bab995c 100644 --- a/src/libutil/include/nix/util/fs-sink.hh +++ b/src/libutil/include/nix/util/fs-sink.hh @@ -36,7 +36,7 @@ struct FileSystemObjectSink virtual void createDirectory(const CanonPath & path) = 0; - using DirectoryCreatedCallback = std::function; + using DirectoryCreatedCallback = fun; /** * Create a directory and invoke a callback with a pair of sink + CanonPath @@ -57,7 +57,7 @@ struct FileSystemObjectSink * This function in general is no re-entrant. Only one file can be * written at a time. */ - virtual void createRegularFile(const CanonPath & path, std::function) = 0; + virtual void createRegularFile(const CanonPath & path, fun) = 0; virtual void createSymlink(const CanonPath & path, const std::string & target) = 0; }; @@ -90,7 +90,7 @@ struct NullFileSystemObjectSink : FileSystemObjectSink void createSymlink(const CanonPath & path, const std::string & target) override {} - void createRegularFile(const CanonPath & path, std::function) override; + void createRegularFile(const CanonPath & path, fun) override; }; /** @@ -99,7 +99,6 @@ struct NullFileSystemObjectSink : FileSystemObjectSink struct RestoreSink : FileSystemObjectSink { std::filesystem::path dstPath; -#ifndef _WIN32 /** * File descriptor for the directory located at dstPath. Used for *at * operations relative to this file descriptor. This sink must *never* @@ -110,7 +109,6 @@ struct RestoreSink : FileSystemObjectSink * is not susceptible to symlink replacement. */ AutoCloseFD dirFd; -#endif bool startFsync = false; explicit RestoreSink(bool startFsync) @@ -120,11 +118,9 @@ struct RestoreSink : FileSystemObjectSink void createDirectory(const CanonPath & path) override; -#ifndef _WIN32 void createDirectory(const CanonPath & path, DirectoryCreatedCallback callback) override; -#endif - void createRegularFile(const CanonPath & path, std::function) override; + void createRegularFile(const CanonPath & path, fun) override; void createSymlink(const CanonPath & path, const std::string & target) override; }; @@ -154,7 +150,7 @@ struct RegularFileSink : FileSystemObjectSink regular = false; } - void createRegularFile(const CanonPath & path, std::function) override; + void createRegularFile(const CanonPath & path, fun) override; }; } // namespace nix diff --git a/src/libutil/include/nix/util/fun.hh b/src/libutil/include/nix/util/fun.hh new file mode 100644 index 000000000000..c480ffe71711 --- /dev/null +++ b/src/libutil/include/nix/util/fun.hh @@ -0,0 +1,93 @@ +#pragma once +///@file +// Tests in: src/libutil-tests/fun.cc + +#include +#include +#include +#include + +namespace nix { + +/** + * A non-nullable wrapper around `std::function`. + * + * Like `ref` guarantees a non-null pointer, `fun` guarantees + * a non-null callable. Construction from an empty `std::function` or + * `nullptr` is rejected. + */ +template +class fun; + +template +class fun +{ +private: + + std::function f; + + void assertCallable() + { + if (!f) + throw std::invalid_argument("null callable cast to fun"); + } + +public: + + using result_type = Ret; + + /** + * Construct from any callable that `std::function` accepts. + * + * Excludes `fun` itself (handled by copy/move), `std::function` + * (handled by explicit constructors below), and `nullptr`. + */ + template + requires( + !std::is_same_v, fun> && !std::is_same_v, std::function> + && !std::is_same_v, std::nullptr_t> + && std::is_constructible_v, F>) + fun(F && callable) + : f(std::forward(callable)) + { + assertCallable(); + } + + /** + * Construct from an existing `std::function`. + * Explicit because an empty `std::function` will throw. + */ + explicit fun(const std::function & fn) + : f(fn) + { + assertCallable(); + } + + explicit fun(std::function && fn) + : f(std::move(fn)) + { + assertCallable(); + } + + /** Prevent construction from nullptr at compile time. */ + fun(std::nullptr_t) = delete; + + template + Ret operator()(Ts &&... args) const + { + return f(std::forward(args)...); + } + + /** Get the underlying `std::function`. */ + const std::function & get_fn() const & + { + return f; + } + + std::function get_fn() && + { + return std::move(f); + } +}; + +} // namespace nix diff --git a/src/libutil/include/nix/util/git.hh b/src/libutil/include/nix/util/git.hh index 5140c76c4930..01f4c4b88762 100644 --- a/src/libutil/include/nix/util/git.hh +++ b/src/libutil/include/nix/util/git.hh @@ -102,7 +102,7 @@ void parseTree( const CanonPath & sinkPath, Source & source, HashAlgorithm hashAlgo, - std::function hook, + fun hook, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); /** @@ -120,7 +120,7 @@ void parse( Source & source, BlobMode rootModeIfBlob, HashAlgorithm hashAlgo, - std::function hook, + fun hook, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); /** @@ -141,7 +141,7 @@ using RestoreHook = SourcePath(Hash); * * @param hashAlgo must be `HashAlgo::SHA1` or `HashAlgo::SHA256` for now. */ -void restore(FileSystemObjectSink & sink, Source & source, HashAlgorithm hashAlgo, std::function hook); +void restore(FileSystemObjectSink & sink, Source & source, HashAlgorithm hashAlgo, fun hook); /** * Dumps a single file to a sink @@ -171,7 +171,7 @@ using DumpHook = TreeEntry(const SourcePath & path); Mode dump( const SourcePath & path, Sink & sink, - std::function hook, + fun hook, PathFilter & filter = defaultPathFilter, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); diff --git a/src/libutil/include/nix/util/hash.hh b/src/libutil/include/nix/util/hash.hh index 54b7051055f9..cc2ea34b99fe 100644 --- a/src/libutil/include/nix/util/hash.hh +++ b/src/libutil/include/nix/util/hash.hh @@ -165,7 +165,7 @@ Hash hashString( * * (Metadata, such as the executable permission bit, is ignored.) */ -Hash hashFile(HashAlgorithm ha, const Path & path); +Hash hashFile(HashAlgorithm ha, const std::filesystem::path & path); /** * The final hash and the number of bytes digested. diff --git a/src/libutil/include/nix/util/json-utils.hh b/src/libutil/include/nix/util/json-utils.hh index ec513ca25d61..555c982cc6e6 100644 --- a/src/libutil/include/nix/util/json-utils.hh +++ b/src/libutil/include/nix/util/json-utils.hh @@ -75,6 +75,15 @@ Strings getStringList(const nlohmann::json & value); StringMap getStringMap(const nlohmann::json & value); StringSet getStringSet(const nlohmann::json & value); +template +static inline std::optional ptrToOwned(const nlohmann::json * ptr) +{ + if (ptr) + return std::optional{*ptr}; + else + return std::nullopt; +} + } // namespace nix namespace nlohmann { @@ -114,13 +123,4 @@ struct adl_serializer> } }; -template -static inline std::optional ptrToOwned(const json * ptr) -{ - if (ptr) - return std::optional{*ptr}; - else - return std::nullopt; -} - } // namespace nlohmann diff --git a/src/libutil/include/nix/util/logging.hh b/src/libutil/include/nix/util/logging.hh index de2c3f683df6..7793426da89d 100644 --- a/src/libutil/include/nix/util/logging.hh +++ b/src/libutil/include/nix/util/logging.hh @@ -5,6 +5,7 @@ #include "nix/util/configuration.hh" #include "nix/util/file-descriptor.hh" #include "nix/util/finally.hh" +#include "nix/util/fun.hh" #include @@ -56,7 +57,7 @@ struct LoggerSettings : Config expression evaluation errors. )"}; - Setting> jsonLogPath{ + Setting> jsonLogPath{ this, {}, "json-log-path", @@ -115,7 +116,7 @@ public: */ struct Suspension { - Finally> _finalize; + Finally> _finalize; }; Suspension suspend(); diff --git a/src/libutil/include/nix/util/memory-source-accessor.hh b/src/libutil/include/nix/util/memory-source-accessor.hh index fc00f34d9c09..154fc2560d89 100644 --- a/src/libutil/include/nix/util/memory-source-accessor.hh +++ b/src/libutil/include/nix/util/memory-source-accessor.hh @@ -119,7 +119,9 @@ struct MemorySourceAccessor : virtual SourceAccessor return root < other.root; } - std::string readFile(const CanonPath & path) override; + void readFile(const CanonPath & path, Sink & sink, fun sizeCallback) override; + using SourceAccessor::readFile; + bool pathExists(const CanonPath & path) override; std::optional maybeLstat(const CanonPath & path) override; DirEntries readDirectory(const CanonPath & path) override; @@ -155,7 +157,7 @@ struct MemorySink : FileSystemObjectSink void createDirectory(const CanonPath & path) override; - void createRegularFile(const CanonPath & path, std::function) override; + void createRegularFile(const CanonPath & path, fun) override; void createSymlink(const CanonPath & path, const std::string & target) override; }; diff --git a/src/libutil/include/nix/util/meson.build b/src/libutil/include/nix/util/meson.build index 8e0336bbdaab..8682f9c4dc16 100644 --- a/src/libutil/include/nix/util/meson.build +++ b/src/libutil/include/nix/util/meson.build @@ -2,7 +2,12 @@ include_dirs = [ include_directories('../..') ] -headers = files( +config_pub_h = configure_file( + configuration : configdata_pub, + output : 'config.hh', +) + +headers = [ config_pub_h ] + files( 'abstract-setting-to-json.hh', 'alignment.hh', 'ansicolor.hh', @@ -18,12 +23,15 @@ headers = files( 'chunked-vector.hh', 'closure.hh', 'comparator.hh', + 'compression-algo.hh', + 'compression-settings.hh', 'compression.hh', 'compute-levels.hh', 'config-global.hh', 'config-impl.hh', 'configuration.hh', 'current-process.hh', + 'demangle.hh', 'english.hh', 'environment-variables.hh', 'error.hh', @@ -35,11 +43,13 @@ headers = files( 'file-descriptor.hh', 'file-path-impl.hh', 'file-path.hh', + 'file-system-at.hh', 'file-system.hh', 'finally.hh', 'fmt.hh', 'forwarding-source-accessor.hh', 'fs-sink.hh', + 'fun.hh', 'git.hh', 'hash.hh', 'hilite.hh', @@ -52,6 +62,8 @@ headers = files( 'mounted-source-accessor.hh', 'muxable-pipe.hh', 'nar-accessor.hh', + 'nar-cache.hh', + 'nar-listing.hh', 'os-string.hh', 'override-provenance-source-accessor.hh', 'pool.hh', diff --git a/src/libutil/include/nix/util/muxable-pipe.hh b/src/libutil/include/nix/util/muxable-pipe.hh index f15c8e5f82d9..0bc525c4273b 100644 --- a/src/libutil/include/nix/util/muxable-pipe.hh +++ b/src/libutil/include/nix/util/muxable-pipe.hh @@ -2,6 +2,7 @@ ///@file #include "nix/util/file-descriptor.hh" +#include "nix/util/fun.hh" #ifdef _WIN32 # include "nix/util/windows-async-pipe.hh" #endif @@ -10,7 +11,6 @@ # include #else # include -# include "nix/util/windows-error.hh" #endif namespace nix { @@ -75,8 +75,8 @@ struct MuxablePipePollState */ void iterate( std::set & channels, - std::function handleRead, - std::function handleEOF); + fun handleRead, + fun handleEOF); }; } // namespace nix diff --git a/src/libutil/include/nix/util/nar-accessor.hh b/src/libutil/include/nix/util/nar-accessor.hh index 745c79f607b1..e7a6cfb2bc55 100644 --- a/src/libutil/include/nix/util/nar-accessor.hh +++ b/src/libutil/include/nix/util/nar-accessor.hh @@ -1,23 +1,39 @@ #pragma once ///@file -#include "nix/util/memory-source-accessor.hh" +#include "nix/util/fun.hh" +#include "nix/util/nar-listing.hh" #include - -#include +#include namespace nix { struct Source; +/** + * A SourceAccessor for NAR files that provides access to the listing structure. + */ +struct NarAccessor : SourceAccessor +{ + /** + * Get the NAR listing structure. + */ + virtual const NarListing & getListing() const = 0; +}; + /** * Return an object that provides access to the contents of a NAR * file. */ -ref makeNarAccessor(std::string && nar); +ref makeNarAccessor(std::string && nar); -ref makeNarAccessor(Source & source); +/** + * This NAR accessor doesn't actually access a NAR, and thus cannot read + * the contents of files. It just conveys the information which is + * gotten from `listing`. + */ +ref makeNarAccessor(NarListing listing); /** * Create a NAR accessor from a NAR listing (in the format produced by @@ -25,64 +41,18 @@ ref makeNarAccessor(Source & source); * readFile() method of the accessor to get the contents of files * inside the NAR. */ -using GetNarBytes = std::function; +using GetNarBytes = fun; /** * The canonical GetNarBytes function for a seekable Source. */ -GetNarBytes seekableGetNarBytes(const Path & path); +GetNarBytes seekableGetNarBytes(const std::filesystem::path & path); GetNarBytes seekableGetNarBytes(Descriptor fd); -ref makeLazyNarAccessor(const nlohmann::json & listing, GetNarBytes getNarBytes); - /** - * Creates a NAR accessor from a given stream and a GetNarBytes getter. - * @param source Consumed eagerly. References to it are not persisted in the resulting SourceAccessor. + * Creates a NAR accessor from a given listing and a `GetNarBytes` getter. */ -ref makeLazyNarAccessor(Source & source, GetNarBytes getNarBytes); - -struct NarListingRegularFile -{ - /** - * @see `SourceAccessor::Stat::fileSize` - */ - std::optional fileSize; - - /** - * @see `SourceAccessor::Stat::narOffset` - * - * We only set to non-`std::nullopt` if it is also non-zero. - */ - std::optional narOffset; - - auto operator<=>(const NarListingRegularFile &) const = default; -}; - -/** - * Abstract syntax for a "NAR listing". - */ -using NarListing = fso::VariantT; - -/** - * Shallow NAR listing where directory children are not recursively expanded. - * Uses a variant that can hold Regular/Symlink fully, but Directory children - * are just unit types indicating presence without content. - */ -using ShallowNarListing = fso::VariantT; - -/** - * Return a deep structured representation of the contents of a NAR (except file - * contents), recursively listing all children. - */ -NarListing listNarDeep(SourceAccessor & accessor, const CanonPath & path); - -/** - * Return a shallow structured representation of the contents of a NAR (except file - * contents), only listing immediate children without recursing. - */ -ShallowNarListing listNarShallow(SourceAccessor & accessor, const CanonPath & path); - -// All json_avoids_null and JSON_IMPL covered by generic templates in memory-source-accessor.hh +ref makeLazyNarAccessor(NarListing listing, GetNarBytes getNarBytes); } // namespace nix diff --git a/src/libutil/include/nix/util/nar-cache.hh b/src/libutil/include/nix/util/nar-cache.hh new file mode 100644 index 000000000000..9e047a3a6b76 --- /dev/null +++ b/src/libutil/include/nix/util/nar-cache.hh @@ -0,0 +1,48 @@ +#pragma once + +#include "nix/util/fun.hh" +#include "nix/util/hash.hh" +#include "nix/util/nar-accessor.hh" +#include "nix/util/ref.hh" +#include "nix/util/source-accessor.hh" + +#include +#include +#include +#include + +namespace nix { + +/** + * A cache for NAR accessors with optional disk caching. + */ +class NarCache +{ + /** + * Optional directory for caching NARs and listings on disk. + */ + std::optional cacheDir; + + /** + * Map from NAR hash to NAR accessor. + */ + std::map> nars; + +public: + + /** + * Create a NAR cache with an optional cache directory for disk storage. + */ + NarCache(std::optional cacheDir = {}); + + /** + * Lookup or create a NAR accessor, optionally using disk cache. + * + * @param narHash The NAR hash to use as cache key + * @param populate Function called with a Sink to populate the NAR if not cached + * @return The cached or newly created accessor + */ + ref getOrInsert(const Hash & narHash, fun populate); +}; + +} // namespace nix diff --git a/src/libutil/include/nix/util/nar-listing.hh b/src/libutil/include/nix/util/nar-listing.hh new file mode 100644 index 000000000000..54cfad76ae8b --- /dev/null +++ b/src/libutil/include/nix/util/nar-listing.hh @@ -0,0 +1,56 @@ +#pragma once +///@file + +#include "nix/util/memory-source-accessor.hh" + +namespace nix { + +struct NarListingRegularFile +{ + /** + * @see `SourceAccessor::Stat::fileSize` + */ + std::optional fileSize; + + /** + * @see `SourceAccessor::Stat::narOffset` + * + * We only set to non-`std::nullopt` if it is also non-zero. + */ + std::optional narOffset; + + auto operator<=>(const NarListingRegularFile &) const = default; +}; + +/** + * Abstract syntax for a "NAR listing". + */ +using NarListing = fso::VariantT; + +/** + * Shallow NAR listing where directory children are not recursively expanded. + * Uses a variant that can hold Regular/Symlink fully, but Directory children + * are just unit types indicating presence without content. + */ +using ShallowNarListing = fso::VariantT; + +/** + * Parse a NAR from a Source and return its listing structure. + */ +NarListing parseNarListing(Source & source); + +/** + * Return a deep structured representation of the contents of a NAR (except file + * contents), recursively listing all children. + */ +NarListing listNarDeep(SourceAccessor & accessor, const CanonPath & path); + +/** + * Return a shallow structured representation of the contents of a NAR (except file + * contents), only listing immediate children without recursing. + */ +ShallowNarListing listNarShallow(SourceAccessor & accessor, const CanonPath & path); + +// All json_avoids_null and JSON_IMPL covered by generic templates in memory-source-accessor.hh + +} // namespace nix diff --git a/src/libutil/include/nix/util/os-string.hh b/src/libutil/include/nix/util/os-string.hh index f0cbcbaba5b1..7e3bf47d38e8 100644 --- a/src/libutil/include/nix/util/os-string.hh +++ b/src/libutil/include/nix/util/os-string.hh @@ -1,6 +1,8 @@ #pragma once ///@file +#include +#include #include #include #include @@ -36,9 +38,62 @@ using OsString = std::basic_string; */ using OsStringView = std::basic_string_view; -std::string os_string_to_string(OsStringView path); +/** + * `nix::StringMap` counterpart for `OsString` + */ +using OsStringMap = std::map>; + +/** + * `nix::Strings` counterpart for `OsString` + */ +using OsStrings = std::list; + +std::string os_string_to_string(OsStringView s); +std::string os_string_to_string(OsString s); OsString string_to_os_string(std::string_view s); +OsString string_to_os_string(std::string s); + +#ifndef _WIN32 + +inline std::string os_string_to_string(OsStringView s) +{ + return std::string(s); +} + +inline std::string os_string_to_string(OsString s) +{ + return s; +} + +inline OsString string_to_os_string(std::string_view s) +{ + return std::string(s); +} + +inline OsString string_to_os_string(std::string s) +{ + return s; +} + +#endif + +/** + * Convert a list of `std::string` to `OsStrings`. + * Takes ownership to enable moves on Unix. + */ +inline OsStrings toOsStrings(std::list ss) +{ +#ifndef _WIN32 + // On Unix, OsStrings is std::list, so just move + return ss; +#else + OsStrings result; + for (auto & s : ss) + result.push_back(string_to_os_string(std::move(s))); + return result; +#endif +} /** * Create string literals with the native character width of paths diff --git a/src/libutil/include/nix/util/pool.hh b/src/libutil/include/nix/util/pool.hh index 952c29ad5de3..5e1f62e43497 100644 --- a/src/libutil/include/nix/util/pool.hh +++ b/src/libutil/include/nix/util/pool.hh @@ -7,6 +7,7 @@ #include #include +#include "nix/util/fun.hh" #include "nix/util/sync.hh" #include "nix/util/ref.hh" @@ -37,13 +38,13 @@ public: /** * A function that produces new instances of R on demand. */ - typedef std::function()> Factory; + typedef fun()> Factory; /** * A function that checks whether an instance of R is still * usable. Unusable instances are removed from the pool. */ - typedef std::function &)> Validator; + typedef fun &)> Validator; private: diff --git a/src/libutil/include/nix/util/posix-source-accessor.hh b/src/libutil/include/nix/util/posix-source-accessor.hh index 006ba0e7e4d1..f9f0309365e7 100644 --- a/src/libutil/include/nix/util/posix-source-accessor.hh +++ b/src/libutil/include/nix/util/posix-source-accessor.hh @@ -31,7 +31,9 @@ public: */ time_t mtime = 0; - void readFile(const CanonPath & path, Sink & sink, std::function sizeCallback) override; + void readFile(const CanonPath & path, Sink & sink, fun sizeCallback) override; + + using SourceAccessor::readFile; bool pathExists(const CanonPath & path) override; @@ -87,7 +89,7 @@ private: */ void assertNoSymlinks(CanonPath path); - std::optional cachedLstat(const CanonPath & path); + std::optional cachedLstat(const CanonPath & path); std::filesystem::path makeAbsPath(const CanonPath & path); }; diff --git a/src/libutil/include/nix/util/processes.hh b/src/libutil/include/nix/util/processes.hh index 23dee8713624..aed903e0dcc5 100644 --- a/src/libutil/include/nix/util/processes.hh +++ b/src/libutil/include/nix/util/processes.hh @@ -3,9 +3,14 @@ #include "nix/util/types.hh" #include "nix/util/error.hh" +#include "nix/util/fun.hh" #include "nix/util/file-descriptor.hh" +#include "nix/util/file-path.hh" #include "nix/util/logging.hh" #include "nix/util/ansicolor.hh" +#include "nix/util/os-string.hh" + +#include #include #include @@ -35,6 +40,10 @@ class Pid #endif public: Pid(); + Pid(const Pid &) = delete; + Pid(Pid && other) noexcept; + Pid & operator=(const Pid &) = delete; + Pid & operator=(Pid && other) noexcept; #ifndef _WIN32 Pid(pid_t pid); void operator=(pid_t pid); @@ -44,8 +53,8 @@ public: void operator=(AutoCloseFD pid); #endif ~Pid(); - int kill(); - int wait(); + int kill(bool allowInterrupts = true); + int wait(bool allowInterrupts = true); // TODO: Implement for Windows #ifndef _WIN32 @@ -53,6 +62,18 @@ public: void setKillSignal(int signal); pid_t release(); #endif + + friend void swap(Pid & lhs, Pid & rhs) noexcept + { + using std::swap; +#ifndef _WIN32 + swap(lhs.pid, rhs.pid); + swap(lhs.separatePG, rhs.separatePG); + swap(lhs.killSignal, rhs.killSignal); +#else + swap(lhs.pid, rhs.pid); +#endif + } }; #ifndef _WIN32 @@ -80,7 +101,7 @@ struct ProcessOptions }; #ifndef _WIN32 -pid_t startProcess(std::function fun, const ProcessOptions & options = ProcessOptions()); +pid_t startProcess(fun processMain, const ProcessOptions & options = ProcessOptions()); #endif /** @@ -88,23 +109,23 @@ pid_t startProcess(std::function fun, const ProcessOptions & options = P * shell backtick operator). */ std::string runProgram( - Path program, + std::filesystem::path program, bool lookupPath = false, - const Strings & args = Strings(), + const OsStrings & args = OsStrings(), const std::optional & input = {}, bool isInteractive = false); struct RunOptions { - Path program; + std::filesystem::path program; bool lookupPath = true; - Strings args; + OsStrings args; #ifndef _WIN32 std::optional uid; std::optional gid; #endif - std::optional chdir; - std::optional environment; + std::optional chdir; + std::optional environment; std::optional input; Source * standardIn = nullptr; Sink * standardOut = nullptr; @@ -116,14 +137,14 @@ std::pair runProgram(RunOptions && options); void runProgram2(const RunOptions & options); -class ExecError : public Error +class ExecError final : public CloneableError { public: int status; template ExecError(int status, const Args &... args) - : Error(args...) + : CloneableError(args...) , status(status) { } diff --git a/src/libutil/include/nix/util/ref.hh b/src/libutil/include/nix/util/ref.hh index 7ba5349a60b1..2fc79e2167bb 100644 --- a/src/libutil/include/nix/util/ref.hh +++ b/src/libutil/include/nix/util/ref.hh @@ -3,9 +3,46 @@ #include #include +#include +#include + +#include "nix/util/demangle.hh" namespace nix { +/** + * Exception thrown by ref::cast() when dynamic_pointer_cast fails. + * Inherits from std::bad_cast for semantic correctness, but carries a message with type info. + */ +class bad_ref_cast : public std::bad_cast +{ + std::string msg; + +public: + bad_ref_cast(std::string msg) + : msg(std::move(msg)) + { + } + + const char * what() const noexcept override + { + return msg.c_str(); + } +}; + +/** + * Concept for implicit ref covariance: From* must be implicitly convertible to To*. + * + * This allows implicit upcasts (Derived -> Base) but rejects downcasts. + */ +// Design note: This named concept is technically redundant but provides a readable hint +// in error messages. Alternative: static_assert can have custom messages, but doesn't +// participate in SFINAE, so std::is_convertible_v, ref> would +// incorrectly return true (the conversion would exist but fail at instantiation +// rather than being excluded). +template +concept RefImplicitlyUpcastableTo = std::is_convertible_v; + /** * A simple non-nullable reference-counted pointer. Actually a wrapper * around std::shared_ptr that prevents null constructions. @@ -76,7 +113,11 @@ public: template ref cast() const { - return ref(std::dynamic_pointer_cast(p)); + auto casted = std::dynamic_pointer_cast(p); + if (!casted) + throw bad_ref_cast( + "ref<" + demangle(typeid(T).name()) + "> cannot be cast to ref<" + demangle(typeid(T2).name()) + ">"); + return ref(std::move(casted)); } template @@ -85,10 +126,15 @@ public: return std::dynamic_pointer_cast(p); } + /** + * Implicit conversion to ref of base type (covariance). + * Downcasts are rejected; use .cast() (throws bad_ref_cast) or .dynamic_pointer_cast() (returns nullptr) instead. + */ template + requires RefImplicitlyUpcastableTo operator ref() const { - return ref((std::shared_ptr) p); + return ref(p); } bool operator==(const ref & other) const diff --git a/src/libutil/include/nix/util/serialise.hh b/src/libutil/include/nix/util/serialise.hh index 8428eb2b98cf..1362bcb6bb67 100644 --- a/src/libutil/include/nix/util/serialise.hh +++ b/src/libutil/include/nix/util/serialise.hh @@ -4,6 +4,8 @@ #include #include +#include "nix/util/compression-algo.hh" +#include "nix/util/fun.hh" #include "nix/util/types.hh" #include "nix/util/util.hh" #include "nix/util/file-descriptor.hh" @@ -120,6 +122,8 @@ struct BufferedSource : virtual Source size_t read(char * data, size_t len) override; + std::string readLine(bool eofOk = false, char terminator = '\n'); + /** * Return true if the buffer is not empty. */ @@ -159,6 +163,8 @@ struct FdSink : BufferedSink } FdSink(FdSink &&) = default; + FdSink(const FdSink &) = delete; + FdSink & operator=(const FdSink &) = delete; FdSink & operator=(FdSink && s) { @@ -200,8 +206,10 @@ struct FdSource : BufferedSource, RestartableSource } FdSource(FdSource &&) = default; - FdSource & operator=(FdSource && s) = default; + FdSource(const FdSource &) = delete; + FdSource & operator=(const FdSource & s) = delete; + ~FdSource() = default; bool good() override; void restart() override; @@ -284,7 +292,7 @@ struct CompressedSource : RestartableSource { private: std::string compressedData; - std::string compressionMethod; + CompressionAlgo compressionMethod; StringSource stringSource; public: @@ -292,9 +300,9 @@ public: * Compress a RestartableSource using the specified compression method. * * @param source The source data to compress - * @param compressionMethod The compression method to use (e.g., "xz", "br") + * @param compressionMethod The compression method to use */ - CompressedSource(RestartableSource & source, const std::string & compressionMethod); + CompressedSource(RestartableSource & source, CompressionAlgo compressionMethod); size_t read(char * data, size_t len) override { @@ -310,11 +318,6 @@ public: { return compressedData.size(); } - - std::string_view getCompressionMethod() const - { - return compressionMethod; - } }; /** @@ -439,8 +442,8 @@ struct LengthSource : Source */ struct LambdaSink : Sink { - typedef std::function data_t; - typedef std::function cleanup_t; + typedef fun data_t; + typedef fun cleanup_t; data_t dataFun; cleanup_t cleanupFun; @@ -452,6 +455,11 @@ struct LambdaSink : Sink { } + LambdaSink(LambdaSink &&) = delete; + LambdaSink(const LambdaSink &) = delete; + LambdaSink & operator=(LambdaSink &&) = delete; + LambdaSink & operator=(const LambdaSink &) = delete; + ~LambdaSink() { cleanupFun(); @@ -468,7 +476,7 @@ struct LambdaSink : Sink */ struct LambdaSource : Source { - typedef std::function lambda_t; + typedef fun lambda_t; lambda_t lambda; @@ -501,18 +509,39 @@ struct ChainSource : Source size_t read(char * data, size_t len) override; }; -std::unique_ptr sourceToSink(std::function fun); +std::unique_ptr sourceToSink(fun reader); /** * Convert a function that feeds data into a Sink into a Source. The * Source executes the function as a coroutine. */ -std::unique_ptr sinkToSource( - std::function fun, std::function eof = []() { throw EndOfFile("coroutine has finished"); }); +std::unique_ptr +sinkToSource(fun writer, fun eof = []() { throw EndOfFile("coroutine has finished"); }); void writePadding(size_t len, Sink & sink); void writeString(std::string_view s, Sink & sink); +/** + * Write a serialisation of an integer to the sink in little endian order. + * + * Types other than uint64_t (including signed types) get implicitly converted to uint64_t.A + * + * Negative number to unsigned conversion is actually well-defined in C++: + * + * [n4950] 7.3.9 Integral conversions: + * the result is the unique value of the destination type that is congruent to the source integer + * modulo 2^N, where N is the width of the destination type. + * + * [n4950] 6.8.2 Fundamental types: + * An unsigned integer type has the same object representation, value + * representation, and alignment requirements (6.7.6) as the corresponding signed + * integer type. For each value x of a signed integer type, the value of the + * corresponding unsigned integer type congruent to x modulo 2 N has the same value + * of corresponding bits in its value representation. + * This is also known as two's complement representation. + * + * @todo Should we even allow negative values to get serialised? + */ inline Sink & operator<<(Sink & sink, uint64_t n) { unsigned char buf[8]; @@ -628,6 +657,11 @@ struct FramedSource : Source { } + FramedSource(FramedSource &&) = delete; + FramedSource(const FramedSource &) = delete; + FramedSource & operator=(FramedSource &&) = delete; + FramedSource & operator=(const FramedSource &) = delete; + ~FramedSource() { try { @@ -677,14 +711,19 @@ struct FramedSource : Source struct FramedSink : nix::BufferedSink { BufferedSink & to; - std::function checkError; + fun checkError; - FramedSink(BufferedSink & to, std::function && checkError) + FramedSink(BufferedSink & to, fun && checkError) : to(to) , checkError(checkError) { } + FramedSink(FramedSink &&) = delete; + FramedSink(const FramedSink &) = delete; + FramedSink & operator=(FramedSink &&) = delete; + FramedSink & operator=(const FramedSink &) = delete; + ~FramedSink() { try { diff --git a/src/libutil/include/nix/util/signals.hh b/src/libutil/include/nix/util/signals.hh index ff26975ad600..0062e8dcd70f 100644 --- a/src/libutil/include/nix/util/signals.hh +++ b/src/libutil/include/nix/util/signals.hh @@ -3,6 +3,7 @@ #include "nix/util/types.hh" #include "nix/util/error.hh" +#include "nix/util/fun.hh" #include "nix/util/logging.hh" #include @@ -47,7 +48,7 @@ struct InterruptCallback * * @note Does nothing on Windows */ -std::unique_ptr createInterruptCallback(std::function callback); +std::unique_ptr createInterruptCallback(fun callback); /** * A RAII class that causes the current thread to receive SIGUSR1 when diff --git a/src/libutil/include/nix/util/signature/local-keys.hh b/src/libutil/include/nix/util/signature/local-keys.hh index 1c0579ce9ec3..789fb831f0f3 100644 --- a/src/libutil/include/nix/util/signature/local-keys.hh +++ b/src/libutil/include/nix/util/signature/local-keys.hh @@ -1,30 +1,44 @@ #pragma once ///@file -#include "nix/util/types.hh" +#include "nix/util/json-impls.hh" #include namespace nix { /** - * Except where otherwise noted, Nix serializes keys and signatures in - * the form: + * A cryptographic signature along with the name of the key that produced it. * - * ``` - * : - * ``` + * Serialized as `:`. */ -struct BorrowedCryptoValue +struct Signature { - std::string_view name; - std::string_view payload; + std::string keyName; /** - * This splits on the colon, the user can then separated decode the - * Base64 payload separately. + * The raw decoded signature bytes. */ - static BorrowedCryptoValue parse(std::string_view); + std::string sig; + + /** + * Parse a signature in the format `:`. + */ + static Signature parse(std::string_view); + + /** + * Parse multiple signatures from a container of strings. + * + * Each string must be in the format `:`. + */ + template + static std::set parseMany(const Container & sigStrs); + + std::string to_string() const; + + static Strings toStrings(const std::set & sigs); + + auto operator<=>(const Signature &) const = default; }; struct Key @@ -61,7 +75,7 @@ struct SecretKey : Key /** * Return a detached signature of the given string. */ - std::string signDetached(std::string_view s) const; + Signature signDetached(std::string_view s) const; PublicKey toPublicKey() const; @@ -82,16 +96,15 @@ struct PublicKey : Key * @return true iff `sig` and this key's names match, and `sig` is a * correct signature over `data` using the given public key. */ - bool verifyDetached(std::string_view data, std::string_view sigs) const; + bool verifyDetached(std::string_view data, const Signature & sig) const; /** * @return true iff `sig` is a correct signature over `data` using the * given public key. * - * @param just the Base64 signature itself, not a colon-separated pair of a - * public key name and signature. + * @param sig the raw signature bytes (not Base64 encoded). */ - bool verifyDetachedAnon(std::string_view data, std::string_view sigs) const; + bool verifyDetachedAnon(std::string_view data, const Signature & sig) const; private: PublicKey(std::string_view name, std::string && key) @@ -110,6 +123,8 @@ typedef std::map PublicKeys; * @return true iff ‘sig’ is a correct signature over ‘data’ using one * of the given public keys. */ -bool verifyDetached(std::string_view data, std::string_view sig, const PublicKeys & publicKeys); +bool verifyDetached(std::string_view data, const Signature & sig, const PublicKeys & publicKeys); } // namespace nix + +JSON_IMPL(nix::Signature) diff --git a/src/libutil/include/nix/util/signature/signer.hh b/src/libutil/include/nix/util/signature/signer.hh index 074c0c6e5968..d03dbe975466 100644 --- a/src/libutil/include/nix/util/signature/signer.hh +++ b/src/libutil/include/nix/util/signature/signer.hh @@ -29,7 +29,7 @@ struct Signer * signature](https://en.wikipedia.org/wiki/Detached_signature), * i.e. just the signature itself without a copy of the signed data. */ - virtual std::string signDetached(std::string_view data) const = 0; + virtual Signature signDetached(std::string_view data) const = 0; /** * View the public key associated with this `Signer`. @@ -48,7 +48,7 @@ struct LocalSigner : Signer { LocalSigner(SecretKey && privateKey); - std::string signDetached(std::string_view s) const override; + Signature signDetached(std::string_view s) const override; const PublicKey & getPublicKey() override; diff --git a/src/libutil/include/nix/util/source-accessor.hh b/src/libutil/include/nix/util/source-accessor.hh index b1948438f2e1..cf0a872d7f5f 100644 --- a/src/libutil/include/nix/util/source-accessor.hh +++ b/src/libutil/include/nix/util/source-accessor.hh @@ -3,6 +3,7 @@ #include #include "nix/util/canon-path.hh" +#include "nix/util/fun.hh" #include "nix/util/hash.hh" #include "nix/util/ref.hh" @@ -31,7 +32,11 @@ enum class SymlinkResolution { Full, }; -MakeError(FileNotFound, Error); +MakeError(SourceAccessorError, Error); +MakeError(FileNotFound, SourceAccessorError); +MakeError(NotASymlink, SourceAccessorError); +MakeError(NotADirectory, SourceAccessorError); +MakeError(NotARegularFile, SourceAccessorError); /** * A read-only filesystem abstraction. This is used by the Nix @@ -59,7 +64,7 @@ struct SourceAccessor : std::enable_shared_from_this * targets of symlinks should only occasionally be done, and only * with care. */ - virtual std::string readFile(const CanonPath & path); + std::string readFile(const CanonPath & path); /** * Write the contents of a file as a sink. `sizeCallback` must be @@ -72,8 +77,7 @@ struct SourceAccessor : std::enable_shared_from_this * @note subclasses of `SourceAccessor` need to implement at least * one of the `readFile()` variants. */ - virtual void - readFile(const CanonPath & path, Sink & sink, std::function sizeCallback = [](uint64_t size) {}); + virtual void readFile(const CanonPath & path, Sink & sink, fun sizeCallback = [](uint64_t size) {}); virtual bool pathExists(const CanonPath & path); @@ -235,6 +239,24 @@ ref makeEmptySourceAccessor(); */ MakeError(RestrictedPathError, Error); +struct SymlinkNotAllowed final : public CloneableError +{ + CanonPath path; + + SymlinkNotAllowed(CanonPath path) + : CloneableError("relative path '%s' points to a symlink, which is not allowed", path.rel()) + , path(std::move(path)) + { + } + + template + SymlinkNotAllowed(CanonPath path, const std::string & fs, Args &&... args) + : CloneableError(fs, std::forward(args)...) + , path(std::move(path)) + { + } +}; + /** * Return an accessor for the root filesystem. */ @@ -246,7 +268,7 @@ ref getFSSourceAccessor(); * elements, and that absolute symlinks are resolved relative to * `root`. */ -ref makeFSSourceAccessor(std::filesystem::path root); +ref makeFSSourceAccessor(std::filesystem::path root, bool trackLastModified = false); /** * Construct an accessor that presents a "union" view of a vector of diff --git a/src/libutil/include/nix/util/source-path.hh b/src/libutil/include/nix/util/source-path.hh index 2810d8c56d51..677457f8ed48 100644 --- a/src/libutil/include/nix/util/source-path.hh +++ b/src/libutil/include/nix/util/source-path.hh @@ -43,7 +43,7 @@ struct SourcePath */ std::string readFile() const; - void readFile(Sink & sink, std::function sizeCallback = [](uint64_t size) {}) const + void readFile(Sink & sink, fun sizeCallback = [](uint64_t size) {}) const { return accessor->readFile(path, sink, sizeCallback); } diff --git a/src/libutil/include/nix/util/sync.hh b/src/libutil/include/nix/util/sync.hh index 3a41d1bd8081..6f3298324ff8 100644 --- a/src/libutil/include/nix/util/sync.hh +++ b/src/libutil/include/nix/util/sync.hh @@ -27,7 +27,7 @@ namespace nix { * Here, "data" is automatically unlocked when "data_" goes out of * scope. */ -template +template class SyncBase { private: @@ -69,39 +69,42 @@ public: { } public: - Lock(Lock && l) - : s(l.s) - { - unreachable(); - } - + Lock(Lock && l) = delete; Lock(const Lock & l) = delete; + Lock & operator=(Lock && l) = delete; + Lock & operator=(const Lock & l) = delete; ~Lock() {} - void wait(std::condition_variable & cv) + void wait(CV & cv) { assert(s); cv.wait(lk); } + template + void wait(CV & cv, Predicate pred) + { + assert(s); + cv.wait(lk, std::move(pred)); + } + template - std::cv_status wait_for(std::condition_variable & cv, const std::chrono::duration & duration) + std::cv_status wait_for(CV & cv, const std::chrono::duration & duration) { assert(s); return cv.wait_for(lk, duration); } template - bool wait_for(std::condition_variable & cv, const std::chrono::duration & duration, Predicate pred) + bool wait_for(CV & cv, const std::chrono::duration & duration, Predicate pred) { assert(s); return cv.wait_for(lk, duration, pred); } template - std::cv_status - wait_until(std::condition_variable & cv, const std::chrono::time_point & duration) + std::cv_status wait_until(CV & cv, const std::chrono::time_point & duration) { assert(s); return cv.wait_until(lk, duration); @@ -110,6 +113,8 @@ public: struct WriteLock : Lock { + using Lock::Lock; + T * operator->() { return &WriteLock::s->data; @@ -131,6 +136,8 @@ public: struct ReadLock : Lock { + using Lock::Lock; + const T * operator->() { return &ReadLock::s->data; @@ -153,10 +160,15 @@ public: }; template -using Sync = SyncBase, std::unique_lock>; +using Sync = + SyncBase, std::unique_lock, std::condition_variable>; template -using SharedSync = - SyncBase, std::shared_lock>; +using SharedSync = SyncBase< + T, + std::shared_mutex, + std::unique_lock, + std::shared_lock, + std::condition_variable>; } // namespace nix diff --git a/src/libutil/include/nix/util/thread-pool.hh b/src/libutil/include/nix/util/thread-pool.hh index 63f1141f6a53..a6bb7cf02d1c 100644 --- a/src/libutil/include/nix/util/thread-pool.hh +++ b/src/libutil/include/nix/util/thread-pool.hh @@ -2,6 +2,7 @@ ///@file #include "nix/util/error.hh" +#include "nix/util/fun.hh" #include "nix/util/sync.hh" #include @@ -31,7 +32,7 @@ public: * * \todo use std::packaged_task? */ - typedef std::function work_t; + typedef fun work_t; /** * Enqueue a function to be executed by the thread pool. @@ -88,8 +89,8 @@ private: template void processGraph( const std::set & nodes, - std::function(const T &)> getEdges, - std::function processNode, + fun(const T &)> getEdges, + fun processNode, bool discoverNodes = false, size_t maxThreads = 0) { diff --git a/src/libutil/include/nix/util/topo-sort.hh b/src/libutil/include/nix/util/topo-sort.hh index fb918117bdac..6218b66a5023 100644 --- a/src/libutil/include/nix/util/topo-sort.hh +++ b/src/libutil/include/nix/util/topo-sort.hh @@ -2,6 +2,7 @@ ///@file #include "nix/util/error.hh" +#include "nix/util/fun.hh" #include #include @@ -24,9 +25,8 @@ TopoSortResult topoSort(std::set items, F && getChildren) std::vector sorted; decltype(items) visited, parents; - std::function>(const T & path, const T * parent)> dfsVisit; - - dfsVisit = [&](const T & path, const T * parent) -> std::optional> { + fun>(const T & path, const T * parent)> dfsVisit = + [&](const T & path, const T * parent) -> std::optional> { if (parents.count(path)) { return Cycle{path, *parent}; } diff --git a/src/libutil/include/nix/util/types.hh b/src/libutil/include/nix/util/types.hh index f8c6c0979585..9bc1428c3102 100644 --- a/src/libutil/include/nix/util/types.hh +++ b/src/libutil/include/nix/util/types.hh @@ -44,20 +44,6 @@ using StringPairs = StringMap; */ using StringSet = std::set>; -/** - * Paths are just strings. - */ -typedef std::string Path; -typedef std::string_view PathView; -typedef std::list Paths; - -/** - * Alias to an ordered set of `Path`s. Uses transparent comparator. - * - * @see StringSet - */ -using PathSet = std::set>; - typedef std::vector> Headers; /** diff --git a/src/libutil/include/nix/util/unix-domain-socket.hh b/src/libutil/include/nix/util/unix-domain-socket.hh index 99fd331ce385..ca706a32c0c8 100644 --- a/src/libutil/include/nix/util/unix-domain-socket.hh +++ b/src/libutil/include/nix/util/unix-domain-socket.hh @@ -19,12 +19,12 @@ AutoCloseFD createUnixDomainSocket(); /** * Create a Unix domain socket in listen mode. */ -AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode); +AutoCloseFD createUnixDomainSocket(const std::filesystem::path & path, mode_t mode); /** * Bind a Unix domain socket to a path. */ -void bind(Socket fd, const std::string & path); +void bind(Socket fd, const std::filesystem::path & path); /** * Connect to a Unix domain socket. diff --git a/src/libutil/include/nix/util/url.hh b/src/libutil/include/nix/util/url.hh index 5f9eab4e9de9..5099b8bdb240 100644 --- a/src/libutil/include/nix/util/url.hh +++ b/src/libutil/include/nix/util/url.hh @@ -1,6 +1,7 @@ #pragma once ///@file +#include #include #include @@ -263,14 +264,10 @@ std::string percentDecode(std::string_view in); std::string percentEncode(std::string_view s, std::string_view keep = ""); /** - * Get the path part of the URL as an absolute or relative Path. - * - * @throws if any path component contains an slash (which would have - * been escaped `%2F` in the rendered URL). This is because OS file - * paths have no escape sequences --- file names cannot contain a - * `/`. + * Render URL path segments to a string by joining with `/`. + * Does not percent-encode the segments. */ -Path renderUrlPathEnsureLegal(const std::vector & urlPath); +std::string renderUrlPathNoPctEncoding(std::span urlPath); /** * Percent encode path. `%2F` for "interior slashes" is the most @@ -288,7 +285,7 @@ std::string encodeQuery(const StringMap & query); /** * Parse a URL into a ParsedURL. * - * @parm lenient Also allow some long-supported Nix URIs that are not quite compliant with RFC3986. + * @param lenient Also allow some long-supported Nix URIs that are not quite compliant with RFC3986. * Here are the deviations: * - Fragments can contain unescaped (not URL encoded) '^', '"' or space literals. * - Queries may contain unescaped '"' or spaces. @@ -352,6 +349,22 @@ ParsedURL fixGitURL(std::string url); */ bool isValidSchemeName(std::string_view scheme); +/** + * Convert a filesystem path to a URL path vector. + * + * On Windows, converts backslashes to forward slashes and prepends a `/` + * before the drive letter (e.g., `C:\foo\bar` becomes `/C:/foo/bar`). + */ +std::vector pathToUrlPath(const std::filesystem::path & path); + +/** + * Convert a URL path vector to a native filesystem path. + * + * On Windows, strips the leading `/` before the drive letter and converts + * to native format (e.g., `/C:/foo/bar` becomes `C:\foo\bar`). + */ +std::filesystem::path urlPathToPath(std::span urlPath); + /** * Either a ParsedURL or a verbatim string. This is necessary because in certain cases URI must be passed * verbatim (e.g. in builtin fetchers), since those are specified by the user. diff --git a/src/libutil/include/nix/util/users.hh b/src/libutil/include/nix/util/users.hh index 7a556fa8b7b2..dec2c4315e67 100644 --- a/src/libutil/include/nix/util/users.hh +++ b/src/libutil/include/nix/util/users.hh @@ -2,13 +2,12 @@ ///@file #include - -#include "nix/util/types.hh" - #ifndef _WIN32 # include #endif +#include "nix/util/types.hh" + namespace nix { std::string getUserName(); diff --git a/src/libutil/include/nix/util/util.hh b/src/libutil/include/nix/util/util.hh index ac14d47a58fc..5f147716c0ad 100644 --- a/src/libutil/include/nix/util/util.hh +++ b/src/libutil/include/nix/util/util.hh @@ -10,6 +10,7 @@ #include #include #include +#include #include #include @@ -175,8 +176,25 @@ template T readLittleEndian(unsigned char * p) { T x = 0; - for (size_t i = 0; i < sizeof(x); ++i, ++p) { - x |= ((T) *p) << (i * 8); + /* Byte types such as char/unsigned char/std::byte are a bit special because + they are allowed to alias anything else. Thus a raw loop iterating + over the bytes here would be quite inefficient and iterate byte-by-byte + (the compiler cannot optimise anything because the pointer might alias + something). Use a memcpy + byteswap here as needed. */ + std::memcpy(&x, p, sizeof(T)); + /* Don't need to do anything if we are not on a big endian machine. */ + if constexpr (std::endian::native != std::endian::little) { + if constexpr (std::is_same_v) { + x = __builtin_bswap64(x); + } else if constexpr (std::is_same_v) { + x = __builtin_bswap32(x); + } else if constexpr (std::is_same_v) { + x = __builtin_bswap16(x); + } else { + /* Signed types don't make their way here. Though it would be fine + since C++20 mandates 2's complement representation. */ + static_assert(false); + } } return x; } @@ -395,6 +413,11 @@ struct MaintainCount counter += delta; } + MaintainCount(MaintainCount &&) = delete; + MaintainCount(const MaintainCount &) = delete; + MaintainCount & operator=(MaintainCount &&) = delete; + MaintainCount & operator=(const MaintainCount &) = delete; + ~MaintainCount() { counter -= delta; diff --git a/src/libutil/include/nix/util/xml-writer.hh b/src/libutil/include/nix/util/xml-writer.hh index 8d084ad11351..e0561f0ab7c9 100644 --- a/src/libutil/include/nix/util/xml-writer.hh +++ b/src/libutil/include/nix/util/xml-writer.hh @@ -50,6 +50,11 @@ public: writer.openElement(name, attrs); } + XMLOpenElement(XMLOpenElement &&) = delete; + XMLOpenElement(const XMLOpenElement &) = delete; + XMLOpenElement & operator=(XMLOpenElement &&) = delete; + XMLOpenElement & operator=(const XMLOpenElement &) = delete; + ~XMLOpenElement() { writer.closeElement(); diff --git a/src/libutil/linux/cgroup.cc b/src/libutil/linux/cgroup.cc index 802b56336d1e..c79a6dd06c73 100644 --- a/src/libutil/linux/cgroup.cc +++ b/src/libutil/linux/cgroup.cc @@ -15,9 +15,9 @@ namespace nix { -std::optional getCgroupFS() +std::optional getCgroupFS() { - static auto res = [&]() -> std::optional { + static auto res = [&]() -> std::optional { auto fp = fopen("/proc/mounts", "r"); if (!fp) return std::nullopt; @@ -32,7 +32,7 @@ std::optional getCgroupFS() } // FIXME: obsolete, check for cgroup2 -StringMap getCgroups(const Path & cgroupFile) +StringMap getCgroups(const std::filesystem::path & cgroupFile) { StringMap cgroups; @@ -40,7 +40,7 @@ StringMap getCgroups(const Path & cgroupFile) static std::regex regex("([0-9]+):([^:]*):(.*)"); std::smatch match; if (!std::regex_match(line, match, regex)) - throw Error("invalid line '%s' in '%s'", line, cgroupFile); + throw Error("invalid line '%s' in %s", line, PathFmt(cgroupFile)); std::string name = hasPrefix(std::string(match[2]), "name=") ? std::string(match[2], 5) : match[2]; cgroups.insert_or_assign(name, match[3]); @@ -84,7 +84,7 @@ static CgroupStats destroyCgroup(const std::filesystem::path & cgroup, bool retu auto procsFile = cgroup / "cgroup.procs"; if (!pathExists(procsFile)) - throw Error("'%s' is not a cgroup", cgroup); + throw Error("%s is not a cgroup", PathFmt(cgroup)); /* Use the fast way to kill every process in a cgroup, if available. */ @@ -112,7 +112,7 @@ static CgroupStats destroyCgroup(const std::filesystem::path & cgroup, bool retu break; if (round > 20) - throw Error("cannot kill cgroup '%s'", cgroup); + throw Error("cannot kill cgroup %s", PathFmt(cgroup)); for (auto & pid_s : pids) { pid_t pid; @@ -130,12 +130,12 @@ static CgroupStats destroyCgroup(const std::filesystem::path & cgroup, bool retu } // FIXME: pid wraparound if (kill(pid, SIGKILL) == -1 && errno != ESRCH) - throw SysError("killing member %d of cgroup '%s'", pid, cgroup); + throw SysError("killing member %d of cgroup %s", pid, PathFmt(cgroup)); } auto sleep = std::chrono::milliseconds((int) std::pow(2.0, std::min(round, 10))); if (sleep.count() > 100) - printError("waiting for %d ms for cgroup '%s' to become empty", sleep.count(), cgroup); + printError("waiting for %d ms for cgroup %s to become empty", sleep.count(), PathFmt(cgroup)); std::this_thread::sleep_for(sleep); round++; } @@ -145,17 +145,17 @@ static CgroupStats destroyCgroup(const std::filesystem::path & cgroup, bool retu stats = getCgroupStats(cgroup); if (rmdir(cgroup.c_str()) == -1) - throw SysError("deleting cgroup %s", cgroup); + throw SysError("deleting cgroup %s", PathFmt(cgroup)); return stats; } -CgroupStats destroyCgroup(const Path & cgroup) +CgroupStats destroyCgroup(const std::filesystem::path & cgroup) { return destroyCgroup(cgroup, true); } -std::string getCurrentCgroup() +CanonPath getCurrentCgroup() { auto cgroupFS = getCgroupFS(); if (!cgroupFS) @@ -165,12 +165,12 @@ std::string getCurrentCgroup() auto ourCgroup = ourCgroups[""]; if (ourCgroup == "") throw Error("cannot determine cgroup name from /proc/self/cgroup"); - return ourCgroup; + return CanonPath{ourCgroup}; } -std::string getRootCgroup() +CanonPath getRootCgroup() { - static std::string rootCgroup = getCurrentCgroup(); + static auto rootCgroup = getCurrentCgroup(); return rootCgroup; } diff --git a/src/libutil/linux/include/nix/util/cgroup.hh b/src/libutil/linux/include/nix/util/cgroup.hh index ad777347670c..fb437215d92b 100644 --- a/src/libutil/linux/include/nix/util/cgroup.hh +++ b/src/libutil/linux/include/nix/util/cgroup.hh @@ -6,12 +6,13 @@ #include #include "nix/util/types.hh" +#include "nix/util/canon-path.hh" namespace nix { -std::optional getCgroupFS(); +std::optional getCgroupFS(); -StringMap getCgroups(const Path & cgroupFile); +StringMap getCgroups(const std::filesystem::path & cgroupFile); struct CgroupStats { @@ -29,16 +30,16 @@ CgroupStats getCgroupStats(const std::filesystem::path & cgroup); * been killed. Also return statistics from the cgroup just before * destruction. */ -CgroupStats destroyCgroup(const Path & cgroup); +CgroupStats destroyCgroup(const std::filesystem::path & cgroup); -std::string getCurrentCgroup(); +CanonPath getCurrentCgroup(); /** * Get the cgroup that should be used as the parent when creating new * sub-cgroups. The first time this is called, the current cgroup will be * returned, and then all subsequent calls will return the original cgroup. */ -std::string getRootCgroup(); +CanonPath getRootCgroup(); /** * Get the PIDs of all processes in the given cgroup. diff --git a/src/libutil/linux/linux-namespaces.cc b/src/libutil/linux/linux-namespaces.cc index b7787cb6fc8c..83a800e4729e 100644 --- a/src/libutil/linux/linux-namespaces.cc +++ b/src/libutil/linux/linux-namespaces.cc @@ -22,13 +22,13 @@ bool userNamespacesSupported() return false; } - Path maxUserNamespaces = "/proc/sys/user/max_user_namespaces"; + std::filesystem::path maxUserNamespaces = "/proc/sys/user/max_user_namespaces"; if (!pathExists(maxUserNamespaces) || trim(readFile(maxUserNamespaces)) == "0") { debug("user namespaces appear to be disabled; check '/proc/sys/user/max_user_namespaces'"); return false; } - Path procSysKernelUnprivilegedUsernsClone = "/proc/sys/kernel/unprivileged_userns_clone"; + std::filesystem::path procSysKernelUnprivilegedUsernsClone = "/proc/sys/kernel/unprivileged_userns_clone"; if (pathExists(procSysKernelUnprivilegedUsernsClone) && trim(readFile(procSysKernelUnprivilegedUsernsClone)) == "0") { debug("user namespaces appear to be disabled; check '/proc/sys/kernel/unprivileged_userns_clone'"); diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index 842381acf66e..b7b43e5ff60f 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -150,18 +150,23 @@ class SimpleLogger : public Logger Verbosity verbosity = lvlInfo; -void writeToStderr(std::string_view s) +static void writeFullLogging(Descriptor fd, std::string_view s) { try { - writeFull(getStandardError(), s, false); + writeFull(fd, s, false); } catch (SystemError & e) { - /* Ignore failing writes to stderr. We need to ignore write - errors to ensure that cleanup code that logs to stderr runs - to completion if the other side of stderr has been closed - unexpectedly. */ + /* Ignore failing logging writes. We need to ignore write + errors to ensure that cleanup code that writes logs runs + to completion if the other side of the logging fd has + been closed unexpectedly. */ } } +void writeToStderr(std::string_view s) +{ + writeFullLogging(getStandardError(), s); +} + std::unique_ptr makeSimpleLogger(bool printBuildLogs) { return std::make_unique(printBuildLogs); @@ -245,15 +250,15 @@ struct JSONLogger : Logger void write(const nlohmann::json & json) { - auto line = - (includeNixPrefix ? "@nix " : "") + json.dump(-1, ' ', false, nlohmann::json::error_handler_t::replace); + auto line = (includeNixPrefix ? "@nix " : "") + + json.dump(-1, ' ', false, nlohmann::json::error_handler_t::replace) + "\n"; /* Acquire a lock to prevent log messages from clobbering each other. */ try { auto state(_state.lock()); if (state->enabled) - writeLine(fd, line); + writeFullLogging(fd, line); } catch (...) { bool enabled = false; std::swap(_state.lock()->enabled, enabled); @@ -370,7 +375,7 @@ std::unique_ptr makeJSONLogger(const std::filesystem::path & path, bool ? connect(path) : toDescriptor(open(path.string().c_str(), O_CREAT | O_APPEND | O_WRONLY, 0644)); if (!fd) - throw SysError("opening log file %1%", path); + throw SysError("opening log file %1%", PathFmt(path)); return std::make_unique(std::move(fd), includeNixPrefix); } diff --git a/src/libutil/memory-source-accessor.cc b/src/libutil/memory-source-accessor.cc index ec21c846ddea..06d81ba5583f 100644 --- a/src/libutil/memory-source-accessor.cc +++ b/src/libutil/memory-source-accessor.cc @@ -53,15 +53,17 @@ MemorySourceAccessor::File * MemorySourceAccessor::open(const CanonPath & path, return cur; } -std::string MemorySourceAccessor::readFile(const CanonPath & path) +void MemorySourceAccessor::readFile(const CanonPath & path, Sink & sink, fun sizeCallback) { auto * f = open(path, std::nullopt); if (!f) - throw Error("file '%s' does not exist", path); - if (auto * r = std::get_if(&f->raw)) - return r->contents; - else - throw Error("file '%s' is not a regular file", path); + throw FileNotFound("file '%s' does not exist", showPath(path)); + if (auto * r = std::get_if(&f->raw)) { + sizeCallback(r->contents.size()); + StringSource source{r->contents}; + source.drainInto(sink); + } else + throw NotARegularFile("file '%s' is not a regular file", showPath(path)); } bool MemorySourceAccessor::pathExists(const CanonPath & path) @@ -105,14 +107,14 @@ MemorySourceAccessor::DirEntries MemorySourceAccessor::readDirectory(const Canon { auto * f = open(path, std::nullopt); if (!f) - throw Error("file '%s' does not exist", path); + throw FileNotFound("file '%s' does not exist", showPath(path)); if (auto * d = std::get_if(&f->raw)) { DirEntries res; for (auto & [name, file] : d->entries) res.insert_or_assign(name, file.lstat().type); return res; } else - throw Error("file '%s' is not a directory", path); + throw NotADirectory("file '%s' is not a directory", showPath(path)); return {}; } @@ -120,11 +122,11 @@ std::string MemorySourceAccessor::readLink(const CanonPath & path) { auto * f = open(path, std::nullopt); if (!f) - throw Error("file '%s' does not exist", path); + throw FileNotFound("file '%s' does not exist", showPath(path)); if (auto * s = std::get_if(&f->raw)) return s->target; else - throw Error("file '%s' is not a symbolic link", path); + throw NotASymlink("file '%s' is not a symbolic link", showPath(path)); } SourcePath MemorySourceAccessor::addFile(CanonPath path, std::string && contents) @@ -135,11 +137,11 @@ SourcePath MemorySourceAccessor::addFile(CanonPath path, std::string && contents auto * f = open(path, File{File::Regular{}}); if (!f) - throw Error("file '%s' cannot be made because some parent file is not a directory", path); + throw Error("file '%s' cannot be created because some parent file is not a directory", showPath(path)); if (auto * r = std::get_if(&f->raw)) r->contents = std::move(contents); else - throw Error("file '%s' is not a regular file", path); + throw NotARegularFile("file '%s' is not a regular file", showPath(path)); return SourcePath{ref(shared_from_this()), path}; } @@ -150,10 +152,10 @@ void MemorySink::createDirectory(const CanonPath & path) { auto * f = dst.open(path, File{File::Directory{}}); if (!f) - throw Error("file '%s' cannot be made because some parent file is not a directory", path); + throw Error("directory '%s' cannot be created because some parent file is not a directory", dst.showPath(path)); if (!std::holds_alternative(f->raw)) - throw Error("file '%s' is not a directory", path); + throw NotADirectory("file '%s' is not a directory", dst.showPath(path)); }; struct CreateMemoryRegularFile : CreateRegularFileSink @@ -170,16 +172,16 @@ struct CreateMemoryRegularFile : CreateRegularFileSink void preallocateContents(uint64_t size) override; }; -void MemorySink::createRegularFile(const CanonPath & path, std::function func) +void MemorySink::createRegularFile(const CanonPath & path, fun func) { auto * f = dst.open(path, File{File::Regular{}}); if (!f) - throw Error("file '%s' cannot be made because some parent file is not a directory", path); + throw Error("file '%s' cannot be created because some parent file is not a directory", path); if (auto * rp = std::get_if(&f->raw)) { CreateMemoryRegularFile crf{*rp}; func(crf); } else - throw Error("file '%s' is not a regular file", path); + throw NotARegularFile("file '%s' is not a regular file", dst.showPath(path)); } void CreateMemoryRegularFile::isExecutable() @@ -201,11 +203,11 @@ void MemorySink::createSymlink(const CanonPath & path, const std::string & targe { auto * f = dst.open(path, File{File::Symlink{}}); if (!f) - throw Error("file '%s' cannot be made because some parent file is not a directory", path); + throw Error("symlink '%s' cannot be created because some parent file is not a directory", dst.showPath(path)); if (auto * s = std::get_if(&f->raw)) s->target = target; else - throw Error("file '%s' is not a symbolic link", path); + throw NotASymlink("file '%s' is not a symbolic link", dst.showPath(path)); } ref makeEmptySourceAccessor() diff --git a/src/libutil/meson.build b/src/libutil/meson.build index 283f4b96f55c..56f8baa41ad9 100644 --- a/src/libutil/meson.build +++ b/src/libutil/meson.build @@ -16,7 +16,8 @@ cxx = meson.get_compiler('cpp') subdir('nix-meson-build-support/deps-lists') -configdata = configuration_data() +configdata_pub = configuration_data() +configdata_priv = configuration_data() deps_private_maybe_subproject = [] deps_public_maybe_subproject = [] @@ -34,9 +35,15 @@ check_funcs = [ foreach funcspec : check_funcs define_name = 'HAVE_' + funcspec[0].underscorify().to_upper() define_value = cxx.has_function(funcspec[0]).to_int() - configdata.set(define_name, define_value, description : funcspec[1]) + configdata_priv.set(define_name, define_value, description : funcspec[1]) endforeach +configdata_pub.set( + 'NIX_UBSAN_ENABLED', + ('undefined' in get_option('b_sanitize')).to_int(), + description : 'Whether nix has been built with UBSan enabled', +) + subdir('nix-meson-build-support/libatomic') if host_machine.system() == 'windows' @@ -104,7 +111,7 @@ cpuid = dependency( version : '>= 0.7.0', required : cpuid_required, ) -configdata.set('HAVE_LIBCPUID', cpuid.found().to_int()) +configdata_priv.set('HAVE_LIBCPUID', cpuid.found().to_int()) deps_private += cpuid nlohmann_json = dependency('nlohmann_json', version : '>= 3.9') @@ -113,7 +120,7 @@ deps_public += nlohmann_json cxx = meson.get_compiler('cpp') config_priv_h = configure_file( - configuration : configdata, + configuration : configdata_priv, output : 'util-config-private.hh', ) @@ -125,6 +132,8 @@ sources = [ config_priv_h ] + files( 'base-n.cc', 'base-nix-32.cc', 'canon-path.cc', + 'compression-algo.cc', + 'compression-settings.cc', 'compression.cc', 'compute-levels.cc', 'config-global.cc', @@ -149,9 +158,12 @@ sources = [ config_priv_h ] + files( 'memory-source-accessor/json.cc', 'mounted-source-accessor.cc', 'nar-accessor.cc', + 'nar-cache.cc', + 'nar-listing.cc', 'pos-table.cc', 'position.cc', 'posix-source-accessor.cc', + 'processes.cc', 'provenance.cc', 'serialise.cc', 'signature/local-keys.cc', diff --git a/src/libutil/mounted-source-accessor.cc b/src/libutil/mounted-source-accessor.cc index 22e5acf70b45..ca6d49275b3e 100644 --- a/src/libutil/mounted-source-accessor.cc +++ b/src/libutil/mounted-source-accessor.cc @@ -21,10 +21,10 @@ struct MountedSourceAccessorImpl : MountedSourceAccessor // FIXME: return dummy parent directories automatically? } - std::string readFile(const CanonPath & path) override + void readFile(const CanonPath & path, Sink & sink, fun sizeCallback) override { auto [accessor, subpath] = resolve(path); - return accessor->readFile(subpath); + return accessor->readFile(subpath, sink, sizeCallback); } Stat lstat(const CanonPath & path) override diff --git a/src/libutil/nar-accessor.cc b/src/libutil/nar-accessor.cc index 5644ca4081ba..fd7cc0acfa48 100644 --- a/src/libutil/nar-accessor.cc +++ b/src/libutil/nar-accessor.cc @@ -1,196 +1,68 @@ #include "nix/util/nar-accessor.hh" -#include "nix/util/archive.hh" - -#include -#include - -#include +#include "nix/util/file-descriptor.hh" +#include "nix/util/error.hh" +#include "nix/util/signals.hh" namespace nix { -struct NarMember +struct NarAccessorImpl : NarAccessor { - SourceAccessor::Stat stat; - - std::string target; - - /* If this is a directory, all the children of the directory. */ - std::map children; -}; - -struct NarMemberConstructor : CreateRegularFileSink -{ -private: - - NarMember & narMember; - - uint64_t & pos; + NarListing root; -public: + std::function getNarBytes; - NarMemberConstructor(NarMember & nm, uint64_t & pos) - : narMember(nm) - , pos(pos) + const NarListing & getListing() const override { + return root; } - void isExecutable() override + NarAccessorImpl(std::string && nar) + : root{[&nar]() { + StringSource source(nar); + return parseNarListing(source); + }()} + , getNarBytes{[nar = std::move(nar)](uint64_t offset, uint64_t length, Sink & sink) { + if (offset > nar.size() || length > nar.size() - offset) + throw Error( + "reading invalid NAR bytes range: requested %1% bytes at offset %2%, but NAR has size %3%", + length, + offset, + nar.size()); + StringSource source(std::string_view(nar.data() + offset, length)); + source.drainInto(sink); + }} { - narMember.stat.isExecutable = true; } - void preallocateContents(uint64_t size) override + NarAccessorImpl(NarListing && listing) + : root{std::move(listing)} { - narMember.stat.fileSize = size; - narMember.stat.narOffset = pos; } - void operator()(std::string_view data) override {} -}; - -struct NarAccessor : public SourceAccessor -{ - std::optional nar; - - GetNarBytes getNarBytes; - - NarMember root; - - struct NarIndexer : FileSystemObjectSink, Source - { - NarAccessor & acc; - Source & source; - - std::stack parents; - - bool isExec = false; - - uint64_t pos = 0; - - NarIndexer(NarAccessor & acc, Source & source) - : acc(acc) - , source(source) - { - } - - NarMember & createMember(const CanonPath & path, NarMember member) - { - size_t level = 0; - for (auto _ : path) { - (void) _; - ++level; - } - - while (parents.size() > level) - parents.pop(); - - if (parents.empty()) { - acc.root = std::move(member); - parents.push(&acc.root); - return acc.root; - } else { - if (parents.top()->stat.type != Type::tDirectory) - throw Error("NAR file missing parent directory of path '%s'", path); - auto result = parents.top()->children.emplace(*path.baseName(), std::move(member)); - auto & ref = result.first->second; - parents.push(&ref); - return ref; - } - } - - void createDirectory(const CanonPath & path) override - { - createMember( - path, - NarMember{.stat = {.type = Type::tDirectory, .fileSize = 0, .isExecutable = false, .narOffset = 0}}); - } - - void createRegularFile(const CanonPath & path, std::function func) override - { - auto & nm = createMember( - path, - NarMember{.stat = {.type = Type::tRegular, .fileSize = 0, .isExecutable = false, .narOffset = 0}}); - NarMemberConstructor nmc{nm, pos}; - nmc.skipContents = true; /* Don't care about contents. */ - func(nmc); - } - - void createSymlink(const CanonPath & path, const std::string & target) override - { - createMember(path, NarMember{.stat = {.type = Type::tSymlink}, .target = target}); - } - - size_t read(char * data, size_t len) override - { - auto n = source.read(data, len); - pos += n; - return n; - } - }; - - NarAccessor(std::string && _nar) - : nar(_nar) + NarAccessorImpl(NarListing && listing, GetNarBytes getNarBytes) + : root{std::move(listing)} + , getNarBytes{std::move(getNarBytes)} { - StringSource source(*nar); - NarIndexer indexer(*this, source); - parseDump(indexer, indexer); } - NarAccessor(Source & source) + NarListing * find(const CanonPath & path) { - NarIndexer indexer(*this, source); - parseDump(indexer, indexer); - } - - NarAccessor(Source & source, GetNarBytes getNarBytes) - : getNarBytes(std::move(getNarBytes)) - { - NarIndexer indexer(*this, source); - parseDump(indexer, indexer); - } - - NarAccessor(const nlohmann::json & listing, GetNarBytes getNarBytes) - : getNarBytes(getNarBytes) - { - [&](this const auto & recurse, NarMember & member, const nlohmann::json & v) -> void { - std::string type = v["type"]; - - if (type == "directory") { - member.stat = {.type = Type::tDirectory}; - for (const auto & [name, function] : v["entries"].items()) { - recurse(member.children[name], function); - } - } else if (type == "regular") { - member.stat = { - .type = Type::tRegular, - .fileSize = v["size"], - .isExecutable = v.value("executable", false), - .narOffset = v["narOffset"]}; - } else if (type == "symlink") { - member.stat = {.type = Type::tSymlink}; - member.target = v.value("target", ""); - } else - return; - }(root, listing); - } - - NarMember * find(const CanonPath & path) - { - NarMember * current = &root; + NarListing * current = &root; for (const auto & i : path) { - if (current->stat.type != Type::tDirectory) + auto * dir = std::get_if(¤t->raw); + if (!dir) return nullptr; - auto child = current->children.find(std::string(i)); - if (child == current->children.end()) + auto * child = nix::get(dir->entries, i); + if (!child) return nullptr; - current = &child->second; + current = child; } return current; } - NarMember & get(const CanonPath & path) + NarListing & get(const CanonPath & path) { auto result = find(path); if (!result) @@ -203,144 +75,101 @@ struct NarAccessor : public SourceAccessor auto i = find(path); if (!i) return std::nullopt; - return i->stat; + return std::visit( + overloaded{ + [](const NarListing::Regular & r) -> Stat { + return { + .type = Type::tRegular, + .fileSize = r.contents.fileSize, + .isExecutable = r.executable, + .narOffset = r.contents.narOffset, + }; + }, + [](const NarListing::Directory &) -> Stat { + return { + .type = Type::tDirectory, + }; + }, + [](const NarListing::Symlink &) -> Stat { + return { + .type = Type::tSymlink, + }; + }, + }, + i->raw); } DirEntries readDirectory(const CanonPath & path) override { - auto i = get(path); + auto & i = get(path); - if (i.stat.type != Type::tDirectory) + auto * dir = std::get_if(&i.raw); + if (!dir) throw Error("path '%1%' inside NAR file is not a directory", path); DirEntries res; - for (const auto & child : i.children) - res.insert_or_assign(child.first, std::nullopt); + for (const auto & [name, child] : dir->entries) + res.insert_or_assign(name, std::nullopt); return res; } - std::string readFile(const CanonPath & path) override + void readFile(const CanonPath & path, Sink & sink, fun sizeCallback) override { - auto i = get(path); - if (i.stat.type != Type::tRegular) + auto & i = get(path); + auto * reg = std::get_if(&i.raw); + if (!reg) throw Error("path '%1%' inside NAR file is not a regular file", path); - if (getNarBytes) - return getNarBytes(*i.stat.narOffset, *i.stat.fileSize); - - assert(nar); - return std::string(*nar, *i.stat.narOffset, *i.stat.fileSize); + assert(getNarBytes); + sizeCallback(reg->contents.fileSize.value()); + return getNarBytes(reg->contents.narOffset.value(), reg->contents.fileSize.value(), sink); } std::string readLink(const CanonPath & path) override { - auto i = get(path); - if (i.stat.type != Type::tSymlink) + auto & i = get(path); + auto * sym = std::get_if(&i.raw); + if (!sym) throw Error("path '%1%' inside NAR file is not a symlink", path); - return i.target; + return sym->target; } }; -ref makeNarAccessor(std::string && nar) -{ - return make_ref(std::move(nar)); -} - -ref makeNarAccessor(Source & source) +ref makeNarAccessor(std::string && nar) { - return make_ref(source); + return make_ref(std::move(nar)); } -ref makeLazyNarAccessor(const nlohmann::json & listing, GetNarBytes getNarBytes) +ref makeNarAccessor(NarListing listing) { - return make_ref(listing, getNarBytes); + return make_ref(std::move(listing)); } -ref makeLazyNarAccessor(Source & source, GetNarBytes getNarBytes) +ref makeLazyNarAccessor(NarListing listing, GetNarBytes getNarBytes) { - return make_ref(source, getNarBytes); + return make_ref(std::move(listing), getNarBytes); } -GetNarBytes seekableGetNarBytes(const Path & path) +GetNarBytes seekableGetNarBytes(const std::filesystem::path & path) { - AutoCloseFD fd = toDescriptor(open( - path.c_str(), - O_RDONLY -#ifdef O_CLOEXEC - | O_CLOEXEC -#endif - )); + auto fd = openFileReadonly(path); if (!fd) - throw SysError("opening NAR cache file '%s'", path); + throw NativeSysError("opening NAR cache file %s", PathFmt(path)); return [inner = seekableGetNarBytes(fd.get()), fd = make_ref(std::move(fd))]( - uint64_t offset, uint64_t length) { return inner(offset, length); }; + uint64_t offset, uint64_t length, Sink & sink) { return inner(offset, length, sink); }; } GetNarBytes seekableGetNarBytes(Descriptor fd) { - return [fd](uint64_t offset, uint64_t length) { - if (::lseek(fromDescriptorReadOnly(fd), offset, SEEK_SET) == -1) - throw SysError("seeking in file"); - - std::string buf(length, 0); - readFull(fd, buf.data(), length); - - return buf; + return [fd](uint64_t offset, uint64_t length, Sink & sink) { + if (offset >= std::numeric_limits::max()) /* Just in case off_t is not 64 bits. */ + throw Error("can't read %1% NAR bytes from offset %2%: offset too big", length, offset); + if (length >= std::numeric_limits::max()) /* Just in case size_t is 32 bits. */ + throw Error("can't read %1% NAR bytes from offset %2%: length is too big", length, offset); + copyFdRange(fd, static_cast(offset), static_cast(length), sink); }; } -template -using ListNarResult = std::conditional_t; - -template -static ListNarResult listNarImpl(SourceAccessor & accessor, const CanonPath & path) -{ - auto st = accessor.lstat(path); - - switch (st.type) { - case SourceAccessor::Type::tRegular: - return typename ListNarResult::Regular{ - .executable = st.isExecutable, - .contents = - NarListingRegularFile{ - .fileSize = st.fileSize, - .narOffset = st.narOffset && *st.narOffset ? st.narOffset : std::nullopt, - }, - }; - case SourceAccessor::Type::tDirectory: { - typename ListNarResult::Directory dir; - for (const auto & [name, type] : accessor.readDirectory(path)) { - if constexpr (deep) { - dir.entries.emplace(name, listNarImpl(accessor, path / name)); - } else { - dir.entries.emplace(name, fso::Opaque{}); - } - } - return dir; - } - case SourceAccessor::Type::tSymlink: - return typename ListNarResult::Symlink{ - .target = accessor.readLink(path), - }; - case SourceAccessor::Type::tBlock: - case SourceAccessor::Type::tChar: - case SourceAccessor::Type::tSocket: - case SourceAccessor::Type::tFifo: - case SourceAccessor::Type::tUnknown: - assert(false); // cannot happen for NARs - } -} - -NarListing listNarDeep(SourceAccessor & accessor, const CanonPath & path) -{ - return listNarImpl(accessor, path); -} - -ShallowNarListing listNarShallow(SourceAccessor & accessor, const CanonPath & path) -{ - return listNarImpl(accessor, path); -} - } // namespace nix diff --git a/src/libutil/nar-cache.cc b/src/libutil/nar-cache.cc new file mode 100644 index 000000000000..866bcddfa675 --- /dev/null +++ b/src/libutil/nar-cache.cc @@ -0,0 +1,84 @@ +#include "nix/util/nar-cache.hh" +#include "nix/util/file-system.hh" + +#include +#include +#include +#include + +namespace nix { + +NarCache::NarCache(std::optional cacheDir_) + : cacheDir(std::move(cacheDir_)) +{ + if (cacheDir) + createDirs(*cacheDir); +} + +ref NarCache::getOrInsert(const Hash & narHash, fun populate) +{ + // Check in-memory cache first + if (auto * accessor = get(nars, narHash)) + return *accessor; + + auto cacheAccessor = [&](ref accessor) { + nars.emplace(narHash, accessor); + return accessor; + }; + + auto getNar = [&]() { + StringSink sink; + populate(sink); + return std::move(sink.s); + }; + + if (cacheDir) { + auto makeCacheFile = [&](const std::string & ext) { + auto res = *cacheDir / narHash.to_string(HashFormat::Nix32, false); + res += "."; + res += ext; + return res; + }; + + auto cacheFile = makeCacheFile("nar"); + auto listingFile = makeCacheFile("ls"); + + if (nix::pathExists(cacheFile)) { + try { + return cacheAccessor(makeLazyNarAccessor( + nlohmann::json::parse(nix::readFile(listingFile)).template get(), + seekableGetNarBytes(cacheFile))); + } catch (SystemError &) { + } + + try { + return cacheAccessor(makeNarAccessor(nix::readFile(cacheFile))); + } catch (SystemError &) { + } + } + + auto nar = getNar(); + + try { + /* FIXME: do this asynchronously. */ + writeFile(cacheFile, nar); + } catch (...) { + ignoreExceptionExceptInterrupt(); + } + + auto narAccessor = makeNarAccessor(std::move(nar)); + + try { + nlohmann::json j = narAccessor->getListing(); + writeFile(listingFile, j.dump()); + } catch (...) { + ignoreExceptionExceptInterrupt(); + } + + return cacheAccessor(narAccessor); + } + + return cacheAccessor(makeNarAccessor(getNar())); +} + +} // namespace nix diff --git a/src/libutil/nar-listing.cc b/src/libutil/nar-listing.cc new file mode 100644 index 000000000000..4722577e81da --- /dev/null +++ b/src/libutil/nar-listing.cc @@ -0,0 +1,181 @@ +#include "nix/util/nar-listing.hh" +#include "nix/util/archive.hh" +#include "nix/util/error.hh" + +#include + +namespace nix { + +NarListing parseNarListing(Source & source) +{ + struct NarMemberConstructor : CreateRegularFileSink + { + private: + + NarListing::Regular & regular; + + uint64_t & pos; + + public: + + NarMemberConstructor(NarListing::Regular & reg, uint64_t & pos) + : regular(reg) + , pos(pos) + { + } + + void isExecutable() override + { + regular.executable = true; + } + + void preallocateContents(uint64_t size) override + { + regular.contents.fileSize = size; + regular.contents.narOffset = pos; + } + + void operator()(std::string_view data) override {} + }; + + struct NarIndexer : FileSystemObjectSink, Source + { + std::optional root; + Source & source; + + std::stack parents; + + uint64_t pos = 0; + + NarIndexer(Source & source) + : source(source) + { + } + + NarListing & createMember(const CanonPath & path, NarListing member) + { + size_t level = 0; + for (auto _ : path) { + (void) _; + ++level; + } + + while (parents.size() > level) + parents.pop(); + + if (parents.empty()) { + root = std::move(member); + parents.push(&*root); + return *root; + } else { + auto * parentDir = std::get_if(&parents.top()->raw); + if (!parentDir) + throw Error("NAR file missing parent directory of path '%s'", path); + auto result = parentDir->entries.emplace(*path.baseName(), std::move(member)); + parents.push(&result.first->second); + return result.first->second; + } + } + + void createDirectory(const CanonPath & path) override + { + createMember(path, NarListing::Directory{}); + } + + void createRegularFile(const CanonPath & path, fun func) override + { + auto & nm = createMember( + path, + NarListing::Regular{ + .executable = false, + .contents = + NarListingRegularFile{ + .fileSize = 0, + .narOffset = pos, + }, + }); + /* We know the downcast will succeed because we just added this */ + auto & reg = std::get(nm.raw); + NarMemberConstructor nmc{reg, pos}; + nmc.skipContents = true; /* Don't care about contents. */ + func(nmc); + } + + void createSymlink(const CanonPath & path, const std::string & target) override + { + createMember(path, NarListing::Symlink{.target = target}); + } + + size_t read(char * data, size_t len) override + { + auto n = source.read(data, len); + pos += n; + return n; + } + + void skip(size_t len) override + { + source.skip(len); + pos += len; + } + }; + + NarIndexer indexer(source); + parseDump(indexer, indexer); + return std::move(*indexer.root); +} + +template +using ListNarResult = std::conditional_t; + +template +static ListNarResult listNarImpl(SourceAccessor & accessor, const CanonPath & path) +{ + auto st = accessor.lstat(path); + + switch (st.type) { + case SourceAccessor::Type::tRegular: + return typename ListNarResult::Regular{ + .executable = st.isExecutable, + .contents = + NarListingRegularFile{ + .fileSize = st.fileSize, + .narOffset = st.narOffset && *st.narOffset ? st.narOffset : std::nullopt, + }, + }; + case SourceAccessor::Type::tDirectory: { + typename ListNarResult::Directory dir; + for (const auto & [name, type] : accessor.readDirectory(path)) { + if constexpr (deep) { + dir.entries.emplace(name, listNarImpl(accessor, path / name)); + } else { + dir.entries.emplace(name, fso::Opaque{}); + } + } + return dir; + } + case SourceAccessor::Type::tSymlink: + return typename ListNarResult::Symlink{ + .target = accessor.readLink(path), + }; + case SourceAccessor::Type::tBlock: + case SourceAccessor::Type::tChar: + case SourceAccessor::Type::tSocket: + case SourceAccessor::Type::tFifo: + case SourceAccessor::Type::tUnknown: + default: + throw Error("file '%s' has an unsupported type", accessor.showPath(path)); + } +} + +NarListing listNarDeep(SourceAccessor & accessor, const CanonPath & path) +{ + return listNarImpl(accessor, path); +} + +ShallowNarListing listNarShallow(SourceAccessor & accessor, const CanonPath & path) +{ + return listNarImpl(accessor, path); +} + +} // namespace nix diff --git a/src/libutil/posix-source-accessor.cc b/src/libutil/posix-source-accessor.cc index 632504e74a05..e3731c9c5745 100644 --- a/src/libutil/posix-source-accessor.cc +++ b/src/libutil/posix-source-accessor.cc @@ -39,7 +39,7 @@ std::filesystem::path PosixSourceAccessor::makeAbsPath(const CanonPath & path) : root / path.rel(); } -void PosixSourceAccessor::readFile(const CanonPath & path, Sink & sink, std::function sizeCallback) +void PosixSourceAccessor::readFile(const CanonPath & path, Sink & sink, fun sizeCallback) { assertNoSymlinks(path); @@ -55,29 +55,11 @@ void PosixSourceAccessor::readFile(const CanonPath & path, Sink & sink, std::fun if (!fd) throw SysError("opening file '%1%'", ap.string()); - struct stat st; - if (fstat(fromDescriptorReadOnly(fd.get()), &st) == -1) - throw SysError("statting file"); + auto size = getFileSize(fd.get()); - sizeCallback(st.st_size); + sizeCallback(size); - off_t left = st.st_size; - - std::array buf; - while (left) { - checkInterrupt(); - ssize_t rd = read(fromDescriptorReadOnly(fd.get()), buf.data(), (size_t) std::min(left, (off_t) buf.size())); - if (rd == -1) { - if (errno != EINTR) - throw SysError("reading from file '%s'", showPath(path)); - } else if (rd == 0) - throw SysError("unexpected end-of-file reading '%s'", showPath(path)); - else { - assert(rd <= left); - sink({(char *) buf.data(), (size_t) rd}); - left -= rd; - } - } + drainFD(fd.get(), sink, {.expectedSize = size}); } bool PosixSourceAccessor::pathExists(const CanonPath & path) @@ -87,14 +69,14 @@ bool PosixSourceAccessor::pathExists(const CanonPath & path) return nix::pathExists(makeAbsPath(path).string()); } -using Cache = boost::concurrent_flat_map>; +using Cache = boost::concurrent_flat_map>; static Cache cache; -std::optional PosixSourceAccessor::cachedLstat(const CanonPath & path) +std::optional PosixSourceAccessor::cachedLstat(const CanonPath & path) { - // Note: we convert std::filesystem::path to Path because the + // Note: we convert std::filesystem::path to std::string because the // former is not hashable on libc++. - Path absPath = makeAbsPath(path).string(); + std::string absPath = makeAbsPath(path).string(); if (auto res = getConcurrent(cache, absPath)) return *res; @@ -188,7 +170,7 @@ SourceAccessor::DirEntries PosixSourceAccessor::readDirectory(const CanonPath & if (e.code() == std::errc::permission_denied || e.code() == std::errc::operation_not_permitted) return std::nullopt; else - throw; + throw SystemError(e.code(), "getting status of '%s'", PathFmt(entry.path())); } }(); res.emplace(entry.path().filename().string(), type); @@ -200,7 +182,7 @@ std::string PosixSourceAccessor::readLink(const CanonPath & path) { if (auto parent = path.parent()) assertNoSymlinks(*parent); - return nix::readLink(makeAbsPath(path).string()); + return nix::readLink(makeAbsPath(path)).string(); } std::optional PosixSourceAccessor::getPhysicalPath(const CanonPath & path) @@ -213,7 +195,7 @@ void PosixSourceAccessor::assertNoSymlinks(CanonPath path) while (!path.isRoot()) { auto st = cachedLstat(path); if (st && S_ISLNK(st->st_mode)) - throw Error("path '%s' is a symlink", showPath(path)); + throw SymlinkNotAllowed(path, "path '%s' is a symlink", showPath(path)); path.pop(); } } @@ -224,8 +206,8 @@ ref getFSSourceAccessor() return rootFS; } -ref makeFSSourceAccessor(std::filesystem::path root) +ref makeFSSourceAccessor(std::filesystem::path root, bool trackLastModified) { - return make_ref(std::move(root)); + return make_ref(std::move(root), trackLastModified); } } // namespace nix diff --git a/src/libutil/processes.cc b/src/libutil/processes.cc new file mode 100644 index 000000000000..a9ce2b521211 --- /dev/null +++ b/src/libutil/processes.cc @@ -0,0 +1,11 @@ +#include "nix/util/processes.hh" + +namespace nix { + +Pid & Pid::operator=(Pid && other) noexcept +{ + swap(*this, other); + return *this; +} + +} // namespace nix diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index e71ec66d26a9..8adbe63841e0 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -1,4 +1,5 @@ #include "nix/util/serialise.hh" +#include "nix/util/file-descriptor.hh" #include "nix/util/compression.hh" #include "nix/util/signals.hh" #include "nix/util/socket.hh" @@ -12,7 +13,6 @@ #ifdef _WIN32 # include -# include "nix/util/windows-error.hh" #else # include #endif @@ -124,7 +124,7 @@ void Source::skip(size_t len) size_t BufferedSource::read(char * data, size_t len) { if (!buffer) - buffer = decltype(buffer)(new char[bufSize]); + buffer = std::make_unique_for_overwrite(bufSize); if (!bufPosIn) bufPosIn = readUnbuffered(buffer.get(), bufSize); @@ -138,6 +138,51 @@ size_t BufferedSource::read(char * data, size_t len) return n; } +std::string BufferedSource::readLine(bool eofOk, char terminator) +{ + if (!buffer) + buffer = std::make_unique_for_overwrite(bufSize); + + std::string line; + while (true) { + if (bufPosOut < bufPosIn) { + auto * start = buffer.get() + bufPosOut; + auto * end = buffer.get() + bufPosIn; + if (auto * newline = static_cast(memchr(start, terminator, end - start))) { + line.append(start, newline - start); + bufPosOut = (newline - buffer.get()) + 1; + if (bufPosOut == bufPosIn) + bufPosOut = bufPosIn = 0; + return line; + } + + line.append(start, end - start); + bufPosOut = bufPosIn = 0; + } + + auto handleEof = [&]() -> std::string { + bufPosOut = bufPosIn = 0; + if (eofOk) + return line; + throw EndOfFile("unexpected EOF reading a line"); + }; + + size_t n = 0; + try { + n = readUnbuffered(buffer.get(), bufSize); + } catch (EndOfFile & e) { + return handleEof(); + } + + if (n == 0) { + return handleEof(); + } + + bufPosIn = n; + bufPosOut = 0; + } +} + bool BufferedSource::hasData() { return bufPosOut < bufPosIn; @@ -145,28 +190,11 @@ bool BufferedSource::hasData() size_t FdSource::readUnbuffered(char * data, size_t len) { -#ifdef _WIN32 - DWORD n; - checkInterrupt(); - if (!::ReadFile(fd, data, len, &n, NULL)) { - _good = false; - throw windows::WinError("ReadFile when FdSource::readUnbuffered"); - } -#else - ssize_t n; - do { - checkInterrupt(); - n = ::read(fd, data, len); - } while (n == -1 && errno == EINTR); - if (n == -1) { - _good = false; - throw SysError("reading from file"); - } + auto n = nix::read(fd, {reinterpret_cast(data), len}); if (n == 0) { _good = false; throw EndOfFile(std::string(*endOfFileError)); } -#endif read += n; return n; } @@ -207,8 +235,7 @@ void FdSource::restart() throw Error("can't seek to the start of a file"); buffer.reset(); read = bufPosIn = bufPosOut = 0; - int fd_ = fromDescriptorReadOnly(fd); - if (lseek(fd_, 0, SEEK_SET) == -1) + if (lseek(fd, 0, SEEK_SET) == -1) throw SysError("seeking to the start of a file"); } @@ -264,7 +291,7 @@ void StringSource::skip(size_t len) pos += len; } -CompressedSource::CompressedSource(RestartableSource & source, const std::string & compressionMethod) +CompressedSource::CompressedSource(RestartableSource & source, CompressionAlgo compressionMethod) : compressedData([&]() { StringSink sink; auto compressionSink = makeCompressionSink(compressionMethod, sink); @@ -277,17 +304,17 @@ CompressedSource::CompressedSource(RestartableSource & source, const std::string { } -std::unique_ptr sourceToSink(std::function fun) +std::unique_ptr sourceToSink(fun reader) { struct SourceToSink : FinishSink { typedef boost::coroutines2::coroutine coro_t; - std::function fun; + fun reader; std::optional coro; - SourceToSink(std::function fun) - : fun(fun) + SourceToSink(fun reader) + : reader(reader) { } @@ -312,7 +339,7 @@ std::unique_ptr sourceToSink(std::function fun) cur.remove_prefix(n); return n; }); - fun(source); + reader(source); }); } @@ -332,21 +359,21 @@ std::unique_ptr sourceToSink(std::function fun) } }; - return std::make_unique(fun); + return std::make_unique(reader); } -std::unique_ptr sinkToSource(std::function fun, std::function eof) +std::unique_ptr sinkToSource(fun writer, fun eof) { struct SinkToSource : Source { typedef boost::coroutines2::coroutine coro_t; - std::function fun; - std::function eof; + fun writer; + fun eof; std::optional coro; - SinkToSource(std::function fun, std::function eof) - : fun(fun) + SinkToSource(fun writer, fun eof) + : writer(writer) , eof(eof) { } @@ -363,12 +390,12 @@ std::unique_ptr sinkToSource(std::function fun, std::funct yield(data); } }); - fun(sink); + writer(sink); }); } if (cur.empty()) { - if (hasCoro) { + if (hasCoro && *coro) { (*coro)(); } if (*coro) { @@ -383,11 +410,21 @@ std::unique_ptr sinkToSource(std::function fun, std::funct size_t n = cur.copy(data, len); cur.remove_prefix(n); + /* This is necessary to ensure that the coroutine gets resumed + after the consumer has finished reading the Source. Otherwise the + coroutine is always abandoned (i.e. it is always destroyed when + suspended). */ + if (cur.empty() && coro && *coro) { + (*coro)(); + if (*coro) + cur = coro->get(); + } + return n; } }; - return std::make_unique(fun, eof); + return std::make_unique(writer, eof); } void writePadding(size_t len, Sink & sink) @@ -494,8 +531,8 @@ T readStrings(Source & source) return ss; } -template Paths readStrings(Source & source); -template PathSet readStrings(Source & source); +template Strings readStrings(Source & source); +template StringSet readStrings(Source & source); Error readError(Source & source) { diff --git a/src/libutil/signature/local-keys.cc b/src/libutil/signature/local-keys.cc index 7dcd92c72aef..51f94cee006a 100644 --- a/src/libutil/signature/local-keys.cc +++ b/src/libutil/signature/local-keys.cc @@ -1,36 +1,96 @@ -#include "nix/util/signature/local-keys.hh" +#include +#include +#include -#include "nix/util/file-system.hh" #include "nix/util/base-n.hh" +#include "nix/util/signature/local-keys.hh" +#include "nix/util/json-utils.hh" #include "nix/util/util.hh" -#include namespace nix { -BorrowedCryptoValue BorrowedCryptoValue::parse(std::string_view s) +namespace { + +/** + * Parse a colon-separated string where the second part is Base64-encoded. + * + * @param s The string to parse in the format `:`. + * @param typeName Name of the type being parsed (for error messages). + * @return A pair of (name, decoded-data). + */ +std::pair parseColonBase64(std::string_view s, std::string_view typeName) { size_t colon = s.find(':'); if (colon == std::string::npos || colon == 0) - return {"", ""}; - return {s.substr(0, colon), s.substr(colon + 1)}; + throw FormatError("%s is corrupt", typeName); + + auto name = std::string(s.substr(0, colon)); + auto data = base64::decode(s.substr(colon + 1)); + + if (name.empty() || data.empty()) + throw FormatError("%s is corrupt", typeName); + + return {std::move(name), std::move(data)}; } -Key::Key(std::string_view s, bool sensitiveValue) +/** + * Serialize a name and data to a colon-separated string with Base64 encoding. + * + * @param name The name part. + * @param data The raw data to be Base64-encoded. + * @return A string in the format `:`. + */ +std::string serializeColonBase64(std::string_view name, std::string_view data) +{ + return std::string(name) + ":" + base64::encode(std::as_bytes(std::span{data.data(), data.size()})); +} + +} // anonymous namespace + +Signature Signature::parse(std::string_view s) +{ + auto [keyName, sig] = parseColonBase64(s, "signature"); + return Signature{ + .keyName = std::move(keyName), + .sig = std::move(sig), + }; +} + +std::string Signature::to_string() const { - auto ss = BorrowedCryptoValue::parse(s); + return serializeColonBase64(keyName, sig); +} - name = ss.name; - key = ss.payload; +template +std::set Signature::parseMany(const Container & sigStrs) +{ + auto parsed = sigStrs | std::views::transform([](const auto & s) { return Signature::parse(s); }); + return std::set(parsed.begin(), parsed.end()); +} - try { - if (name == "" || key == "") - throw FormatError("key is corrupt"); +template std::set Signature::parseMany(const Strings &); +template std::set Signature::parseMany(const StringSet &); - key = base64::decode(key); +Strings Signature::toStrings(const std::set & sigs) +{ + Strings res; + for (const auto & sig : sigs) { + res.push_back(sig.to_string()); + } + + return res; +} + +Key::Key(std::string_view s, bool sensitiveValue) +{ + try { + auto [parsedName, parsedKey] = parseColonBase64(s, "key"); + name = std::move(parsedName); + key = std::move(parsedKey); } catch (Error & e) { std::string extra; if (!sensitiveValue) - extra = fmt(" with raw value '%s'", key); + extra = fmt(" with raw value '%s'", s); e.addTrace({}, "while decoding key named '%s'%s", name, extra); throw; } @@ -38,7 +98,7 @@ Key::Key(std::string_view s, bool sensitiveValue) std::string Key::to_string() const { - return name + ":" + base64::encode(std::as_bytes(std::span{key})); + return serializeColonBase64(name, key); } SecretKey::SecretKey(std::string_view s) @@ -48,12 +108,15 @@ SecretKey::SecretKey(std::string_view s) throw Error("secret key is not valid"); } -std::string SecretKey::signDetached(std::string_view data) const +Signature SecretKey::signDetached(std::string_view data) const { unsigned char sig[crypto_sign_BYTES]; unsigned long long sigLen; crypto_sign_detached(sig, &sigLen, (unsigned char *) data.data(), data.size(), (unsigned char *) key.data()); - return name + ":" + base64::encode(std::as_bytes(std::span(sig, sigLen))); + return Signature{ + .keyName = name, + .sig = std::string((char *) sig, sigLen), + }; } PublicKey SecretKey::toPublicKey() const @@ -80,41 +143,47 @@ PublicKey::PublicKey(std::string_view s) throw Error("public key is not valid"); } -bool PublicKey::verifyDetached(std::string_view data, std::string_view sig) const +bool PublicKey::verifyDetached(std::string_view data, const Signature & sig) const { - auto ss = BorrowedCryptoValue::parse(sig); - - if (ss.name != std::string_view{name}) + if (sig.keyName != name) return false; - return verifyDetachedAnon(data, ss.payload); + return verifyDetachedAnon(data, sig); } -bool PublicKey::verifyDetachedAnon(std::string_view data, std::string_view sig) const +bool PublicKey::verifyDetachedAnon(std::string_view data, const Signature & sig) const { - std::string sig2; - try { - sig2 = base64::decode(sig); - } catch (Error & e) { - e.addTrace({}, "while decoding signature '%s'", sig); - } - if (sig2.size() != crypto_sign_BYTES) + if (sig.sig.size() != crypto_sign_BYTES) throw Error("signature is not valid"); return crypto_sign_verify_detached( - (unsigned char *) sig2.data(), (unsigned char *) data.data(), data.size(), (unsigned char *) key.data()) + (unsigned char *) sig.sig.data(), + (unsigned char *) data.data(), + data.size(), + (unsigned char *) key.data()) == 0; } -bool verifyDetached(std::string_view data, std::string_view sig, const PublicKeys & publicKeys) +bool verifyDetached(std::string_view data, const Signature & sig, const PublicKeys & publicKeys) { - auto ss = BorrowedCryptoValue::parse(sig); - - auto key = publicKeys.find(std::string(ss.name)); + auto key = publicKeys.find(sig.keyName); if (key == publicKeys.end()) return false; - return key->second.verifyDetachedAnon(data, ss.payload); + return key->second.verifyDetachedAnon(data, sig); } } // namespace nix + +namespace nlohmann { +void adl_serializer::to_json(json & j, const Signature & s) +{ + j = s.to_string(); +} + +Signature adl_serializer::from_json(const json & j) +{ + return Signature::parse(getString(j)); +} + +} // namespace nlohmann diff --git a/src/libutil/signature/signer.cc b/src/libutil/signature/signer.cc index 9f6f663e92c8..fff03fc30db1 100644 --- a/src/libutil/signature/signer.cc +++ b/src/libutil/signature/signer.cc @@ -11,7 +11,7 @@ LocalSigner::LocalSigner(SecretKey && privateKey) { } -std::string LocalSigner::signDetached(std::string_view s) const +Signature LocalSigner::signDetached(std::string_view s) const { return privateKey.signDetached(s); } diff --git a/src/libutil/source-accessor.cc b/src/libutil/source-accessor.cc index 76f3edc17a04..49c9b587e91b 100644 --- a/src/libutil/source-accessor.cc +++ b/src/libutil/source-accessor.cc @@ -56,7 +56,7 @@ std::string SourceAccessor::readFile(const CanonPath & path) return std::move(sink.s); } -void SourceAccessor::readFile(const CanonPath & path, Sink & sink, std::function sizeCallback) +void SourceAccessor::readFile(const CanonPath & path, Sink & sink, fun sizeCallback) { auto s = readFile(path); sizeCallback(s.size()); @@ -114,7 +114,7 @@ CanonPath SourceAccessor::resolveSymlinks(const CanonPath & path, SymlinkResolut if (!linksAllowed--) throw Error("infinite symlink recursion in path '%s'", showPath(path)); auto target = readLink(res); - if (isAbsolute(target)) { + if (std::filesystem::path(target).is_absolute()) { res = CanonPath::root; } else { res.pop(); diff --git a/src/libutil/tarfile.cc b/src/libutil/tarfile.cc index 0757b3a81f80..e71c1103735b 100644 --- a/src/libutil/tarfile.cc +++ b/src/libutil/tarfile.cc @@ -136,7 +136,7 @@ static void extract_archive(TarArchive & archive, const std::filesystem::path & if (!name) throw Error("cannot get archive member name: %s", archive_error_string(archive.archive)); if (r == ARCHIVE_WARN) - warn(archive_error_string(archive.archive)); + warn("getting archive member '%1%': %2%", name, archive_error_string(archive.archive)); else archive.check(r); @@ -193,7 +193,7 @@ time_t unpackTarfileToSink(TarArchive & archive, ExtendedFileSystemObjectSink & throw Error("cannot get archive member name: %s", archive_error_string(archive.archive)); auto cpath = CanonPath{path}; if (r == ARCHIVE_WARN) - warn(archive_error_string(archive.archive)); + warn("getting archive member '%1%': %2%", path, archive_error_string(archive.archive)); else archive.check(r); diff --git a/src/libutil/terminal.cc b/src/libutil/terminal.cc index 2a167b52b3fb..52b9e51a221b 100644 --- a/src/libutil/terminal.cc +++ b/src/libutil/terminal.cc @@ -163,7 +163,7 @@ std::string filterANSIEscapes(std::string_view s, bool filterAll, unsigned int w // Note: this object intentionally leaks to avoid a destructor ordering issue (specifically, ~ProgressBar() calling // getWindowSize() after windowSize has been destroyed). -static auto * const windowSize = new Sync>({0, 0}); +static auto * const windowSize = new Sync>{{0, 0}}; void updateWindowSize() { diff --git a/src/libutil/thread-pool.cc b/src/libutil/thread-pool.cc index 24bdeef86706..9a0a14b1323d 100644 --- a/src/libutil/thread-pool.cc +++ b/src/libutil/thread-pool.cc @@ -92,7 +92,7 @@ void ThreadPool::doWork(bool mainThread) std::exception_ptr exc; while (true) { - work_t w; + std::function w; { auto state(state_.lock()); diff --git a/src/libutil/union-source-accessor.cc b/src/libutil/union-source-accessor.cc index 7be670400b05..dab1de37e8ee 100644 --- a/src/libutil/union-source-accessor.cc +++ b/src/libutil/union-source-accessor.cc @@ -12,12 +12,14 @@ struct UnionSourceAccessor : SourceAccessor displayPrefix.clear(); } - std::string readFile(const CanonPath & path) override + void readFile(const CanonPath & path, Sink & sink, fun sizeCallback) override { for (auto & accessor : accessors) { auto st = accessor->maybeLstat(path); - if (st) - return accessor->readFile(path); + if (st) { + accessor->readFile(path, sink, sizeCallback); + return; + } } throw FileNotFound("path '%s' does not exist", showPath(path)); } diff --git a/src/libutil/unix-domain-socket.cc b/src/libutil/unix-domain-socket.cc index 50df7438bd09..b13bf028d873 100644 --- a/src/libutil/unix-domain-socket.cc +++ b/src/libutil/unix-domain-socket.cc @@ -32,21 +32,23 @@ AutoCloseFD createUnixDomainSocket() return fdSocket; } -AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode) +AutoCloseFD createUnixDomainSocket(const std::filesystem::path & path, mode_t mode) { auto fdSocket = nix::createUnixDomainSocket(); - bind(fdSocket.get(), path); + bind(toSocket(fdSocket.get()), path); - if (chmod(path.c_str(), mode) == -1) - throw SysError("changing permissions on '%1%'", path); + chmod(path, mode); if (listen(toSocket(fdSocket.get()), 100) == -1) - throw SysError("cannot listen on socket '%1%'", path); + throw SysError("cannot listen on socket %s", PathFmt(path)); return fdSocket; } +/** + * Use string for path, because no `struct sockaddr_un` variant supports native wide character paths. + */ static void bindConnectProcHelper(std::string_view operationName, auto && operation, Socket fd, const std::string & path) { @@ -69,9 +71,9 @@ bindConnectProcHelper(std::string_view operationName, auto && operation, Socket Pid pid = startProcess([&] { try { pipe.readSide.close(); - Path dir = dirOf(path); - if (chdir(dir.c_str()) == -1) - throw SysError("chdir to '%s' failed", dir); + auto dir = std::filesystem::path(path).parent_path(); + if (chdir(dir.string().c_str()) == -1) + throw SysError("chdir to %s failed", PathFmt(dir)); std::string base(baseNameOf(path)); if (base.size() + 1 >= sizeof(addr.sun_path)) throw Error("socket path '%s' is too long", base); @@ -101,11 +103,11 @@ bindConnectProcHelper(std::string_view operationName, auto && operation, Socket } } -void bind(Socket fd, const std::string & path) +void bind(Socket fd, const std::filesystem::path & path) { - unlink(path.c_str()); + tryUnlink(path); - bindConnectProcHelper("bind", ::bind, fd, path); + bindConnectProcHelper("bind", ::bind, fd, path.string()); } void connect(Socket fd, const std::filesystem::path & path) diff --git a/src/libutil/unix/environment-variables.cc b/src/libutil/unix/environment-variables.cc index c68e3bcad0a6..2adef0cf5fca 100644 --- a/src/libutil/unix/environment-variables.cc +++ b/src/libutil/unix/environment-variables.cc @@ -1,7 +1,10 @@ #include +#include #include "nix/util/environment-variables.hh" +extern char ** environ __attribute__((weak)); + namespace nix { int setEnv(const char * name, const char * value) @@ -14,9 +17,33 @@ std::optional getEnvOs(const std::string & key) return getEnv(key); } +OsStringMap getEnvOs() +{ + OsStringMap env; + for (size_t i = 0; environ[i]; ++i) { + auto s = environ[i]; + auto eq = strchr(s, '='); + if (!eq) + // invalid env, just keep going + continue; + env.emplace(std::string(s, eq), std::string(eq + 1)); + } + return env; +} + +StringMap getEnv() +{ + return getEnvOs(); +} + int setEnvOs(const OsString & name, const OsString & value) { return setEnv(name.c_str(), value.c_str()); } +int unsetEnvOs(const OsChar * name) +{ + return unsetenv(name); +} + } // namespace nix diff --git a/src/libutil/unix/file-descriptor.cc b/src/libutil/unix/file-descriptor.cc index 501f07aec657..05ec38a67cc5 100644 --- a/src/libutil/unix/file-descriptor.cc +++ b/src/libutil/unix/file-descriptor.cc @@ -1,4 +1,3 @@ -#include "nix/util/canon-path.hh" #include "nix/util/file-system.hh" #include "nix/util/signals.hh" #include "nix/util/finally.hh" @@ -6,157 +5,54 @@ #include #include -#include - -#if defined(__linux__) -# include /* pull __NR_* definitions */ -#endif - -#if defined(__linux__) && defined(__NR_openat2) -# define HAVE_OPENAT2 1 -# include -#else -# define HAVE_OPENAT2 0 -#endif +#include #include "util-config-private.hh" #include "util-unix-config-private.hh" namespace nix { -namespace { - -// This function is needed to handle non-blocking reads/writes. This is needed in the buildhook, because -// somehow the json logger file descriptor ends up being non-blocking and breaks remote-building. -// TODO: get rid of buildhook and remove this function again (https://github.com/NixOS/nix/issues/12688) -void pollFD(int fd, int events) +std::make_unsigned_t getFileSize(Descriptor fd) { - struct pollfd pfd; - pfd.fd = fd; - pfd.events = events; - int ret = poll(&pfd, 1, -1); - if (ret == -1) { - throw SysError("poll on file descriptor failed"); - } + auto st = nix::fstat(fd); + return st.st_size; } -} // namespace -std::string readFile(int fd) +size_t read(Descriptor fd, std::span buffer) { - struct stat st; - if (fstat(fd, &st) == -1) - throw SysError("statting file"); - - return drainFD(fd, true, st.st_size); + ssize_t n; + do { + checkInterrupt(); + n = ::read(fd, buffer.data(), buffer.size()); + } while (n == -1 && errno == EINTR); + if (n == -1) + throw SysError("read of %1% bytes", buffer.size()); + return static_cast(n); } -void readFull(int fd, char * buf, size_t count) +size_t readOffset(Descriptor fd, off_t offset, std::span buffer) { - while (count) { + ssize_t n; + do { checkInterrupt(); - ssize_t res = read(fd, buf, count); - if (res == -1) { - switch (errno) { - case EINTR: - continue; - case EAGAIN: - pollFD(fd, POLLIN); - continue; - } - throw SysError("reading from file"); - } - if (res == 0) - throw EndOfFile("unexpected end-of-file"); - count -= res; - buf += res; - } + n = pread(fd, buffer.data(), buffer.size(), offset); + } while (n == -1 && errno == EINTR); + if (n == -1) + throw SysError("pread of %1% bytes at offset %2%", buffer.size(), offset); + return static_cast(n); } -void writeFull(int fd, std::string_view s, bool allowInterrupts) +size_t write(Descriptor fd, std::span buffer, bool allowInterrupts) { - while (!s.empty()) { + ssize_t n; + do { if (allowInterrupts) checkInterrupt(); - ssize_t res = write(fd, s.data(), s.size()); - if (res == -1) { - switch (errno) { - case EINTR: - continue; - case EAGAIN: - pollFD(fd, POLLOUT); - continue; - } - throw SysError("writing to file"); - } - if (res > 0) - s.remove_prefix(res); - } -} - -std::string readLine(int fd, bool eofOk) -{ - std::string s; - while (1) { - checkInterrupt(); - char ch; - // FIXME: inefficient - ssize_t rd = read(fd, &ch, 1); - if (rd == -1) { - switch (errno) { - case EINTR: - continue; - case EAGAIN: { - pollFD(fd, POLLIN); - continue; - } - default: - throw SysError("reading a line"); - } - } else if (rd == 0) { - if (eofOk) - return s; - else - throw EndOfFile("unexpected EOF reading a line"); - } else { - if (ch == '\n') - return s; - s += ch; - } - } -} - -void drainFD(int fd, Sink & sink, bool block) -{ - // silence GCC maybe-uninitialized warning in finally - int saved = 0; - - if (!block) { - saved = fcntl(fd, F_GETFL); - if (fcntl(fd, F_SETFL, saved | O_NONBLOCK) == -1) - throw SysError("making file descriptor non-blocking"); - } - - Finally finally([&]() { - if (!block) { - if (fcntl(fd, F_SETFL, saved) == -1) - throw SysError("making file descriptor blocking"); - } - }); - - std::vector buf(64 * 1024); - while (1) { - checkInterrupt(); - ssize_t rd = read(fd, buf.data(), buf.size()); - if (rd == -1) { - if (!block && (errno == EAGAIN || errno == EWOULDBLOCK)) - break; - if (errno != EINTR) - throw SysError("reading from file"); - } else if (rd == 0) - break; - else - sink({reinterpret_cast(buf.data()), (size_t) rd}); - } + n = ::write(fd, buffer.data(), buffer.size()); + } while (n == -1 && errno == EINTR); + if (n == -1) + throw SysError("write of %1% bytes", buffer.size()); + return static_cast(n); } ////////////////////////////////////////////////////////////////////// @@ -235,107 +131,17 @@ void unix::closeOnExec(int fd) throw SysError("setting close-on-exec flag"); } -#ifdef __linux__ - -namespace linux { - -std::optional openat2(Descriptor dirFd, const char * path, uint64_t flags, uint64_t mode, uint64_t resolve) -{ -# if HAVE_OPENAT2 - /* Cache the result of whether openat2 is not supported. */ - static std::atomic_flag unsupported{}; - - if (!unsupported.test()) { - /* No glibc wrapper yet, but there's a patch: - * https://patchwork.sourceware.org/project/glibc/patch/20251029200519.3203914-1-adhemerval.zanella@linaro.org/ - */ - auto how = ::open_how{.flags = flags, .mode = mode, .resolve = resolve}; - auto res = ::syscall(__NR_openat2, dirFd, path, &how, sizeof(how)); - /* Cache that the syscall is not supported. */ - if (res < 0 && errno == ENOSYS) { - unsupported.test_and_set(); - return std::nullopt; - } - - return res; - } -# endif - return std::nullopt; -} - -} // namespace linux - -#endif - -static Descriptor -openFileEnsureBeneathNoSymlinksIterative(Descriptor dirFd, const CanonPath & path, int flags, mode_t mode) -{ - AutoCloseFD parentFd; - auto nrComponents = std::ranges::distance(path); - assert(nrComponents >= 1); - auto components = std::views::take(path, nrComponents - 1); /* Everything but last component */ - auto getParentFd = [&]() { return parentFd ? parentFd.get() : dirFd; }; - - /* This rather convoluted loop is necessary to avoid TOCTOU when validating that - no inner path component is a symlink. */ - for (auto it = components.begin(); it != components.end(); ++it) { - auto component = std::string(*it); /* Copy into a string to make NUL terminated. */ - assert(component != ".." && !component.starts_with('/')); /* In case invariant is broken somehow.. */ - - AutoCloseFD parentFd2 = ::openat( - getParentFd(), /* First iteration uses dirFd. */ - component.c_str(), - O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC -#ifdef __linux__ - | O_PATH /* Linux-specific optimization. Files are open only for path resolution purposes. */ -#endif -#ifdef __FreeBSD__ - | O_RESOLVE_BENEATH /* Further guard against any possible SNAFUs. */ -#endif - ); - - if (!parentFd2) { - /* Construct the CanonPath for error message. */ - auto path2 = std::ranges::fold_left(components.begin(), ++it, CanonPath::root, [](auto lhs, auto rhs) { - lhs.push(rhs); - return lhs; - }); - - if (errno == ENOTDIR) /* Path component might be a symlink. */ { - struct ::stat st; - if (::fstatat(getParentFd(), component.c_str(), &st, AT_SYMLINK_NOFOLLOW) == 0 && S_ISLNK(st.st_mode)) - throw unix::SymlinkNotAllowed(path2); - errno = ENOTDIR; /* Restore the errno. */ - } else if (errno == ELOOP) { - throw unix::SymlinkNotAllowed(path2); - } - - return INVALID_DESCRIPTOR; - } - - parentFd = std::move(parentFd2); - } - - auto res = ::openat(getParentFd(), std::string(path.baseName().value()).c_str(), flags | O_NOFOLLOW, mode); - if (res < 0 && errno == ELOOP) - throw unix::SymlinkNotAllowed(path); - return res; -} - -Descriptor unix::openFileEnsureBeneathNoSymlinks(Descriptor dirFd, const CanonPath & path, int flags, mode_t mode) +void syncDescriptor(Descriptor fd) { - assert(!path.rel().starts_with('/')); /* Just in case the invariant is somehow broken. */ - assert(!path.isRoot()); -#if HAVE_OPENAT2 - auto maybeFd = linux::openat2( - dirFd, path.rel_c_str(), flags, static_cast(mode), RESOLVE_BENEATH | RESOLVE_NO_SYMLINKS); - if (maybeFd) { - if (*maybeFd < 0 && errno == ELOOP) - throw unix::SymlinkNotAllowed(path); - return *maybeFd; - } + int result = +#if defined(__APPLE__) + ::fcntl(fd, F_FULLFSYNC) +#else + ::fsync(fd) #endif - return openFileEnsureBeneathNoSymlinksIterative(dirFd, path, flags, mode); + ; + if (result == -1) + throw NativeSysError("fsync file descriptor %1%", fd); } } // namespace nix diff --git a/src/libutil/unix/file-path.cc b/src/libutil/unix/file-path.cc index 53b1fca366b4..55ccddb45683 100644 --- a/src/libutil/unix/file-path.cc +++ b/src/libutil/unix/file-path.cc @@ -8,14 +8,9 @@ namespace nix { -std::optional maybePath(PathView path) +std::filesystem::path toOwnedPath(PathView path) { - return {path}; -} - -std::filesystem::path pathNG(PathView path) -{ - return path; + return {std::string{path}}; } } // namespace nix diff --git a/src/libutil/unix/file-system-at.cc b/src/libutil/unix/file-system-at.cc new file mode 100644 index 000000000000..739cac0177ca --- /dev/null +++ b/src/libutil/unix/file-system-at.cc @@ -0,0 +1,225 @@ +#include "nix/util/file-system-at.hh" +#include "nix/util/file-system.hh" +#include "nix/util/signals.hh" +#include "nix/util/source-accessor.hh" + +#include +#include + +#if defined(__linux__) +# include /* pull __NR_* definitions */ +#endif + +#if defined(__linux__) && defined(__NR_openat2) +# define HAVE_OPENAT2 1 +# include +#else +# define HAVE_OPENAT2 0 +#endif + +#if defined(__linux__) && defined(__NR_fchmodat2) +# define HAVE_FCHMODAT2 1 +#else +# define HAVE_FCHMODAT2 0 +#endif + +namespace nix { + +#ifdef __linux__ + +namespace linux { + +std::optional openat2(Descriptor dirFd, const char * path, uint64_t flags, uint64_t mode, uint64_t resolve) +{ +# if HAVE_OPENAT2 + /* Cache the result of whether openat2 is not supported. */ + static std::atomic_flag unsupported{}; + + if (!unsupported.test()) { + /* No glibc wrapper yet, but there's a patch: + * https://patchwork.sourceware.org/project/glibc/patch/20251029200519.3203914-1-adhemerval.zanella@linaro.org/ + */ + auto how = ::open_how{.flags = flags, .mode = mode, .resolve = resolve}; + auto res = ::syscall(__NR_openat2, dirFd, path, &how, sizeof(how)); + /* Cache that the syscall is not supported. */ + if (res < 0 && errno == ENOSYS) { + unsupported.test_and_set(); + return std::nullopt; + } + + return res; + } +# endif + return std::nullopt; +} + +} // namespace linux + +#endif + +void unix::fchmodatTryNoFollow(Descriptor dirFd, const CanonPath & path, mode_t mode) +{ + assert(!path.isRoot()); + +#if HAVE_FCHMODAT2 + /* Cache whether fchmodat2 is not supported. */ + static std::atomic_flag fchmodat2Unsupported{}; + if (!fchmodat2Unsupported.test()) { + /* Try with fchmodat2 first. */ + auto res = ::syscall(__NR_fchmodat2, dirFd, path.rel_c_str(), mode, AT_SYMLINK_NOFOLLOW); + /* Cache that the syscall is not supported. */ + if (res < 0) { + if (errno == ENOSYS) + fchmodat2Unsupported.test_and_set(); + else { + throw SysError([&] { return HintFmt("fchmodat2 %s", PathFmt(descriptorToPath(dirFd) / path.rel())); }); + } + } else + return; + } +#endif + +#ifdef __linux__ + AutoCloseFD pathFd = ::openat(dirFd, path.rel_c_str(), O_PATH | O_NOFOLLOW | O_CLOEXEC); + if (!pathFd) { + throw SysError([&] { + return HintFmt( + "opening %s to get an O_PATH file descriptor (fchmodat2 is unsupported)", + PathFmt(descriptorToPath(dirFd) / path.rel())); + }); + } + + struct ::stat st; + /* Possible since https://github.com/torvalds/linux/commit/55815f70147dcfa3ead5738fd56d3574e2e3c1c2 (3.6) */ + if (::fstat(pathFd.get(), &st) == -1) + throw SysError("statting '%s' relative to parent directory via O_PATH file descriptor", path.rel()); + + if (S_ISLNK(st.st_mode)) + throw SysError(EOPNOTSUPP, "can't change mode of symlink %s", PathFmt(descriptorToPath(dirFd) / path.rel())); + + static std::atomic_flag dontHaveProc{}; + if (!dontHaveProc.test()) { + static const CanonPath selfProcFd = CanonPath("/proc/self/fd"); + + auto selfProcFdPath = selfProcFd / std::to_string(pathFd.get()); + if (int res = ::chmod(selfProcFdPath.c_str(), mode); res == -1) { + if (errno == ENOENT) + dontHaveProc.test_and_set(); + else { + throw SysError([&] { + return HintFmt("chmod %s (%s)", selfProcFdPath, PathFmt(descriptorToPath(dirFd) / path.rel())); + }); + } + } else + return; + } + + static std::atomic warned = false; + warnOnce(warned, "kernel doesn't support fchmodat2 and procfs isn't mounted, falling back to fchmodat"); +#endif + + int res = ::fchmodat( + dirFd, + path.rel_c_str(), + mode, +#if defined(__APPLE__) || defined(__FreeBSD__) + AT_SYMLINK_NOFOLLOW +#else + 0 +#endif + ); + + if (res == -1) { + throw SysError([&] { return HintFmt("fchmodat %s", PathFmt(descriptorToPath(dirFd) / path.rel())); }); + } +} + +static AutoCloseFD +openFileEnsureBeneathNoSymlinksIterative(Descriptor dirFd, const CanonPath & path, int flags, mode_t mode) +{ + AutoCloseFD parentFd; + auto nrComponents = std::ranges::distance(path); + assert(nrComponents >= 1); + auto components = std::views::take(path, nrComponents - 1); /* Everything but last component */ + auto getParentFd = [&]() { return parentFd ? parentFd.get() : dirFd; }; + + /* This rather convoluted loop is necessary to avoid TOCTOU when validating that + no inner path component is a symlink. */ + for (auto it = components.begin(); it != components.end(); ++it) { + auto component = std::string(*it); /* Copy into a string to make NUL terminated. */ + assert(component != ".." && !component.starts_with('/')); /* In case invariant is broken somehow.. */ + + AutoCloseFD parentFd2 = ::openat( + getParentFd(), /* First iteration uses dirFd. */ + component.c_str(), + O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC +#ifdef __linux__ + | O_PATH /* Linux-specific optimization. Files are open only for path resolution purposes. */ +#endif +#ifdef __FreeBSD__ + | O_RESOLVE_BENEATH /* Further guard against any possible SNAFUs. */ +#endif + ); + + if (!parentFd2) { + /* Construct the CanonPath for error message. */ + auto path2 = std::ranges::fold_left(components.begin(), ++it, CanonPath::root, [](auto lhs, auto rhs) { + lhs.push(rhs); + return lhs; + }); + + if (errno == ENOTDIR) /* Path component might be a symlink. */ { + struct ::stat st; + if (::fstatat(getParentFd(), component.c_str(), &st, AT_SYMLINK_NOFOLLOW) == 0 && S_ISLNK(st.st_mode)) + throw SymlinkNotAllowed(path2); + errno = ENOTDIR; /* Restore the errno. */ + } else if (errno == ELOOP) { + throw SymlinkNotAllowed(path2); + } + + return AutoCloseFD{}; + } + + parentFd = std::move(parentFd2); + } + + AutoCloseFD res = ::openat(getParentFd(), std::string(path.baseName().value()).c_str(), flags | O_NOFOLLOW, mode); + if (!res && errno == ELOOP) + throw SymlinkNotAllowed(path); + return res; +} + +AutoCloseFD openFileEnsureBeneathNoSymlinks(Descriptor dirFd, const CanonPath & path, int flags, mode_t mode) +{ + assert(!path.rel().starts_with('/')); /* Just in case the invariant is somehow broken. */ + assert(!path.isRoot()); +#if HAVE_OPENAT2 + auto maybeFd = linux::openat2( + dirFd, path.rel_c_str(), flags, static_cast(mode), RESOLVE_BENEATH | RESOLVE_NO_SYMLINKS); + if (maybeFd) { + if (*maybeFd < 0 && errno == ELOOP) + throw SymlinkNotAllowed(path); + return AutoCloseFD{*maybeFd}; + } +#endif + return openFileEnsureBeneathNoSymlinksIterative(dirFd, path, flags, mode); +} + +OsString readLinkAt(Descriptor dirFd, const CanonPath & path) +{ + assert(!path.isRoot()); + assert(!path.rel().starts_with('/')); /* Just in case the invariant is somehow broken. */ + std::vector buf; + for (ssize_t bufSize = PATH_MAX / 4; true; bufSize += bufSize / 2) { + checkInterrupt(); + buf.resize(bufSize); + ssize_t rlSize = ::readlinkat(dirFd, path.rel_c_str(), buf.data(), bufSize); + if (rlSize == -1) { + throw SysError( + [&] { return HintFmt("reading symbolic link %1%", PathFmt(descriptorToPath(dirFd) / path.rel())); }); + } else if (rlSize < bufSize) + return {buf.data(), static_cast(rlSize)}; + } +} + +} // namespace nix diff --git a/src/libutil/unix/file-system.cc b/src/libutil/unix/file-system.cc index 77b83858f6d5..55fd7584fbf0 100644 --- a/src/libutil/unix/file-system.cc +++ b/src/libutil/unix/file-system.cc @@ -8,15 +8,72 @@ #include #include +#ifdef __FreeBSD__ +# include +# include +#endif + #include "nix/util/file-system.hh" +#include "nix/util/file-system-at.hh" +#include "nix/util/environment-variables.hh" +#include "nix/util/signals.hh" +#include "nix/util/util.hh" #include "util-unix-config-private.hh" namespace nix { -Descriptor openDirectory(const std::filesystem::path & path) +AutoCloseFD openDirectory(const std::filesystem::path & path) +{ + return AutoCloseFD{open(path.c_str(), O_RDONLY | O_DIRECTORY | O_CLOEXEC)}; +} + +AutoCloseFD openFileReadonly(const std::filesystem::path & path) +{ + return AutoCloseFD{open(path.c_str(), O_RDONLY | O_CLOEXEC)}; +} + +AutoCloseFD openNewFileForWrite(const std::filesystem::path & path, mode_t mode, OpenNewFileForWriteParams params) +{ + auto flags = O_WRONLY | O_CREAT | O_CLOEXEC; + if (params.truncateExisting) { + flags |= O_TRUNC; + if (!params.followSymlinksOnTruncate) + flags |= O_NOFOLLOW; + } else { + flags |= O_EXCL; /* O_CREAT | O_EXCL already ensures that symlinks are not followed. */ + } + return AutoCloseFD{open(path.c_str(), flags, mode)}; +} + +std::filesystem::path descriptorToPath(Descriptor fd) +{ + if (fd == STDIN_FILENO) + return ""; + if (fd == STDOUT_FILENO) + return ""; + if (fd == STDERR_FILENO) + return ""; + +#if defined(__linux__) + try { + return readLink("/proc/self/fd/" + std::to_string(fd)); + } catch (SystemError &) { + } +#elif HAVE_F_GETPATH + /* F_GETPATH requires PATH_MAX buffer per POSIX */ + char buf[PATH_MAX]; + if (fcntl(fd, F_GETPATH, buf) != -1) + return buf; +#endif + + /* Fallback for unknown fd or unsupported platform */ + return ""; +} + +std::filesystem::path defaultTempDir() { - return open(path.c_str(), O_RDONLY | O_DIRECTORY | O_CLOEXEC); + return getEnvNonEmpty("TMPDIR").value_or("/tmp"); } void setWriteTime( @@ -38,7 +95,7 @@ void setWriteTime( }, }; if (utimensat(AT_FDCWD, path.c_str(), times, AT_SYMLINK_NOFOLLOW) == -1) - throw SysError("changing modification time of %s (using `utimensat`)", path); + throw SysError("changing modification time of %s (using `utimensat`)", PathFmt(path)); #else struct timeval times[2] = { { @@ -52,18 +109,167 @@ void setWriteTime( }; # if HAVE_LUTIMES if (lutimes(path.c_str(), times) == -1) - throw SysError("changing modification time of %s", path); + throw SysError("changing modification time of %s", PathFmt{path}); # else bool isSymlink = optIsSymlink ? *optIsSymlink : std::filesystem::is_symlink(path); if (!isSymlink) { if (utimes(path.c_str(), times) == -1) - throw SysError("changing modification time of %s (not a symlink)", path); + throw SysError("changing modification time of %s (not a symlink)", PathFmt{path}); } else { - throw Error("Cannot change modification time of symlink %s", path); + throw Error("Cannot change modification time of symlink %s", PathFmt{path}); } # endif #endif } +#ifdef __FreeBSD__ +# define MOUNTEDPATHS_PARAM , std::set & mountedPaths +# define MOUNTEDPATHS_ARG , mountedPaths +#else +# define MOUNTEDPATHS_PARAM +# define MOUNTEDPATHS_ARG +#endif + +static void _deletePath( + Descriptor parentfd, + const std::filesystem::path & path, + uint64_t & bytesFreed, + std::exception_ptr & ex MOUNTEDPATHS_PARAM) +{ + checkInterrupt(); +#ifdef __FreeBSD__ + // In case of emergency (unmount fails for some reason) not recurse into mountpoints. + // This prevents us from tearing up the nullfs-mounted nix store. + if (mountedPaths.find(path) != mountedPaths.end()) { + return; + } +#endif + + auto name = CanonPath::fromFilename(path.filename().native()); + + PosixStat st; + if (fstatat(parentfd, name.rel_c_str(), &st, AT_SYMLINK_NOFOLLOW) == -1) { + if (errno == ENOENT) + return; + throw SysError("getting status of %1%", PathFmt(path)); + } + + if (!S_ISDIR(st.st_mode)) { + /* We are about to delete a file. Will it likely free space? */ + + switch (st.st_nlink) { + /* Yes: last link. */ + case 1: + bytesFreed += st.st_size; + break; + /* Maybe: yes, if 'auto-optimise-store' or manual optimisation + was performed. Instead of checking for real let's assume + it's an optimised file and space will be freed. + + In worst case we will double count on freed space for files + with exactly two hardlinks for unoptimised packages. + */ + case 2: + bytesFreed += st.st_size; + break; + /* No: 3+ links. */ + default: + break; + } + } + + if (S_ISDIR(st.st_mode)) { + /* Make the directory accessible. */ + const auto PERM_MASK = S_IRUSR | S_IWUSR | S_IXUSR; + if ((st.st_mode & PERM_MASK) != PERM_MASK) + try { + unix::fchmodatTryNoFollow(parentfd, name, st.st_mode | PERM_MASK); + } catch (SysError & e) { + e.addTrace({}, "while making directory %1% accessible for deletion", PathFmt(path)); + if (e.errNo == EOPNOTSUPP) + e.addTrace({}, "%1% is now a symlink, expected directory", PathFmt(path)); + throw; + } + + int fd = openat(parentfd, name.rel_c_str(), O_RDONLY | O_DIRECTORY | O_NOFOLLOW); + if (fd == -1) + throw SysError("opening directory %1%", PathFmt(path)); + AutoCloseDir dir(fdopendir(fd)); + if (!dir) + throw SysError("opening directory %1%", PathFmt(path)); + + struct dirent * dirent; + while (errno = 0, dirent = readdir(dir.get())) { /* sic */ + checkInterrupt(); + std::string childName = dirent->d_name; + if (childName == "." || childName == "..") + continue; + _deletePath(dirfd(dir.get()), path / childName, bytesFreed, ex MOUNTEDPATHS_ARG); + } + if (errno) + throw SysError("reading directory %1%", PathFmt(path)); + } + + int flags = S_ISDIR(st.st_mode) ? AT_REMOVEDIR : 0; + if (unlinkat(parentfd, name.rel_c_str(), flags) == -1) { + if (errno == ENOENT) + return; + try { + throw SysError("cannot unlink %1%", PathFmt(path)); + } catch (...) { + if (!ex) + ex = std::current_exception(); + else + ignoreExceptionExceptInterrupt(); + } + } +} + +static void _deletePath(const std::filesystem::path & path, uint64_t & bytesFreed MOUNTEDPATHS_PARAM) +{ + assert(path.is_absolute()); + auto parentDirPath = path.parent_path(); + assert(parentDirPath != path); + + AutoCloseFD dirfd = openDirectory(parentDirPath); + if (!dirfd) { + if (errno == ENOENT) + return; + throw SysError("opening directory %s", PathFmt(parentDirPath)); + } + + std::exception_ptr ex; + + _deletePath(dirfd.get(), path, bytesFreed, ex MOUNTEDPATHS_ARG); + + if (ex) + std::rethrow_exception(ex); +} + +void deletePath(const std::filesystem::path & path) +{ + uint64_t dummy; + deletePath(path, dummy); +} + +void deletePath(const std::filesystem::path & path, uint64_t & bytesFreed) +{ + // Activity act(*logger, lvlDebug, "recursively deleting path '%1%'", path); +#ifdef __FreeBSD__ + std::set mountedPaths; + struct statfs * mntbuf; + int count; + if ((count = getmntinfo(&mntbuf, MNT_WAIT)) < 0) { + throw SysError("getmntinfo"); + } + + for (int i = 0; i < count; i++) { + mountedPaths.emplace(mntbuf[i].f_mntonname); + } +#endif + bytesFreed = 0; + _deletePath(path, bytesFreed MOUNTEDPATHS_ARG); +} + } // namespace nix diff --git a/src/libutil/unix/include/nix/util/monitor-fd.hh b/src/libutil/unix/include/nix/util/monitor-fd.hh index b87bf5ca4f70..de2338ef09ac 100644 --- a/src/libutil/unix/include/nix/util/monitor-fd.hh +++ b/src/libutil/unix/include/nix/util/monitor-fd.hh @@ -27,6 +27,10 @@ private: public: MonitorFdHup(int fd); + MonitorFdHup(MonitorFdHup &&) = delete; + MonitorFdHup(const MonitorFdHup &) = delete; + MonitorFdHup & operator=(MonitorFdHup &&) = delete; + MonitorFdHup & operator=(const MonitorFdHup &) = delete; ~MonitorFdHup() { diff --git a/src/libutil/unix/include/nix/util/signals-impl.hh b/src/libutil/unix/include/nix/util/signals-impl.hh index 2456119beba2..e51ce082a780 100644 --- a/src/libutil/unix/include/nix/util/signals-impl.hh +++ b/src/libutil/unix/include/nix/util/signals-impl.hh @@ -14,7 +14,6 @@ #include "nix/util/error.hh" #include "nix/util/logging.hh" #include "nix/util/ansicolor.hh" -#include "nix/util/signals.hh" #include #include diff --git a/src/libutil/unix/meson.build b/src/libutil/unix/meson.build index 8f89b65ab650..dbe142d3a88c 100644 --- a/src/libutil/unix/meson.build +++ b/src/libutil/unix/meson.build @@ -8,6 +8,12 @@ configdata_unix.set( description : 'Optionally used for changing the files and symlinks.', ) +configdata_unix.set( + 'HAVE_F_GETPATH', + cxx.has_header_symbol('fcntl.h', 'F_GETPATH').to_int(), + description : 'Optionally used for getting the path of a file descriptor (macOS).', +) + # Check for each of these functions, and create a define like `#define # HAVE_CLOSE_RANGE 1`. check_funcs_unix = [ @@ -53,12 +59,13 @@ sources += files( 'environment-variables.cc', 'file-descriptor.cc', 'file-path.cc', + 'file-system-at.cc', 'file-system.cc', 'muxable-pipe.cc', - 'os-string.cc', 'processes.cc', 'signals.cc', 'users.cc', + 'xdg-dirs.cc', ) subdir('include/nix/util') diff --git a/src/libutil/unix/muxable-pipe.cc b/src/libutil/unix/muxable-pipe.cc index 1b8b09adcf50..12c971f94d7c 100644 --- a/src/libutil/unix/muxable-pipe.cc +++ b/src/libutil/unix/muxable-pipe.cc @@ -17,8 +17,8 @@ void MuxablePipePollState::poll(std::optional timeout) void MuxablePipePollState::iterate( std::set & channels, - std::function handleRead, - std::function handleEOF) + fun handleRead, + fun handleEOF) { std::set fds2(channels); std::vector buffer(4096); @@ -27,7 +27,7 @@ void MuxablePipePollState::iterate( assert(fdPollStatusId); assert(*fdPollStatusId < pollStatus.size()); if (pollStatus.at(*fdPollStatusId).revents) { - ssize_t rd = ::read(fromDescriptorReadOnly(k), buffer.data(), buffer.size()); + ssize_t rd = ::read(k, buffer.data(), buffer.size()); // FIXME: is there a cleaner way to handle pt close // than EIO? Is this even standard? if (rd == 0 || (rd == -1 && errno == EIO)) { diff --git a/src/libutil/unix/os-string.cc b/src/libutil/unix/os-string.cc deleted file mode 100644 index 08d275bc671a..000000000000 --- a/src/libutil/unix/os-string.cc +++ /dev/null @@ -1,21 +0,0 @@ -#include -#include -#include -#include - -#include "nix/util/file-path.hh" -#include "nix/util/util.hh" - -namespace nix { - -std::string os_string_to_string(PathViewNG::string_view path) -{ - return std::string{path}; -} - -std::filesystem::path::string_type string_to_os_string(std::string_view s) -{ - return std::string{s}; -} - -} // namespace nix diff --git a/src/libutil/unix/processes.cc b/src/libutil/unix/processes.cc index 9582ff840bf9..95ff7e7d6bf3 100644 --- a/src/libutil/unix/processes.cc +++ b/src/libutil/unix/processes.cc @@ -1,12 +1,14 @@ #include "nix/util/current-process.hh" #include "nix/util/environment-variables.hh" #include "nix/util/executable-path.hh" +#include "nix/util/fmt.hh" #include "nix/util/signals.hh" #include "nix/util/processes.hh" #include "nix/util/finally.hh" #include "nix/util/serialise.hh" #include +#include #include #include #include @@ -35,15 +37,25 @@ namespace nix { Pid::Pid() {} +Pid::Pid(Pid && other) noexcept + : pid(other.pid) + , separatePG(other.separatePG) + , killSignal(other.killSignal) +{ + other.release(); +} + Pid::Pid(pid_t pid) : pid(pid) { } Pid::~Pid() -{ +try { if (pid != -1) - kill(); + kill(/*allowInterrupts=*/false); +} catch (...) { + ignoreExceptionInDestructor(); } void Pid::operator=(pid_t pid) @@ -59,7 +71,7 @@ Pid::operator pid_t() return pid; } -int Pid::kill() +int Pid::kill(bool allowInterrupts) { assert(pid != -1); @@ -78,10 +90,10 @@ int Pid::kill() logError(SysError("killing process %d", pid).info()); } - return wait(); + return wait(allowInterrupts); } -int Pid::wait() +int Pid::wait(bool allowInterrupts) { assert(pid != -1); while (1) { @@ -93,7 +105,8 @@ int Pid::wait() } if (errno != EINTR) throw SysError("cannot get exit status of PID %d", pid); - checkInterrupt(); + if (allowInterrupts) + checkInterrupt(); } } @@ -110,7 +123,12 @@ void Pid::setKillSignal(int signal) pid_t Pid::release() { pid_t p = pid; + /* We use the move assignment operator rather than setting the individual fields so we aren't duplicating the + default values from the header, which would be hard to keep in sync. If we just used the assignment operator + without manually resetting pid first it would kill that process, however, so we do manually reset that one field. + */ pid = -1; + *this = Pid(); return p; } @@ -162,7 +180,7 @@ void killUser(uid_t uid) ////////////////////////////////////////////////////////////////////// -using ChildWrapperFunction = std::function; +using ChildWrapperFunction = fun; /* Wrapper around vfork to prevent the child process from clobbering the caller's stack frame in the parent. */ @@ -190,7 +208,7 @@ static int childEntry(void * arg) } #endif -pid_t startProcess(std::function fun, const ProcessOptions & options) +pid_t startProcess(fun processMain, const ProcessOptions & options) { auto newLogger = makeSimpleLogger(); ChildWrapperFunction wrapper = [&] { @@ -208,7 +226,7 @@ pid_t startProcess(std::function fun, const ProcessOptions & options) if (options.dieWithParent && prctl(PR_SET_PDEATHSIG, SIGKILL) == -1) throw SysError("setting death signal"); #endif - fun(); + processMain(); } catch (std::exception & e) { try { std::cerr << options.errorPrefix << e.what() << "\n"; @@ -251,7 +269,11 @@ pid_t startProcess(std::function fun, const ProcessOptions & options) } std::string runProgram( - Path program, bool lookupPath, const Strings & args, const std::optional & input, bool isInteractive) + std::filesystem::path program, + bool lookupPath, + const OsStrings & args, + const std::optional & input, + bool isInteractive) { auto res = runProgram( RunOptions{ @@ -262,7 +284,7 @@ std::string runProgram( .isInteractive = isInteractive}); if (!statusOk(res.first)) - throw ExecError(res.first, "program '%1%' %2%", program, statusToString(res.first)); + throw ExecError(res.first, "program %s %s", PathFmt(program), statusToString(res.first)); return res.second; } @@ -337,7 +359,7 @@ void runProgram2(const RunOptions & options) throw SysError("setuid failed"); Strings args_(options.args); - args_.push_front(options.program); + args_.push_front(options.program.native()); restoreProcessContext(); @@ -348,7 +370,7 @@ void runProgram2(const RunOptions & options) else execv(options.program.c_str(), stringsToCharPtrs(args_).data()); - throw SysError("executing '%1%'", options.program); + throw SysError("executing %s", PathFmt(options.program)); }, processOptions); @@ -396,7 +418,7 @@ void runProgram2(const RunOptions & options) promise.get_future().get(); if (status) - throw ExecError(status, "program '%1%' %2%", options.program, statusToString(status)); + throw ExecError(status, "program %1% %2%", PathFmt(options.program), statusToString(status)); } ////////////////////////////////////////////////////////////////////// diff --git a/src/libutil/unix/signals.cc b/src/libutil/unix/signals.cc index de441492a894..6f11010587a1 100644 --- a/src/libutil/unix/signals.cc +++ b/src/libutil/unix/signals.cc @@ -1,6 +1,7 @@ #include "nix/util/signals.hh" #include "nix/util/util.hh" #include "nix/util/error.hh" +#include "nix/util/fun.hh" #include "nix/util/sync.hh" #include "nix/util/terminal.hh" @@ -39,7 +40,7 @@ struct InterruptCallbacks Token nextToken = 0; /* Used as a list, see InterruptCallbacks comment. */ - std::map> callbacks; + std::map> callbacks; }; static Sync _interruptCallbacks; @@ -141,6 +142,16 @@ struct InterruptCallbackImpl : InterruptCallback { InterruptCallbacks::Token token; + InterruptCallbackImpl(InterruptCallbacks::Token token) + : token(token) + { + } + + InterruptCallbackImpl(InterruptCallbackImpl &&) = delete; + InterruptCallbackImpl(const InterruptCallbackImpl &) = delete; + InterruptCallbackImpl & operator=(InterruptCallbackImpl &&) = delete; + InterruptCallbackImpl & operator=(const InterruptCallbackImpl &) = delete; + ~InterruptCallbackImpl() override { auto interruptCallbacks(_interruptCallbacks.lock()); @@ -148,16 +159,12 @@ struct InterruptCallbackImpl : InterruptCallback } }; -std::unique_ptr createInterruptCallback(std::function callback) +std::unique_ptr createInterruptCallback(fun callback) { auto interruptCallbacks(_interruptCallbacks.lock()); auto token = interruptCallbacks->nextToken++; interruptCallbacks->callbacks.emplace(token, callback); - - std::unique_ptr res{new InterruptCallbackImpl{}}; - res->token = token; - - return std::unique_ptr(res.release()); + return std::make_unique(token); } } // namespace nix diff --git a/src/libutil/unix/users.cc b/src/libutil/unix/users.cc index 870bbe3767f5..5d05dc79f36f 100644 --- a/src/libutil/unix/users.cc +++ b/src/libutil/unix/users.cc @@ -35,18 +35,16 @@ std::filesystem::path getHome() auto homeDir = getEnv("HOME"); if (homeDir) { // Only use $HOME if doesn't exist or is owned by the current user. - struct stat st; - int result = stat(homeDir->c_str(), &st); - if (result != 0) { - if (errno != ENOENT) { - warn( - "couldn't stat $HOME ('%s') for reason other than not existing ('%d'), falling back to the one defined in the 'passwd' file", - *homeDir, - errno); - homeDir.reset(); - } - } else if (st.st_uid != geteuid()) { - unownedUserHomeDir.swap(homeDir); + try { + auto st = maybeStat(homeDir->c_str()); + if (st && st->st_uid != geteuid()) + unownedUserHomeDir.swap(homeDir); + } catch (SysError & e) { + warn( + "couldn't stat $HOME ('%s') for reason other than not existing, falling back to the one defined in the 'passwd' file: %s", + *homeDir, + e.what()); + homeDir.reset(); } } if (!homeDir) { diff --git a/src/libutil/unix/xdg-dirs.cc b/src/libutil/unix/xdg-dirs.cc new file mode 100644 index 000000000000..cc66f59f3a93 --- /dev/null +++ b/src/libutil/unix/xdg-dirs.cc @@ -0,0 +1,58 @@ +#include "nix/util/util.hh" +#include "nix/util/users.hh" +#include "nix/util/environment-variables.hh" + +namespace nix::unix::xdg { + +std::filesystem::path getCacheHome() +{ + auto xdgDir = getEnv("XDG_CACHE_HOME"); + if (xdgDir) { + return *xdgDir; + } else { + return getHome() / ".cache"; + } +} + +std::filesystem::path getConfigHome() +{ + auto xdgDir = getEnv("XDG_CONFIG_HOME"); + if (xdgDir) { + return *xdgDir; + } else { + return getHome() / ".config"; + } +} + +std::vector getConfigDirs() +{ + auto configDirs = getEnv("XDG_CONFIG_DIRS").value_or("/etc/xdg"); + auto tokens = tokenizeString>(configDirs, ":"); + std::vector result; + for (auto & token : tokens) { + result.push_back(std::filesystem::path{token}); + } + return result; +} + +std::filesystem::path getDataHome() +{ + auto xdgDir = getEnv("XDG_DATA_HOME"); + if (xdgDir) { + return *xdgDir; + } else { + return getHome() / ".local" / "share"; + } +} + +std::filesystem::path getStateHome() +{ + auto xdgDir = getEnv("XDG_STATE_HOME"); + if (xdgDir) { + return *xdgDir; + } else { + return getHome() / ".local" / "state"; + } +} + +} // namespace nix::unix::xdg diff --git a/src/libutil/unix/xdg-dirs.hh b/src/libutil/unix/xdg-dirs.hh new file mode 100644 index 000000000000..848e3f39a17a --- /dev/null +++ b/src/libutil/unix/xdg-dirs.hh @@ -0,0 +1,40 @@ +#pragma once +///@file +// Private header - not installed + +#include +#include + +namespace nix::unix::xdg { + +/** + * Get the XDG Base Directory for cache files. + * Returns $XDG_CACHE_HOME or ~/.cache + */ +std::filesystem::path getCacheHome(); + +/** + * Get the XDG Base Directory for configuration files. + * Returns $XDG_CONFIG_HOME or ~/.config + */ +std::filesystem::path getConfigHome(); + +/** + * Get the XDG Base Directory list for configuration files. + * Returns parsed $XDG_CONFIG_DIRS or /etc/xdg as a vector + */ +std::vector getConfigDirs(); + +/** + * Get the XDG Base Directory for data files. + * Returns $XDG_DATA_HOME or ~/.local/share + */ +std::filesystem::path getDataHome(); + +/** + * Get the XDG Base Directory for state files. + * Returns $XDG_STATE_HOME or ~/.local/state + */ +std::filesystem::path getStateHome(); + +} // namespace nix::unix::xdg diff --git a/src/libutil/url.cc b/src/libutil/url.cc index d9f61078cc48..f518af6897bd 100644 --- a/src/libutil/url.cc +++ b/src/libutil/url.cc @@ -321,19 +321,8 @@ std::string encodeQuery(const StringMap & ss) return res; } -Path renderUrlPathEnsureLegal(const std::vector & urlPath) +std::string renderUrlPathNoPctEncoding(std::span urlPath) { - for (const auto & comp : urlPath) { - /* This is only really valid for UNIX. Windows has more restrictions. */ - if (comp.contains('/')) - throw BadURL("URL path component '%s' contains '/', which is not allowed in file names", comp); - if (comp.contains(char(0))) { - using namespace std::string_view_literals; - auto str = replaceStrings(comp, "\0"sv, "␀"sv); - throw BadURL("URL path component '%s' contains NUL byte which is not allowed", str); - } - } - return concatStringsSep("/", urlPath); } @@ -341,7 +330,7 @@ std::string ParsedURL::renderPath(bool encode) const { if (encode) return encodeUrlPath(path); - return concatStringsSep("/", path); + return renderUrlPathNoPctEncoding(path); } std::string ParsedURL::renderSanitized() const @@ -426,12 +415,24 @@ ParsedURL fixGitURL(std::string url) std::regex scpRegex("([^/]*)@(.*):(.*)"); if (!hasPrefix(url, "/") && std::regex_match(url, scpRegex)) url = std::regex_replace(url, scpRegex, "ssh://$1@$2/$3"); - if (!hasPrefix(url, "file:") && !hasPrefix(url, "git+file:") && url.find("://") == std::string::npos) - return ParsedURL{ - .scheme = "file", - .authority = ParsedURL::Authority{}, - .path = splitString>(url, "/"), - }; + if (!hasPrefix(url, "file:") && !hasPrefix(url, "git+file:") && url.find("://") == std::string::npos) { + auto path = splitString>(url, "/"); + // Reject SCP-like URLs without user (e.g., "github.com:path") - colon in first component + if (!path.empty() && path[0].find(':') != std::string::npos) + throw BadURL("SCP-like URL '%s' is not supported; use SSH URL syntax instead (ssh://...)", url); + // Absolute paths get an empty authority (file:///path), relative paths get none (file:path) + if (hasPrefix(url, "/")) + return ParsedURL{ + .scheme = "file", + .authority = ParsedURL::Authority{}, + .path = path, + }; + else + return ParsedURL{ + .scheme = "file", + .path = path, + }; + } auto parsed = parseURL(url); // Drop the superfluous "git+" from the scheme. auto scheme = parseUrlScheme(parsed.scheme); @@ -449,6 +450,74 @@ bool isValidSchemeName(std::string_view s) return std::regex_match(s.begin(), s.end(), regex, std::regex_constants::match_default); } +std::vector pathToUrlPath(const std::filesystem::path & path) +{ + std::vector urlPath; + + // Prepend empty segment for absolute paths (those with a root directory) + if (path.has_root_directory()) + urlPath.push_back(""); + + // Handle Windows drive letter (root_name like "C:") + if (path.has_root_name()) + urlPath.push_back(path.root_name().generic_string()); + + // Iterate only over the relative path portion + for (const auto & component : path.relative_path()) + urlPath.push_back(component.generic_string()); + + // Add trailing empty segment for paths ending with separator (including root-only paths) + if (path.filename().empty()) + urlPath.push_back(""); + + return urlPath; +} + +std::filesystem::path urlPathToPath(std::span urlPath) +{ + for (const auto & comp : urlPath) { + /* This is only really valid for UNIX. Windows has more restrictions. */ + if (comp.contains('/')) + throw BadURL("URL path component '%s' contains '/', which is not allowed in file names", comp); + if (comp.contains(char(0))) { + using namespace std::string_view_literals; + auto str = replaceStrings(comp, "\0"sv, "␀"sv); + throw BadURL("URL path component '%s' contains NUL byte which is not allowed", str); + } + } + + std::filesystem::path result; + auto it = urlPath.begin(); + + if (it == urlPath.end()) + return result; + + // Empty first segment means absolute path (leading "/") + if (it->empty()) { + ++it; + result = "/"; +#ifdef _WIN32 + // On Windows, check if next segment is a drive letter (e.g., "C:"). + // If it isn't then this is something like a UNC path rather than a + // DOS path. + if (it != urlPath.end()) { + std::filesystem::path segment{*it}; + if (segment.has_root_name()) { + segment /= "/"; + result = std::move(segment); + ++it; + } + } +#endif + } + + // Append remaining segments + for (; it != urlPath.end(); ++it) + result /= *it; + + return result; +} + std::ostream & operator<<(std::ostream & os, const VerbatimURL & url) { os << url.to_string(); diff --git a/src/libutil/users.cc b/src/libutil/users.cc index 1fa643730cd1..f05dfcf760e2 100644 --- a/src/libutil/users.cc +++ b/src/libutil/users.cc @@ -1,81 +1,77 @@ #include "nix/util/util.hh" #include "nix/util/users.hh" #include "nix/util/environment-variables.hh" +#include "nix/util/executable-path.hh" #include "nix/util/file-system.hh" +#ifndef _WIN32 +# include "unix/xdg-dirs.hh" +#else +# include "nix/util/windows-known-folders.hh" +#endif + namespace nix { std::filesystem::path getCacheDir() { - auto dir = getEnv("NIX_CACHE_HOME"); - if (dir) { + auto dir = getEnvOs(OS_STR("NIX_CACHE_HOME")); + if (dir) return *dir; - } else { - auto xdgDir = getEnv("XDG_CACHE_HOME"); - if (xdgDir) { - return std::filesystem::path{*xdgDir} / "nix"; - } else { - return getHome() / ".cache" / "nix"; - } - } +#ifndef _WIN32 + return unix::xdg::getCacheHome() / "nix"; +#else + return windows::known_folders::getLocalAppData() / "nix" / "cache"; +#endif } std::filesystem::path getConfigDir() { - auto dir = getEnv("NIX_CONFIG_HOME"); - if (dir) { + auto dir = getEnvOs(OS_STR("NIX_CONFIG_HOME")); + if (dir) return *dir; - } else { - auto xdgDir = getEnv("XDG_CONFIG_HOME"); - if (xdgDir) { - return std::filesystem::path{*xdgDir} / "nix"; - } else { - return getHome() / ".config" / "nix"; - } - } +#ifndef _WIN32 + return unix::xdg::getConfigHome() / "nix"; +#else + return windows::known_folders::getRoamingAppData() / "nix" / "config"; +#endif } std::vector getConfigDirs() { std::filesystem::path configHome = getConfigDir(); - auto configDirs = getEnv("XDG_CONFIG_DIRS").value_or("/etc/xdg"); - auto tokens = tokenizeString>(configDirs, ":"); std::vector result; result.push_back(configHome); - for (auto & token : tokens) { - result.push_back(std::filesystem::path{token} / "nix"); +#ifndef _WIN32 + auto xdgConfigDirs = unix::xdg::getConfigDirs(); + for (auto & dir : xdgConfigDirs) { + result.push_back(dir / "nix"); } +#endif return result; } std::filesystem::path getDataDir() { - auto dir = getEnv("NIX_DATA_HOME"); - if (dir) { + auto dir = getEnvOs(OS_STR("NIX_DATA_HOME")); + if (dir) return *dir; - } else { - auto xdgDir = getEnv("XDG_DATA_HOME"); - if (xdgDir) { - return std::filesystem::path{*xdgDir} / "nix"; - } else { - return getHome() / ".local" / "share" / "nix"; - } - } +#ifndef _WIN32 + return unix::xdg::getDataHome() / "nix"; +#else + return windows::known_folders::getLocalAppData() / "nix" / "data"; +#endif } std::filesystem::path getStateDir() { - auto dir = getEnv("NIX_STATE_HOME"); - if (dir) { + auto dir = getEnvOs(OS_STR("NIX_STATE_HOME")); + if (dir) return *dir; - } else { - auto xdgDir = getEnv("XDG_STATE_HOME"); - if (xdgDir) { - return std::filesystem::path{*xdgDir} / "nix"; - } else { - return getHome() / ".local" / "state" / "nix"; - } - } +#ifndef _WIN32 + return unix::xdg::getStateHome() / "nix"; +#else + return windows::known_folders::getLocalAppData() / "nix" / "state"; +#endif } std::filesystem::path createNixStateDir() @@ -89,9 +85,10 @@ std::string expandTilde(std::string_view path) { // TODO: expand ~user ? auto tilde = path.substr(0, 2); - if (tilde == "~/" || tilde == "~") - return getHome().string() + std::string(path.substr(1)); - else + if (tilde == "~/" || tilde == "~") { + auto suffix = path.size() >= 2 ? std::string(path.substr(2)) : std::string{}; + return (getHome() / suffix).string(); + } else return std::string(path); } diff --git a/src/libutil/windows/current-process.cc b/src/libutil/windows/current-process.cc index 4bc866bb3efe..3d252257e197 100644 --- a/src/libutil/windows/current-process.cc +++ b/src/libutil/windows/current-process.cc @@ -1,5 +1,5 @@ #include "nix/util/current-process.hh" -#include "nix/util/windows-error.hh" +#include "nix/util/error.hh" #include #ifdef _WIN32 @@ -16,8 +16,7 @@ std::chrono::microseconds getCpuUserTime() FILETIME userTime; if (!GetProcessTimes(GetCurrentProcess(), &creationTime, &exitTime, &kernelTime, &userTime)) { - auto lastError = GetLastError(); - throw windows::WinError(lastError, "failed to get CPU time"); + throw windows::WinError("failed to get CPU time"); } ULARGE_INTEGER uLargeInt; diff --git a/src/libutil/windows/environment-variables.cc b/src/libutil/windows/environment-variables.cc index c76c12345538..1a93eca61987 100644 --- a/src/libutil/windows/environment-variables.cc +++ b/src/libutil/windows/environment-variables.cc @@ -1,7 +1,8 @@ #include "nix/util/environment-variables.hh" #ifdef _WIN32 -# include "processenv.h" +# include +# include namespace nix { @@ -30,11 +31,50 @@ std::optional getEnvOs(const OsString & key) return value; } +OsStringMap getEnvOs() +{ + OsStringMap env; + + auto freeStrings = [](wchar_t * strings) { FreeEnvironmentStringsW(strings); }; + auto envStrings = std::unique_ptr(GetEnvironmentStringsW(), freeStrings); + auto s = envStrings.get(); + + while (true) { + auto eq = StrChrW(s, L'='); + // Object ends with an empty string, which naturally won't have an = + if (eq == nullptr) + break; + + auto value_len = lstrlenW(eq + 1); + + env[OsString(s, eq - s)] = OsString(eq + 1, value_len); + + // 1 to skip L'=', then value, then 1 to skip L'\0' + s = eq + 1 + value_len + 1; + } + + return env; +} + +StringMap getEnv() +{ + StringMap env; + for (auto & [name, value] : getEnvOs()) { + env.emplace(os_string_to_string(name), os_string_to_string(value)); + } + return env; +} + int unsetenv(const char * name) { return -SetEnvironmentVariableA(name, nullptr); } +int unsetEnvOs(const OsChar * name) +{ + return -SetEnvironmentVariableW(name, nullptr); +} + int setEnv(const char * name, const char * value) { return -SetEnvironmentVariableA(name, value); diff --git a/src/libutil/windows/file-descriptor.cc b/src/libutil/windows/file-descriptor.cc index 3c3e7ea454ad..726859135193 100644 --- a/src/libutil/windows/file-descriptor.cc +++ b/src/libutil/windows/file-descriptor.cc @@ -2,103 +2,70 @@ #include "nix/util/signals.hh" #include "nix/util/finally.hh" #include "nix/util/serialise.hh" -#include "nix/util/windows-error.hh" -#include "nix/util/file-path.hh" -#ifdef _WIN32 -# include -# include -# include -# include -# define WIN32_LEAN_AND_MEAN -# include +#include + +#include +#include +#include +#include +#define WIN32_LEAN_AND_MEAN +#include namespace nix { using namespace nix::windows; -std::string readFile(HANDLE handle) +std::make_unsigned_t getFileSize(Descriptor fd) { LARGE_INTEGER li; - if (!GetFileSizeEx(handle, &li)) - throw WinError("%s:%d statting file", __FILE__, __LINE__); - - return drainFD(handle, true, li.QuadPart); -} - -void readFull(HANDLE handle, char * buf, size_t count) -{ - while (count) { - checkInterrupt(); - DWORD res; - if (!ReadFile(handle, (char *) buf, count, &res, NULL)) - throw WinError("%s:%d reading from file", __FILE__, __LINE__); - if (res == 0) - throw EndOfFile("unexpected end-of-file"); - count -= res; - buf += res; + if (!GetFileSizeEx(fd, &li)) { + throw WinError([&] { return HintFmt("getting size of file %s", PathFmt(descriptorToPath(fd))); }); } + return li.QuadPart; } -void writeFull(HANDLE handle, std::string_view s, bool allowInterrupts) +size_t read(Descriptor fd, std::span buffer) { - while (!s.empty()) { - if (allowInterrupts) - checkInterrupt(); - DWORD res; -# if _WIN32_WINNT >= 0x0600 - auto path = handleToPath(handle); // debug; do it before because handleToPath changes lasterror - if (!WriteFile(handle, s.data(), s.size(), &res, NULL)) { - throw WinError("writing to file %1%:%2%", handle, path); - } -# else - if (!WriteFile(handle, s.data(), s.size(), &res, NULL)) { - throw WinError("writing to file %1%", handle); - } -# endif - if (res > 0) - s.remove_prefix(res); + checkInterrupt(); // For consistency with unix, and its EINTR loop + DWORD n; + if (!ReadFile(fd, buffer.data(), static_cast(buffer.size()), &n, NULL)) { + auto lastError = GetLastError(); + if (lastError == ERROR_BROKEN_PIPE) + n = 0; // Treat as EOF + else + throw WinError(lastError, "reading %1% bytes from %2%", buffer.size(), PathFmt(descriptorToPath(fd))); } + return static_cast(n); } -std::string readLine(HANDLE handle, bool eofOk) +size_t readOffset(Descriptor fd, off_t offset, std::span buffer) { - std::string s; - while (1) { - checkInterrupt(); - char ch; - // FIXME: inefficient - DWORD rd; - if (!ReadFile(handle, &ch, 1, &rd, NULL)) { - throw WinError("reading a line"); - } else if (rd == 0) { - if (eofOk) - return s; - else - throw EndOfFile("unexpected EOF reading a line"); - } else { - if (ch == '\n') - return s; - s += ch; - } + checkInterrupt(); // For consistency with unix, and its EINTR loop + OVERLAPPED ov = {}; + ov.Offset = static_cast(offset); + if constexpr (sizeof(offset) > 4) /* We don't build with 32 bit off_t, but let's be safe. */ + ov.OffsetHigh = static_cast(offset >> 32); + DWORD n; + if (!ReadFile(fd, buffer.data(), static_cast(buffer.size()), &n, &ov)) { + throw WinError([&] { + return HintFmt( + "reading %1% bytes at offset %2% from %3%", buffer.size(), offset, PathFmt(descriptorToPath(fd))); + }); } + return static_cast(n); } -void drainFD(HANDLE handle, Sink & sink /*, bool block*/) +size_t write(Descriptor fd, std::span buffer, bool allowInterrupts) { - std::vector buf(64 * 1024); - while (1) { - checkInterrupt(); - DWORD rd; - if (!ReadFile(handle, buf.data(), buf.size(), &rd, NULL)) { - WinError winError("%s:%d reading from handle %p", __FILE__, __LINE__, handle); - if (winError.lastError == ERROR_BROKEN_PIPE) - break; - throw winError; - } else if (rd == 0) - break; - sink({(char *) buf.data(), (size_t) rd}); + if (allowInterrupts) + checkInterrupt(); // For consistency with unix + DWORD n; + if (!WriteFile(fd, buffer.data(), static_cast(buffer.size()), &n, NULL)) { + throw WinError( + [&] { return HintFmt("writing %1% bytes to %2%", buffer.size(), PathFmt(descriptorToPath(fd))); }); } + return static_cast(n); } ////////////////////////////////////////////////////////////////////// @@ -120,36 +87,42 @@ void Pipe::create() ////////////////////////////////////////////////////////////////////// -# if _WIN32_WINNT >= 0x0600 - -std::wstring windows::handleToFileName(HANDLE handle) +off_t lseek(HANDLE h, off_t offset, int whence) { - std::vector buf(0x100); - DWORD dw = GetFinalPathNameByHandleW(handle, buf.data(), buf.size(), FILE_NAME_OPENED); - if (dw == 0) { - if (handle == GetStdHandle(STD_INPUT_HANDLE)) - return L""; - if (handle == GetStdHandle(STD_OUTPUT_HANDLE)) - return L""; - if (handle == GetStdHandle(STD_ERROR_HANDLE)) - return L""; - return (boost::wformat(L"") % handle).str(); + DWORD method; + switch (whence) { + case SEEK_SET: + method = FILE_BEGIN; + break; + case SEEK_CUR: + method = FILE_CURRENT; + break; + case SEEK_END: + method = FILE_END; + break; + default: + throw Error("lseek: invalid whence %d", whence); } - if (dw > buf.size()) { - buf.resize(dw); - if (GetFinalPathNameByHandleW(handle, buf.data(), buf.size(), FILE_NAME_OPENED) != dw - 1) - throw WinError("GetFinalPathNameByHandleW"); - dw -= 1; + + LARGE_INTEGER li; + li.QuadPart = offset; + LARGE_INTEGER newPos; + + if (!SetFilePointerEx(h, li, &newPos, method)) { + /* Convert to a POSIX error, since caller code works with this as if it were + a POSIX lseek. */ + errno = std::error_code(GetLastError(), std::system_category()).default_error_condition().value(); + return -1; } - return std::wstring(buf.data(), dw); + + return newPos.QuadPart; } -Path windows::handleToPath(HANDLE handle) +void syncDescriptor(Descriptor fd) { - return os_string_to_string(handleToFileName(handle)); + if (!::FlushFileBuffers(fd)) { + throw WinError([&] { return HintFmt("flushing file %s", PathFmt(descriptorToPath(fd))); }); + } } -# endif - } // namespace nix -#endif diff --git a/src/libutil/windows/file-path.cc b/src/libutil/windows/file-path.cc index 7913b3d5d282..afe43e060b55 100644 --- a/src/libutil/windows/file-path.cc +++ b/src/libutil/windows/file-path.cc @@ -13,26 +13,26 @@ std::optional maybePath(PathView path) { if (path.length() >= 3 && (('A' <= path[0] && path[0] <= 'Z') || ('a' <= path[0] && path[0] <= 'z')) && path[1] == ':' && WindowsPathTrait::isPathSep(path[2])) { - std::filesystem::path::string_type sw = string_to_os_string(std::string{"\\\\?\\"} + path); + std::filesystem::path::string_type sw = std::wstring{L"\\\\?\\"} + std::wstring{path}; std::replace(sw.begin(), sw.end(), '/', '\\'); return sw; } if (path.length() >= 7 && path[0] == '\\' && path[1] == '\\' && (path[2] == '.' || path[2] == '?') && path[3] == '\\' && ('A' <= path[4] && path[4] <= 'Z') && path[5] == ':' && WindowsPathTrait::isPathSep(path[6])) { - std::filesystem::path::string_type sw = string_to_os_string(path); + std::filesystem::path::string_type sw{path}; std::replace(sw.begin(), sw.end(), '/', '\\'); return sw; } return std::optional(); } -std::filesystem::path pathNG(PathView path) +std::filesystem::path toOwnedPath(PathView path) { std::optional sw = maybePath(path); if (!sw) { // FIXME why are we not using the regular error handling? - std::cerr << "invalid path for WinAPI call [" << path << "]" << std::endl; + std::wcerr << L"invalid path for WinAPI call [" << path << L"]" << std::endl; _exit(111); } return *sw; diff --git a/src/libutil/windows/file-system-at.cc b/src/libutil/windows/file-system-at.cc new file mode 100644 index 000000000000..ebbe840007d0 --- /dev/null +++ b/src/libutil/windows/file-system-at.cc @@ -0,0 +1,309 @@ +#include "nix/util/file-system-at.hh" +#include "nix/util/file-system.hh" +#include "nix/util/signals.hh" +#include "nix/util/file-path.hh" +#include "nix/util/source-accessor.hh" + +#include +#include +#define WIN32_LEAN_AND_MEAN +#include +#include +#include + +namespace nix { + +using namespace nix::windows; + +namespace windows { + +namespace { + +/** + * Open a file/directory relative to a directory handle using NtCreateFile. + * + * @param dirFd Directory handle to open relative to + * @param pathComponent Single path component (not a full path) + * @param desiredAccess Access rights requested + * @param createOptions NT create options flags + * @param createDisposition FILE_OPEN, FILE_CREATE, etc. + * @return Handle to the opened file/directory (caller must close) + */ +AutoCloseFD ntOpenAt( + Descriptor dirFd, + std::wstring_view pathComponent, + ACCESS_MASK desiredAccess, + ULONG createOptions, + ULONG createDisposition = FILE_OPEN) +{ + /* Set up UNICODE_STRING for the relative path */ + UNICODE_STRING pathStr; + pathStr.Buffer = const_cast(pathComponent.data()); + pathStr.Length = static_cast(pathComponent.size() * sizeof(wchar_t)); + pathStr.MaximumLength = pathStr.Length; + + /* Set up OBJECT_ATTRIBUTES to open relative to dirFd */ + OBJECT_ATTRIBUTES objAttrs; + InitializeObjectAttributes( + &objAttrs, + &pathStr, + 0, // No special flags + dirFd, // RootDirectory + nullptr // No security descriptor + ); + + /* Open using NT API */ + IO_STATUS_BLOCK ioStatus; + HANDLE h; + NTSTATUS status = NtCreateFile( + &h, + desiredAccess, + &objAttrs, + &ioStatus, + nullptr, // No allocation size + FILE_ATTRIBUTE_NORMAL, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + createDisposition, + createOptions | FILE_SYNCHRONOUS_IO_NONALERT, + nullptr, // No EA buffer + 0 // No EA length + ); + + if (status != 0) + throw WinError( + RtlNtStatusToDosError(status), "opening %s relative to directory handle", PathFmt(pathComponent)); + + return AutoCloseFD{h}; +} + +/** + * Open a symlink relative to a directory handle without following it. + * + * @param dirFd Directory handle to open relative to + * @param path Relative path to the symlink + * @return Handle to the symlink + */ +AutoCloseFD openSymlinkAt(Descriptor dirFd, const CanonPath & path) +{ + assert(!path.isRoot()); + assert(!path.rel().starts_with('/')); /* Just in case the invariant is somehow broken. */ + + std::wstring wpath = string_to_os_string(path.rel()); + return ntOpenAt(dirFd, wpath, FILE_READ_ATTRIBUTES | SYNCHRONIZE, FILE_OPEN_REPARSE_POINT); +} + +/** + * This struct isn't defined in the normal Windows SDK, but only in the Windows Driver Kit. + * + * I (@Ericson2314) would not normally do something like this, but LLVM + * has decided that this is in fact stable, per + * https://github.com/llvm/llvm-project/blob/main/libcxx/src/filesystem/posix_compat.h, + * so I guess that is good enough for us. GCC doesn't support symlinks + * at all on windows so we have to put it here, not grab it from private + * c++ standard library headers anyways. + */ +struct ReparseDataBuffer +{ + unsigned long ReparseTag; + unsigned short ReparseDataLength; + unsigned short Reserved; + + union + { + struct + { + unsigned short SubstituteNameOffset; + unsigned short SubstituteNameLength; + unsigned short PrintNameOffset; + unsigned short PrintNameLength; + unsigned long Flags; + wchar_t PathBuffer[1]; + } SymbolicLinkReparseBuffer; + + struct + { + unsigned short SubstituteNameOffset; + unsigned short SubstituteNameLength; + unsigned short PrintNameOffset; + unsigned short PrintNameLength; + wchar_t PathBuffer[1]; + } MountPointReparseBuffer; + + struct + { + unsigned char DataBuffer[1]; + } GenericReparseBuffer; + }; +}; + +/** + * Read the target of a symlink from an open handle. + * + * @param linkHandle Handle to a symlink (must have been opened with FILE_OPEN_REPARSE_POINT) + * @return The symlink target as a wide string + */ +OsString readSymlinkTarget(HANDLE linkHandle) +{ + uint8_t buf[MAXIMUM_REPARSE_DATA_BUFFER_SIZE]; + DWORD out; + + checkInterrupt(); + + if (!DeviceIoControl(linkHandle, FSCTL_GET_REPARSE_POINT, nullptr, 0, buf, sizeof(buf), &out, nullptr)) + throw WinError("reading reparse point for handle %d", linkHandle); + + const auto * reparse = reinterpret_cast(buf); + size_t path_buf_offset = offsetof(ReparseDataBuffer, SymbolicLinkReparseBuffer.PathBuffer[0]); + + if (out < path_buf_offset) { + auto fullPath = descriptorToPath(linkHandle); + throw WinError( + DWORD{ERROR_REPARSE_TAG_INVALID}, "invalid reparse data for %d:%s", linkHandle, PathFmt(fullPath)); + } + + if (reparse->ReparseTag != IO_REPARSE_TAG_SYMLINK) { + auto fullPath = descriptorToPath(linkHandle); + throw WinError(DWORD{ERROR_REPARSE_TAG_INVALID}, "not a symlink: %d:%s", linkHandle, PathFmt(fullPath)); + } + + const auto & symlink = reparse->SymbolicLinkReparseBuffer; + unsigned short name_offset, name_length; + + /* Prefer PrintName over SubstituteName if available */ + if (symlink.PrintNameLength == 0) { + name_offset = symlink.SubstituteNameOffset; + name_length = symlink.SubstituteNameLength; + } else { + name_offset = symlink.PrintNameOffset; + name_length = symlink.PrintNameLength; + } + + if (path_buf_offset + name_offset + name_length > out) { + auto fullPath = descriptorToPath(linkHandle); + throw WinError( + DWORD{ERROR_REPARSE_TAG_INVALID}, "invalid symlink data for %d:%s", linkHandle, PathFmt(fullPath)); + } + + /* Extract the target path */ + const wchar_t * target_start = &symlink.PathBuffer[name_offset / sizeof(wchar_t)]; + size_t target_len = name_length / sizeof(wchar_t); + + return {target_start, target_len}; +} + +/** + * Check if a handle refers to a reparse point (e.g., symlink). + * + * @param handle Open file/directory handle + * @return true if the handle refers to a reparse point + */ +bool isReparsePoint(HANDLE handle) +{ + FILE_BASIC_INFO basicInfo; + if (!GetFileInformationByHandleEx(handle, FileBasicInfo, &basicInfo, sizeof(basicInfo))) + throw WinError("GetFileInformationByHandleEx"); + + return (basicInfo.FileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) != 0; +} + +} // anonymous namespace + +} // namespace windows + +AutoCloseFD openFileEnsureBeneathNoSymlinks( + Descriptor dirFd, const CanonPath & path, ACCESS_MASK desiredAccess, ULONG createOptions, ULONG createDisposition) +{ + assert(!path.isRoot()); + assert(!path.rel().starts_with('/')); /* Just in case the invariant is somehow broken. */ + + AutoCloseFD parentFd; + auto nrComponents = std::ranges::distance(path); + assert(nrComponents >= 1); + auto components = std::views::take(path, nrComponents - 1); /* Everything but last component */ + auto getParentFd = [&]() { return parentFd ? parentFd.get() : dirFd; }; + + /* Helper to construct CanonPath from components up to (and including) the given iterator */ + auto pathUpTo = [&](auto it) { + return std::ranges::fold_left(components.begin(), it, CanonPath::root, [](auto lhs, auto rhs) { + lhs.push(rhs); + return lhs; + }); + }; + + /* Helper to check if a component is a symlink and throw SymlinkNotAllowed if so */ + auto throwIfSymlink = [&](std::wstring_view component, const CanonPath & pathForError) { + try { + auto testHandle = + ntOpenAt(getParentFd(), component, FILE_READ_ATTRIBUTES | SYNCHRONIZE, FILE_OPEN_REPARSE_POINT); + if (isReparsePoint(testHandle.get())) + throw SymlinkNotAllowed(pathForError); + } catch (SymlinkNotAllowed &) { + throw; + } catch (...) { + /* If we can't determine, ignore and let caller handle original error */ + } + }; + + /* Iterate through each path component to ensure no symlinks in intermediate directories. + * This prevents TOCTOU issues by opening each component relative to the parent. */ + for (auto it = components.begin(); it != components.end(); ++it) { + std::wstring wcomponent = string_to_os_string(std::string(*it)); + + /* Open directory without following symlinks */ + AutoCloseFD parentFd2; + try { + parentFd2 = ntOpenAt( + getParentFd(), + wcomponent, + FILE_TRAVERSE | SYNCHRONIZE, // Just need traversal rights + FILE_DIRECTORY_FILE | FILE_OPEN_REPARSE_POINT // Open directory, don't follow symlinks + ); + } catch (WinError & e) { + /* Check if this is because it's a symlink */ + if (e.lastError == ERROR_CANT_ACCESS_FILE || e.lastError == ERROR_ACCESS_DENIED) { + throwIfSymlink(wcomponent, pathUpTo(std::next(it))); + } + throw; + } + + /* Check if what we opened is actually a symlink */ + if (isReparsePoint(parentFd2.get())) { + throw SymlinkNotAllowed(pathUpTo(std::next(it))); + } + + parentFd = std::move(parentFd2); + } + + /* Now open the final component with requested flags */ + std::wstring finalComponent = string_to_os_string(std::string(path.baseName().value())); + + AutoCloseFD finalHandle; + try { + finalHandle = ntOpenAt( + getParentFd(), + finalComponent, + desiredAccess, + createOptions | FILE_OPEN_REPARSE_POINT, // Don't follow symlinks on final component either + createDisposition); + } catch (WinError & e) { + /* Check if final component is a symlink when we requested to not follow it */ + if (e.lastError == ERROR_CANT_ACCESS_FILE) { + throwIfSymlink(finalComponent, path); + } + throw; + } + + /* Final check: did we accidentally open a symlink? */ + if (isReparsePoint(finalHandle.get())) + throw SymlinkNotAllowed(path); + + return finalHandle; +} + +OsString readLinkAt(Descriptor dirFd, const CanonPath & path) +{ + AutoCloseFD linkHandle(windows::openSymlinkAt(dirFd, path)); + return windows::readSymlinkTarget(linkHandle.get()); +} + +} // namespace nix diff --git a/src/libutil/windows/file-system.cc b/src/libutil/windows/file-system.cc index 22572772e26a..a707343ecedf 100644 --- a/src/libutil/windows/file-system.cc +++ b/src/libutil/windows/file-system.cc @@ -1,9 +1,16 @@ #include "nix/util/file-system.hh" #include "nix/util/logging.hh" +#include "nix/util/signals.hh" + +#define WIN32_LEAN_AND_MEAN +#include + +#include -#ifdef _WIN32 namespace nix { +using namespace nix::windows; + void setWriteTime( const std::filesystem::path & path, time_t accessedTime, time_t modificationTime, std::optional optIsSymlink) { @@ -13,20 +20,89 @@ void setWriteTime( // doesn't support access time just modification time. // // System clock vs File clock issues also make that annoying. - warn("Changing file times is not yet implemented on Windows, path is %s", path); + warn("Changing file times is not yet implemented on Windows, path is %s", PathFmt(path)); } -Descriptor openDirectory(const std::filesystem::path & path) +AutoCloseFD openDirectory(const std::filesystem::path & path) { - return CreateFileW( + return AutoCloseFD{CreateFileW( path.c_str(), GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, - NULL, + /*lpSecurityAttributes=*/nullptr, OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS, - NULL); + /*hTemplateFile=*/nullptr)}; +} + +AutoCloseFD openFileReadonly(const std::filesystem::path & path) +{ + return AutoCloseFD{CreateFileW( + path.c_str(), + GENERIC_READ, + FILE_SHARE_READ | FILE_SHARE_DELETE, + /*lpSecurityAttributes=*/nullptr, + OPEN_EXISTING, + FILE_ATTRIBUTE_NORMAL, + /*hTemplateFile=*/nullptr)}; +} + +AutoCloseFD +openNewFileForWrite(const std::filesystem::path & path, [[maybe_unused]] mode_t mode, OpenNewFileForWriteParams params) +{ + return AutoCloseFD{CreateFileW( + path.c_str(), + GENERIC_WRITE, + FILE_SHARE_READ | FILE_SHARE_DELETE, + /*lpSecurityAttributes=*/nullptr, + params.truncateExisting ? CREATE_ALWAYS : CREATE_NEW, /* TODO: Reparse points. */ + FILE_ATTRIBUTE_NORMAL, + /*hTemplateFile=*/nullptr)}; +} + +std::filesystem::path defaultTempDir() +{ + wchar_t buf[MAX_PATH + 1]; + DWORD len = GetTempPathW(MAX_PATH + 1, buf); + if (len == 0 || len > MAX_PATH) + throw WinError("getting default temporary directory"); + return std::filesystem::path(buf); +} + +void deletePath(const std::filesystem::path & path) +{ + std::error_code ec; + std::filesystem::remove_all(path, ec); + if (ec && ec != std::errc::no_such_file_or_directory) + throw SysError(ec.default_error_condition().value(), "recursively deleting %1%", PathFmt(path)); +} + +void deletePath(const std::filesystem::path & path, uint64_t & bytesFreed) +{ + bytesFreed = 0; + deletePath(path); +} + +std::filesystem::path descriptorToPath(Descriptor handle) +{ + std::vector buf(0x100); + DWORD dw = GetFinalPathNameByHandleW(handle, buf.data(), buf.size(), FILE_NAME_OPENED); + if (dw == 0) { + if (handle == GetStdHandle(STD_INPUT_HANDLE)) + return L""; + if (handle == GetStdHandle(STD_OUTPUT_HANDLE)) + return L""; + if (handle == GetStdHandle(STD_ERROR_HANDLE)) + return L""; + return (boost::wformat(L"") % handle).str(); + } + if (dw > buf.size()) { + buf.resize(dw); + if (GetFinalPathNameByHandleW(handle, buf.data(), buf.size(), FILE_NAME_OPENED) != dw - 1) + throw WinError("GetFinalPathNameByHandleW"); + dw -= 1; + } + return std::filesystem::path{std::wstring{buf.data(), dw}}; } } // namespace nix -#endif diff --git a/src/libutil/windows/include/nix/util/meson.build b/src/libutil/windows/include/nix/util/meson.build index 1bd56c4bd177..15f958ec7441 100644 --- a/src/libutil/windows/include/nix/util/meson.build +++ b/src/libutil/windows/include/nix/util/meson.build @@ -5,5 +5,6 @@ include_dirs += include_directories('../..') headers += files( 'signals-impl.hh', 'windows-async-pipe.hh', - 'windows-error.hh', + 'windows-environment.hh', + 'windows-known-folders.hh', ) diff --git a/src/libutil/windows/include/nix/util/windows-environment.hh b/src/libutil/windows/include/nix/util/windows-environment.hh new file mode 100644 index 000000000000..efbfd448aa13 --- /dev/null +++ b/src/libutil/windows/include/nix/util/windows-environment.hh @@ -0,0 +1,12 @@ +#pragma once + +///@file + +namespace nix::windows { + +/** + * Check if the current process is running under Wine. + */ +bool isWine(); + +} // namespace nix::windows diff --git a/src/libutil/windows/include/nix/util/windows-error.hh b/src/libutil/windows/include/nix/util/windows-error.hh deleted file mode 100644 index a45425ad4354..000000000000 --- a/src/libutil/windows/include/nix/util/windows-error.hh +++ /dev/null @@ -1,54 +0,0 @@ -#pragma once -///@file - -#ifdef _WIN32 -# include - -# include "nix/util/error.hh" - -namespace nix::windows { - -/** - * Windows Error type. - * - * Unless you need to catch a specific error number, don't catch this in - * portable code. Catch `SystemError` instead. - */ -class WinError : public SystemError -{ -public: - DWORD lastError; - - /** - * Construct using the explicitly-provided error number. - * `FormatMessageA` will be used to try to add additional - * information to the message. - */ - template - WinError(DWORD lastError, const Args &... args) - : SystemError("") - , lastError(lastError) - { - auto hf = HintFmt(args...); - err.msg = HintFmt("%1%: %2%", Uncolored(hf.str()), renderError(lastError)); - } - - /** - * Construct using `GetLastError()` and the ambient "last error". - * - * Be sure to not perform another last-error-modifying operation - * before calling this constructor! - */ - template - WinError(const Args &... args) - : WinError(GetLastError(), args...) - { - } - -private: - - std::string renderError(DWORD lastError); -}; - -} // namespace nix::windows -#endif diff --git a/src/libutil/windows/include/nix/util/windows-known-folders.hh b/src/libutil/windows/include/nix/util/windows-known-folders.hh new file mode 100644 index 000000000000..ea0a844eff7a --- /dev/null +++ b/src/libutil/windows/include/nix/util/windows-known-folders.hh @@ -0,0 +1,23 @@ +#pragma once +///@file + +#include + +namespace nix::windows::known_folders { + +/** + * Get the Windows LocalAppData known folder. + */ +std::filesystem::path getLocalAppData(); + +/** + * Get the Windows RoamingAppData known folder. + */ +std::filesystem::path getRoamingAppData(); + +/** + * Get the Windows ProgramData known folder. + */ +std::filesystem::path getProgramData(); + +} // namespace nix::windows::known_folders diff --git a/src/libutil/windows/known-folders.cc b/src/libutil/windows/known-folders.cc new file mode 100644 index 000000000000..1d2867428459 --- /dev/null +++ b/src/libutil/windows/known-folders.cc @@ -0,0 +1,41 @@ +#include "nix/util/windows-known-folders.hh" +#include "nix/util/util.hh" +#include "nix/util/users.hh" + +#define WIN32_LEAN_AND_MEAN +#include +#include + +namespace nix::windows::known_folders { + +using namespace nix::windows; + +static std::filesystem::path getKnownFolder(REFKNOWNFOLDERID rfid) +{ + PWSTR str = nullptr; + auto res = SHGetKnownFolderPath(rfid, /*dwFlags=*/0, /*hToken=*/nullptr, &str); + Finally cleanup([&]() { CoTaskMemFree(str); }); + if (SUCCEEDED(res)) + return std::filesystem::path(str); + throw WinError(static_cast(res), "failed to get known folder path"); +} + +std::filesystem::path getLocalAppData() +{ + static const auto path = getKnownFolder(FOLDERID_LocalAppData); + return path; +} + +std::filesystem::path getRoamingAppData() +{ + static const auto path = getKnownFolder(FOLDERID_RoamingAppData); + return path; +} + +std::filesystem::path getProgramData() +{ + static const auto path = getKnownFolder(FOLDERID_ProgramData); + return path; +} + +} // namespace nix::windows::known_folders diff --git a/src/libutil/windows/meson.build b/src/libutil/windows/meson.build index fb4de2017d79..9aa61e61514f 100644 --- a/src/libutil/windows/meson.build +++ b/src/libutil/windows/meson.build @@ -3,12 +3,15 @@ sources += files( 'environment-variables.cc', 'file-descriptor.cc', 'file-path.cc', + 'file-system-at.cc', 'file-system.cc', + 'known-folders.cc', 'muxable-pipe.cc', 'os-string.cc', 'processes.cc', 'users.cc', 'windows-async-pipe.cc', + 'windows-environment.cc', 'windows-error.cc', ) diff --git a/src/libutil/windows/muxable-pipe.cc b/src/libutil/windows/muxable-pipe.cc index b2eff70e6113..5c427d67aec1 100644 --- a/src/libutil/windows/muxable-pipe.cc +++ b/src/libutil/windows/muxable-pipe.cc @@ -1,6 +1,5 @@ #ifdef _WIN32 # include -# include "nix/util/windows-error.hh" # include "nix/util/logging.hh" # include "nix/util/util.hh" @@ -8,15 +7,17 @@ namespace nix { +using namespace nix::windows; + void MuxablePipePollState::poll(HANDLE ioport, std::optional timeout) { /* We are on at least Windows Vista / Server 2008 and can get many (countof(oentries)) statuses in one API call. */ if (!GetQueuedCompletionStatusEx( ioport, oentries, sizeof(oentries) / sizeof(*oentries), &removed, timeout ? *timeout : INFINITE, false)) { - windows::WinError winError("GetQueuedCompletionStatusEx"); - if (winError.lastError != WAIT_TIMEOUT) - throw winError; + auto lastError = GetLastError(); + if (lastError != WAIT_TIMEOUT) + throw WinError(lastError, "GetQueuedCompletionStatusEx"); assert(removed == 0); } else { assert(0 < removed && removed <= sizeof(oentries) / sizeof(*oentries)); @@ -25,8 +26,8 @@ void MuxablePipePollState::poll(HANDLE ioport, std::optional timeo void MuxablePipePollState::iterate( std::set & channels, - std::function handleRead, - std::function handleEOF) + fun handleRead, + fun handleEOF) { auto p = channels.begin(); while (p != channels.end()) { @@ -53,12 +54,12 @@ void MuxablePipePollState::iterate( // here is possible (but not obligatory) to call // `handleRead` and repeat ReadFile immediately } else { - windows::WinError winError("ReadFile(%s, ..)", (*p)->readSide.get()); - if (winError.lastError == ERROR_BROKEN_PIPE) { + auto lastError = GetLastError(); + if (lastError == ERROR_BROKEN_PIPE) { handleEOF((*p)->readSide.get()); nextp = channels.erase(p); // no need to maintain `channels` ? - } else if (winError.lastError != ERROR_IO_PENDING) - throw winError; + } else if (lastError != ERROR_IO_PENDING) + throw WinError(lastError, "ReadFile(%s, ..)", (*p)->readSide.get()); } } break; diff --git a/src/libutil/windows/os-string.cc b/src/libutil/windows/os-string.cc index d6f8e36705cc..2fface419552 100644 --- a/src/libutil/windows/os-string.cc +++ b/src/libutil/windows/os-string.cc @@ -3,24 +3,32 @@ #include #include -#include "nix/util/file-path.hh" -#include "nix/util/file-path-impl.hh" -#include "nix/util/util.hh" +#include "nix/util/os-string.hh" #ifdef _WIN32 namespace nix { -std::string os_string_to_string(PathViewNG::string_view path) +std::string os_string_to_string(OsStringView s) { std::wstring_convert> converter; - return converter.to_bytes(std::filesystem::path::string_type{path}); + return converter.to_bytes(s.data(), s.data() + s.size()); } -std::filesystem::path::string_type string_to_os_string(std::string_view s) +std::string os_string_to_string(OsString s) +{ + return os_string_to_string(OsStringView{s}); +} + +OsString string_to_os_string(std::string_view s) { std::wstring_convert> converter; - return converter.from_bytes(std::string{s}); + return converter.from_bytes(s.data(), s.data() + s.size()); +} + +OsString string_to_os_string(std::string s) +{ + return string_to_os_string(std::string_view{s}); } } // namespace nix diff --git a/src/libutil/windows/processes.cc b/src/libutil/windows/processes.cc index f8f2900e55db..9c7fa5d35ffb 100644 --- a/src/libutil/windows/processes.cc +++ b/src/libutil/windows/processes.cc @@ -4,17 +4,19 @@ #include "nix/util/executable-path.hh" #include "nix/util/file-descriptor.hh" #include "nix/util/file-path.hh" +#include "nix/util/fmt.hh" +#include "nix/util/os-string.hh" #include "nix/util/signals.hh" #include "nix/util/processes.hh" #include "nix/util/finally.hh" #include "nix/util/serialise.hh" #include "nix/util/file-system.hh" #include "nix/util/util.hh" -#include "nix/util/windows-error.hh" #include #include #include +#include #include #include #include @@ -34,6 +36,11 @@ using namespace nix::windows; Pid::Pid() {} +Pid::Pid(Pid && other) noexcept + : pid(std::move(other.pid)) +{ +} + Pid::Pid(AutoCloseFD pid) : pid(std::move(pid)) { @@ -52,29 +59,30 @@ void Pid::operator=(AutoCloseFD pid) this->pid = std::move(pid); } -// TODO: Implement (not needed for process spawning yet) -int Pid::kill() +int Pid::kill(bool allowInterrupts) { assert(pid.get() != INVALID_DESCRIPTOR); debug("killing process %1%", pid.get()); - throw UnimplementedError("Pid::kill unimplemented"); + if (!TerminateProcess(pid.get(), 1)) + logError(WinError("terminating process %1%", pid.get()).info()); + + return wait(allowInterrupts); } -int Pid::wait() +// Note that `allowInterrupts` is ignored for now, but there to match +// Unix. +int Pid::wait(bool allowInterrupts) { - // https://github.com/nix-windows/nix/blob/windows-meson/src/libutil/util.cc#L1938 assert(pid.get() != INVALID_DESCRIPTOR); DWORD status = WaitForSingleObject(pid.get(), INFINITE); - if (status != WAIT_OBJECT_0) { - debug("WaitForSingleObject returned %1%", status); - } + if (status != WAIT_OBJECT_0) + throw WinError("waiting for process %1%", pid.get()); DWORD exitCode = 0; - if (GetExitCodeProcess(pid.get(), &exitCode) == FALSE) { - debug("GetExitCodeProcess failed on pid %1%", pid.get()); - } + if (GetExitCodeProcess(pid.get(), &exitCode) == FALSE) + throw WinError("getting exit code of process %1%", pid.get()); pid.close(); return exitCode; @@ -82,7 +90,11 @@ int Pid::wait() // TODO: Merge this with Unix's runProgram since it's identical logic. std::string runProgram( - Path program, bool lookupPath, const Strings & args, const std::optional & input, bool isInteractive) + std::filesystem::path program, + bool lookupPath, + const OsStrings & args, + const std::optional & input, + bool isInteractive) { auto res = runProgram( RunOptions{ @@ -93,17 +105,17 @@ std::string runProgram( .isInteractive = isInteractive}); if (!statusOk(res.first)) - throw ExecError(res.first, "program '%1%' %2%", program, statusToString(res.first)); + throw ExecError(res.first, "program %s %s", PathFmt(program), statusToString(res.first)); return res.second; } -std::optional getProgramInterpreter(const Path & program) +std::optional getProgramInterpreter(const std::filesystem::path & program) { // These extensions are automatically handled by Windows and don't require an interpreter. static constexpr const char * exts[] = {".exe", ".cmd", ".bat"}; for (const auto ext : exts) { - if (hasSuffix(program, ext)) { + if (hasSuffix(program.string(), ext)) { return {}; } } @@ -145,23 +157,23 @@ AutoCloseFD nullFD() // Adapted from // https://blogs.msdn.microsoft.com/twistylittlepassagesallalike/2011/04/23/everyone-quotes-command-line-arguments-the-wrong-way/ -std::string windowsEscape(const std::string & str, bool cmd) +OsString windowsEscape(const OsString & str, bool cmd) { // TODO: This doesn't handle cmd.exe escaping. if (cmd) { throw UnimplementedError("cmd.exe escaping is not implemented"); } - if (str.find_first_of(" \t\n\v\"") == str.npos && !str.empty()) { + if (str.find_first_of(L" \t\n\v\"") == str.npos && !str.empty()) { // No need to escape this one, the nonempty contents don't have a special character return str; } - std::string buffer; + OsString buffer; // Add the opening quote - buffer += '"'; + buffer += L'"'; for (auto iter = str.begin();; ++iter) { size_t backslashes = 0; - while (iter != str.end() && *iter == '\\') { + while (iter != str.end() && *iter == L'\\') { ++iter; ++backslashes; } @@ -172,24 +184,24 @@ std::string windowsEscape(const std::string & str, bool cmd) // Both of these cases break the escaping if not handled. Otherwise backslashes are fine as-is if (iter == str.end()) { // Need to escape each backslash - buffer.append(backslashes * 2, '\\'); + buffer.append(backslashes * 2, L'\\'); // Exit since we've reached the end of the string break; - } else if (*iter == '"') { + } else if (*iter == L'"') { // Need to escape each backslash and the intermediate quote character - buffer.append(backslashes * 2, '\\'); - buffer += "\\\""; + buffer.append(backslashes * 2, L'\\'); + buffer += L"\\\""; } else { // Don't escape the backslashes since they won't break the delimiter - buffer.append(backslashes, '\\'); + buffer.append(backslashes, L'\\'); buffer += *iter; } } // Add the closing quote - return buffer + '"'; + return buffer + L'"'; } -Pid spawnProcess(const Path & realProgram, const RunOptions & options, Pipe & out, Pipe & in) +Pid spawnProcess(const std::filesystem::path & realProgram, const RunOptions & options, Pipe & out, Pipe & in) { // Setup pipes. if (options.standardOut) { @@ -212,40 +224,43 @@ Pid spawnProcess(const Path & realProgram, const RunOptions & options, Pipe & ou startInfo.hStdOutput = out.writeSide.get(); startInfo.hStdError = out.writeSide.get(); - std::string envline; - // Retain the current processes' environment variables. - for (const auto & envVar : getEnv()) { - envline += (envVar.first + '=' + envVar.second + '\0'); - } - // Also add new ones specified in options. + auto env = getEnvOs(); + if (options.environment) { for (const auto & envVar : *options.environment) { - envline += (envVar.first + '=' + envVar.second + '\0'); + env[envVar.first] = envVar.second; } } - std::string cmdline = windowsEscape(realProgram, false); + OsString envline; + + for (const auto & envVar : env) { + envline += (envVar.first + L'=' + envVar.second + L'\0'); + } + + OsString cmdline = windowsEscape(realProgram.native(), false); for (const auto & arg : options.args) { // TODO: This isn't the right way to escape windows command // See https://learn.microsoft.com/en-us/windows/win32/api/shellapi/nf-shellapi-commandlinetoargvw - cmdline += ' ' + windowsEscape(arg, false); + cmdline += L' '; + cmdline += windowsEscape(arg, false); } PROCESS_INFORMATION procInfo = {0}; if (CreateProcessW( // EXE path is provided in the cmdline NULL, - string_to_os_string(cmdline).data(), + cmdline.data(), NULL, NULL, TRUE, CREATE_UNICODE_ENVIRONMENT | CREATE_SUSPENDED, - string_to_os_string(envline).data(), - options.chdir.has_value() ? string_to_os_string(*options.chdir).data() : NULL, + envline.data(), + options.chdir.has_value() ? options.chdir->c_str() : NULL, &startInfo, &procInfo) == 0) { - throw WinError("CreateProcessW failed (%1%)", cmdline); + throw WinError("CreateProcessW failed (%1%)", os_string_to_string(cmdline)); } // Convert these to use RAII @@ -313,7 +328,7 @@ void runProgram2(const RunOptions & options) if (source) in.create(); - Path realProgram = options.program; + std::filesystem::path realProgram = options.program; // TODO: Implement shebang / program interpreter lookup on Windows auto interpreter = getProgramInterpreter(realProgram); @@ -366,7 +381,7 @@ void runProgram2(const RunOptions & options) promise.get_future().get(); if (status) - throw ExecError(status, "program '%1%' %2%", options.program, statusToString(status)); + throw ExecError(status, "program %1% %2%", PathFmt(options.program), statusToString(status)); } std::string statusToString(int status) diff --git a/src/libutil/windows/users.cc b/src/libutil/windows/users.cc index eb92e7ab6ae2..caab6745d4f4 100644 --- a/src/libutil/windows/users.cc +++ b/src/libutil/windows/users.cc @@ -2,7 +2,6 @@ #include "nix/util/users.hh" #include "nix/util/environment-variables.hh" #include "nix/util/file-system.hh" -#include "nix/util/windows-error.hh" #ifdef _WIN32 # define WIN32_LEAN_AND_MEAN @@ -38,9 +37,9 @@ std::string getUserName() std::filesystem::path getHome() { static std::filesystem::path homeDir = []() { - std::filesystem::path homeDir = getEnv("USERPROFILE").value_or("C:\\Users\\Default"); + std::filesystem::path homeDir = getEnvOs(L"USERPROFILE").value_or(L"C:\\Users\\Default"); assert(!homeDir.empty()); - return canonPath(homeDir.string()); + return std::filesystem::path{canonPath(homeDir.string())}; }(); return homeDir; } diff --git a/src/libutil/windows/windows-async-pipe.cc b/src/libutil/windows/windows-async-pipe.cc index f6a82a139756..09b72277a918 100644 --- a/src/libutil/windows/windows-async-pipe.cc +++ b/src/libutil/windows/windows-async-pipe.cc @@ -2,7 +2,6 @@ #ifdef _WIN32 # include "nix/util/windows-async-pipe.hh" -# include "nix/util/windows-error.hh" namespace nix::windows { diff --git a/src/libutil/windows/windows-environment.cc b/src/libutil/windows/windows-environment.cc new file mode 100644 index 000000000000..8226fd0e221e --- /dev/null +++ b/src/libutil/windows/windows-environment.cc @@ -0,0 +1,14 @@ +#include "nix/util/windows-environment.hh" + +#define WIN32_LEAN_AND_MEAN +#include + +namespace nix::windows { + +bool isWine() +{ + HMODULE hntdll = GetModuleHandle("ntdll.dll"); + return GetProcAddress(hntdll, "wine_get_version") != nullptr; +} + +} // namespace nix::windows diff --git a/src/libutil/windows/windows-error.cc b/src/libutil/windows/windows-error.cc index dd731dce22d8..f04b70a2161e 100644 --- a/src/libutil/windows/windows-error.cc +++ b/src/libutil/windows/windows-error.cc @@ -1,5 +1,7 @@ #ifdef _WIN32 -# include "nix/util/windows-error.hh" + +# include "nix/util/error.hh" + # include # define WIN32_LEAN_AND_MEAN # include diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index e87f49546074..134d31fac357 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -10,7 +10,7 @@ using namespace nix; struct CmdAddToStore : MixDryRun, StoreCommand { - Path path; + std::filesystem::path path; std::optional namePart; ContentAddressMethod caMethod = ContentAddressMethod::Raw::NixArchive; HashAlgorithm hashAlgo = HashAlgorithm::SHA256; @@ -36,7 +36,7 @@ struct CmdAddToStore : MixDryRun, StoreCommand void run(ref store) override { if (!namePart) - namePart = baseNameOf(path); + namePart = path.filename().string(); auto sourcePath = PosixSourceAccessor::createAtRoot(makeParentCanonical(path)); diff --git a/src/nix/build-remote/build-remote.cc b/src/nix/build-remote/build-remote.cc index f62712d30ea9..d46f3011a4ed 100644 --- a/src/nix/build-remote/build-remote.cc +++ b/src/nix/build-remote/build-remote.cc @@ -1,6 +1,7 @@ #include #include #include +#include #include #include #include @@ -35,11 +36,11 @@ std::string escapeUri(std::string uri) return uri; } -static std::string currentLoad; +static std::filesystem::path currentLoad; static AutoCloseFD openSlotLock(const Machine & m, uint64_t slot) { - return openLockFile(fmt("%s/%s-%d", currentLoad, escapeUri(m.storeUri.render()), slot), true); + return openLockFile(currentLoad / fmt("%s-%d", escapeUri(m.storeUri.render()), slot), true); } static bool allSupportedLocally(Store & store, const StringSet & requiredFeatures) @@ -76,8 +77,8 @@ static int main_build_remote(int argc, char ** argv) settings.set(name, value); } - auto maxBuildJobs = settings.maxBuildJobs; - settings.maxBuildJobs.set("1"); // hack to make tests with local?root= work + auto maxBuildJobs = settings.getWorkerSettings().maxBuildJobs; + settings.getWorkerSettings().maxBuildJobs.set("1"); // hack to make tests with local?root= work initPlugins(); @@ -85,16 +86,15 @@ static int main_build_remote(int argc, char ** argv) /* It would be more appropriate to use $XDG_RUNTIME_DIR, since that gets cleared on reboot, but it wouldn't work on macOS. */ - auto currentLoadName = "/current-load"; if (auto localStore = store.dynamic_pointer_cast()) - currentLoad = std::string{localStore->config.stateDir} + currentLoadName; + currentLoad = localStore->config.stateDir.get() / "current-load"; else - currentLoad = settings.nixStateDir + currentLoadName; + currentLoad = std::filesystem::path{settings.nixStateDir} / "current-load"; std::shared_ptr sshStore; AutoCloseFD bestSlotLock; - auto machines = getMachines(); + auto machines = Machine::parseConfig({settings.thisSystem}, settings.getWorkerSettings().builders); debug("got %d remote builders", machines.size()); if (machines.empty()) { @@ -134,7 +134,7 @@ static int main_build_remote(int argc, char ** argv) while (true) { bestSlotLock = -1; - AutoCloseFD lock = openLockFile(currentLoad + "/main-lock", true); + AutoCloseFD lock = openLockFile(currentLoad / "main-lock", true); lockFile(lock.get(), ltWrite, true); bool rightType = false; @@ -237,7 +237,7 @@ static int main_build_remote(int argc, char ** argv) sshStore = bestMachine->openStore(); sshStore->connect(); } catch (std::exception & e) { - auto msg = chomp(drainFD(5, false)); + auto msg = chomp(drainFD(5, {.block = false})); printError("cannot build on '%s': %s%s", storeUri, e.what(), msg.empty() ? "" : ": " + msg); bestMachine->enabled = false; continue; @@ -254,18 +254,18 @@ static int main_build_remote(int argc, char ** argv) std::cerr << "# accept\n" << storeUri << "\n"; - auto inputs = readStrings(source); + auto inputs = readStrings(source); auto wantedOutputs = readStrings(source); AutoCloseFD uploadLock; { auto setUpdateLock = [&](auto && fileName) { - uploadLock = openLockFile(currentLoad + "/" + escapeUri(fileName) + ".upload-lock", true); + uploadLock = openLockFile(currentLoad / (escapeUri(fileName) + ".upload-lock"), true); }; try { setUpdateLock(storeUri); - } catch (SysError & e) { - if (e.errNo != ENAMETOOLONG) + } catch (SystemError & e) { + if (!e.is(std::errc::filename_too_long)) throw; // Try again hashing the store URL so we have a shorter path auto h = hashString(HashAlgorithm::MD5, storeUri); @@ -284,7 +284,7 @@ static int main_build_remote(int argc, char ** argv) signal(SIGALRM, old); } - auto substitute = settings.buildersUseSubstitutes ? Substitute : NoSubstitute; + auto substitute = settings.getWorkerSettings().buildersUseSubstitutes ? Substitute : NoSubstitute; { Activity act(*logger, lvlTalkative, actUnknown, fmt("copying dependencies to '%s'", storeUri)); @@ -333,7 +333,7 @@ static int main_build_remote(int argc, char ** argv) : ""); } throw Error( - "build of '%s' on '%s' failed: %s", store->printStorePath(*drvPath), storeUri, failureP->errorMsg); + "build of '%s' on '%s' failed: %s", store->printStorePath(*drvPath), storeUri, failureP->message()); } } else { copyClosure(*store, *sshStore, StorePathSet{*drvPath}, NoRepair, NoCheckSigs, substitute); diff --git a/src/nix/build.cc b/src/nix/build.cc index 2d4f426a4954..4d2adfd5a6c9 100644 --- a/src/nix/build.cc +++ b/src/nix/build.cc @@ -182,7 +182,7 @@ struct CmdBuild : InstallablesCommand, MixOutLinkByDefault, MixDryRun, MixJSON, BuiltPaths buildables2; for (auto & b : buildables) buildables2.push_back(b.path); - updateProfile(buildables2); + updateProfile(*store, buildables2); } }; diff --git a/src/nix/bundle.cc b/src/nix/bundle.cc index 20bfd4d6df60..0e5d18035719 100644 --- a/src/nix/bundle.cc +++ b/src/nix/bundle.cc @@ -7,16 +7,12 @@ #include "nix/expr/eval-inline.hh" #include "nix/store/globals.hh" -namespace nix::fs { -using namespace std::filesystem; -} - using namespace nix; struct CmdBundle : InstallableValueCommand { std::string bundler = "github:NixOS/bundlers"; - std::optional outLink; + std::optional outLink; CmdBundle() { @@ -122,7 +118,7 @@ struct CmdBundle : InstallableValueCommand } // TODO: will crash if not a localFSStore? - store.dynamic_pointer_cast()->addPermRoot(outPath, absPath(*outLink)); + store.dynamic_pointer_cast()->addPermRoot(outPath, absPath(*outLink).string()); } }; diff --git a/src/nix/cat.cc b/src/nix/cat.cc index dcf47f1fa2bf..09416b1f44e7 100644 --- a/src/nix/cat.cc +++ b/src/nix/cat.cc @@ -18,7 +18,9 @@ struct MixCat : virtual Args throw Error("path '%1%' is not a regular file", path.abs()); logger->stop(); - writeFull(getStandardOutput(), accessor->readFile(path)); + FdSink output{getStandardOutput()}; + accessor->readFile(path, output); + output.flush(); } }; @@ -46,13 +48,13 @@ struct CmdCatStore : StoreCommand, MixCat void run(ref store) override { auto [storePath, rest] = store->toStorePath(path); - cat(store->requireStoreObjectAccessor(storePath), CanonPath{rest}); + cat(store->requireStoreObjectAccessor(storePath), rest); } }; struct CmdCatNar : StoreCommand, MixCat { - Path narPath; + std::filesystem::path narPath; std::string path; @@ -76,9 +78,9 @@ struct CmdCatNar : StoreCommand, MixCat void run(ref store) override { - AutoCloseFD fd = toDescriptor(open(narPath.c_str(), O_RDONLY)); + auto fd = openFileReadonly(narPath); if (!fd) - throw SysError("opening NAR file '%s'", narPath); + throw NativeSysError("opening NAR file %s", PathFmt(narPath)); auto source = FdSource{fd.get()}; struct CatRegularFileSink : NullFileSystemObjectSink @@ -86,7 +88,7 @@ struct CmdCatNar : StoreCommand, MixCat CanonPath neededPath = CanonPath::root; bool found = false; - void createRegularFile(const CanonPath & path, std::function crf) override + void createRegularFile(const CanonPath & path, fun crf) override { struct : CreateRegularFileSink, FdSink { diff --git a/src/nix/config-check.cc b/src/nix/config-check.cc index e1efb40ebec4..76c2eaa74ec5 100644 --- a/src/nix/config-check.cc +++ b/src/nix/config-check.cc @@ -11,10 +11,6 @@ #include "nix/util/executable-path.hh" #include "nix/store/globals.hh" -namespace nix::fs { -using namespace std::filesystem; -} - using namespace nix; namespace { @@ -22,9 +18,8 @@ namespace { std::string formatProtocol(unsigned int proto) { if (proto) { - auto major = GET_PROTOCOL_MAJOR(proto) >> 8; - auto minor = GET_PROTOCOL_MINOR(proto); - return fmt("%1%.%2%", major, minor); + auto version = WorkerProto::Version::Number::fromWire(proto); + return fmt("%1%.%2%", version.major, version.minor); } return "unknown"; } @@ -95,7 +90,9 @@ struct CmdConfigCheck : StoreCommand dirs.insert(std::filesystem::canonical(candidate).parent_path()); } - if (dirs.size() != 1) { + if (dirs.empty()) { + return checkFail("No nix-env found in PATH."); + } else if (dirs.size() > 1) { std::ostringstream ss; ss << "Multiple versions of nix found in PATH:\n"; for (auto & dir : dirs) @@ -151,9 +148,10 @@ struct CmdConfigCheck : StoreCommand bool checkStoreProtocol(unsigned int storeProto) { - unsigned int clientProto = GET_PROTOCOL_MAJOR(SERVE_PROTOCOL_VERSION) == GET_PROTOCOL_MAJOR(storeProto) - ? SERVE_PROTOCOL_VERSION - : PROTOCOL_VERSION; + auto storeVersion = WorkerProto::Version::Number::fromWire(storeProto); + unsigned int clientProto = (storeVersion.major == ServeProto::latest.major) + ? ServeProto::latest.toWire() + : WorkerProto::latest.number.toWire(); if (clientProto != storeProto) { std::ostringstream ss; diff --git a/src/nix/copy.cc b/src/nix/copy.cc index 706edc6c9c56..86306e7fdb8e 100644 --- a/src/nix/copy.cc +++ b/src/nix/copy.cc @@ -63,7 +63,7 @@ struct CmdCopy : virtual CopyCommand, virtual BuiltPathsCommand, MixProfile, Mix copyPaths(*srcStore, *dstStore, stuffToCopy, NoRepair, checkSigs, substitute); - updateProfile(rootPaths); + updateProfile(*dstStore, rootPaths); if (outLink) { if (auto store2 = dstStore.dynamic_pointer_cast()) diff --git a/src/nix/derivation-add.cc b/src/nix/derivation-add.cc index bbaa87597152..2dbc89d6f8e7 100644 --- a/src/nix/derivation-add.cc +++ b/src/nix/derivation-add.cc @@ -5,6 +5,7 @@ #include "nix/store/store-api.hh" #include "nix/util/archive.hh" #include "nix/store/derivations.hh" +#include "nix/store/globals.hh" #include using namespace nix; @@ -35,9 +36,8 @@ struct CmdAddDerivation : MixDryRun, StoreCommand auto drv = Derivation::parseJsonAndValidate(*store, json); - auto drvPath = writeDerivation(*store, drv, NoRepair, /* read only */ dryRun); - - writeDerivation(*store, drv, NoRepair, dryRun); + auto drvPath = + (dryRun || settings.readOnlyMode) ? computeStorePath(*store, drv) : store->writeDerivation(drv, NoRepair); logger->cout("%s", store->printStorePath(drvPath)); } diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 9536bf83e486..19cb0151594d 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -22,10 +22,6 @@ #include "nix/util/strings.hh" -namespace nix::fs { -using namespace std::filesystem; -} - using namespace nix; struct DevelopSettings : Config @@ -292,7 +288,7 @@ static StorePath getDerivationEnvironment(ref store, ref evalStore } drv.fillInOutputPaths(*evalStore); - auto shellDrvPath = writeDerivation(*evalStore, drv); + auto shellDrvPath = evalStore->writeDerivation(drv); /* Build the derivation. */ store->buildPaths( @@ -336,7 +332,7 @@ struct Common : InstallableCommand, MixProfile "UID", }; - std::vector> redirects; + std::vector> redirects; Common() { @@ -344,7 +340,7 @@ struct Common : InstallableCommand, MixProfile .longName = "redirect", .description = "Redirect a store path to a mutable location.", .labels = {"installable", "outputs-dir"}, - .handler = {[&](std::string installable, std::string outputsDir) { + .handler = {[&](std::string installable, std::filesystem::path outputsDir) { redirects.push_back({installable, outputsDir}); }}, }); @@ -418,8 +414,8 @@ struct Common : InstallableCommand, MixProfile if (script.find(from) == std::string::npos) warn("'%s' (path '%s') is not used by this build environment", installable->what(), from); else { - printInfo("redirecting '%s' to '%s'", from, dir); - rewrites.insert({from, dir}); + printInfo("redirecting '%s' to '%s'", from, PathFmt(dir)); + rewrites.insert({from, dir.string()}); } } } @@ -444,7 +440,7 @@ struct Common : InstallableCommand, MixProfile * that's accessible from the interactive shell session. */ void fixupStructuredAttrs( - PathViewNG::string_view ext, + PathView::string_view ext, const std::string & envVar, const std::string & content, StringMap & rewrites, @@ -490,7 +486,7 @@ struct Common : InstallableCommand, MixProfile { auto shellOutPath = getShellOutPath(store, installable); - updateProfile(shellOutPath); + updateProfile(*store, shellOutPath); debug("reading environment file '%s'", store->printStorePath(shellOutPath)); @@ -589,7 +585,7 @@ struct CmdDevelop : Common, MixEnvironment if (verbosity >= lvlDebug) script += "set -x\n"; - script += fmt("command rm -f '%s'\n", rcFilePath); + script += fmt("command rm -f '%s'\n", rcFilePath.string()); if (phase) { if (!command.empty()) @@ -626,7 +622,7 @@ struct CmdDevelop : Common, MixEnvironment // prevent garbage collection until shell exits setEnv("NIX_GCROOT", store->printStorePath(gcroot).c_str()); - Path shell = "bash"; + std::filesystem::path shell = "bash"; bool foundInteractive = false; try { @@ -669,11 +665,12 @@ struct CmdDevelop : Common, MixEnvironment // Override SHELL with the one chosen for this environment. // This is to make sure the system shell doesn't leak into the build environment. - setEnv("SHELL", shell.c_str()); - // https://github.com/NixOS/nix/issues/5873 - script += fmt("SHELL=\"%s\"\n", shell); + setEnvOs(OS_STR("SHELL"), shell.c_str()); + /* See: https://github.com/NixOS/nix/issues/5873 + Format via .string() and not PathFmt intentionally. */ + script += fmt("SHELL=\"%s\"\n", shell.string()); if (foundInteractive) - script += fmt("PATH=\"%s${PATH:+:$PATH}\"\n", std::filesystem::path(shell).parent_path()); + script += fmt("PATH=\"%s${PATH:+:$PATH}\"\n", std::filesystem::path(shell).parent_path().string()); writeFull(rcFileFd.get(), script); #ifdef _WIN32 // TODO re-enable on Windows @@ -681,8 +678,8 @@ struct CmdDevelop : Common, MixEnvironment #else // If running a phase or single command, don't want an interactive shell running after // Ctrl-C, so don't pass --rcfile - auto args = phase || !command.empty() ? Strings{std::string(baseNameOf(shell)), rcFilePath} - : Strings{std::string(baseNameOf(shell)), "--rcfile", rcFilePath}; + auto args = phase || !command.empty() ? Strings{shell.filename().string(), rcFilePath} + : Strings{shell.filename().string(), "--rcfile", rcFilePath}; // Need to chdir since phases assume in flake directory if (phase) { @@ -692,7 +689,7 @@ struct CmdDevelop : Common, MixEnvironment auto sourcePath = installableFlake->getLockedFlake()->flake.resolvedRef.input.getSourcePath(); if (sourcePath) { if (chdir(sourcePath->c_str()) == -1) { - throw SysError("chdir to %s failed", *sourcePath); + throw SysError("chdir to %s failed", PathFmt(*sourcePath)); } } } diff --git a/src/nix/dump-path.cc b/src/nix/dump-path.cc index 62fd89877612..2a6253b026fb 100644 --- a/src/nix/dump-path.cc +++ b/src/nix/dump-path.cc @@ -39,7 +39,7 @@ static auto rDumpPath = registerCommand2({"store", "dump-path"}); struct CmdDumpPath2 : Command { - Path path; + std::filesystem::path path; CmdDumpPath2() { @@ -61,7 +61,7 @@ struct CmdDumpPath2 : Command void run() override { auto sink = getNarSink(); - dumpPath(path, sink); + dumpPath(path.string(), sink); sink.flush(); } }; diff --git a/src/nix/env.cc b/src/nix/env.cc index a80bcda67076..79d187d85e59 100644 --- a/src/nix/env.cc +++ b/src/nix/env.cc @@ -97,7 +97,7 @@ struct CmdShell : InstallablesCommand, MixEnvironment auto propPath = state->storeFS->resolveSymlinks( CanonPath(store->printStorePath(path)) / "nix-support" / "propagated-user-env-packages"); if (auto st = state->storeFS->maybeLstat(propPath); st && st->type == SourceAccessor::tRegular) { - for (auto & p : tokenizeString(state->storeFS->readFile(propPath))) + for (auto & p : tokenizeString(state->storeFS->readFile(propPath))) todo.push(store->parseStorePath(p)); } } diff --git a/src/nix/eval.cc b/src/nix/eval.cc index 2f1ba63956fd..db23382003ea 100644 --- a/src/nix/eval.cc +++ b/src/nix/eval.cc @@ -10,10 +10,6 @@ using namespace nix; -namespace nix::fs { -using namespace std::filesystem; -} - struct CmdEval : MixJSON, InstallableValueCommand, MixReadOnlyOption { bool raw = false; @@ -89,7 +85,7 @@ struct CmdEval : MixJSON, InstallableValueCommand, MixReadOnlyOption state->forceValue(v, pos); if (v.type() == nString) // FIXME: disallow strings with contexts? - writeFile(path.string(), v.string_view()); + writeFile(path, v.string_view()); else if (v.type() == nAttrs) { [[maybe_unused]] bool directoryCreated = std::filesystem::create_directory(path); // Directory should not already exist diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 65bc28b9040c..f50a2b1bc305 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -1,14 +1,17 @@ +#include "nix/cmd/common-eval-args.hh" #include "nix/main/common-args.hh" #include "nix/main/shared.hh" #include "nix/expr/eval.hh" #include "nix/expr/eval-inline.hh" #include "nix/expr/eval-settings.hh" #include "nix/expr/get-drvs.hh" +#include "nix/util/os-string.hh" #include "nix/util/signals.hh" #include "nix/store/store-open.hh" #include "nix/store/derivations.hh" #include "nix/store/outputs-spec.hh" #include "nix/expr/attr-path.hh" +#include "nix/fetchers/fetch-settings.hh" #include "nix/fetchers/fetchers.hh" #include "nix/fetchers/registry.hh" #include "nix/expr/eval-cache.hh" @@ -31,10 +34,6 @@ // FIXME is this supposed to be private or not? #include "flake-command.hh" -namespace nix::fs { -using namespace std::filesystem; -} - using namespace nix; using namespace nix::flake; using json = nlohmann::json; @@ -94,9 +93,12 @@ struct CmdFlakeUpdate : FlakeCommand .optional = true, .handler = {[&](std::vector inputsToUpdate) { for (const auto & inputToUpdate : inputsToUpdate) { - InputAttrPath inputAttrPath; + std::optional inputAttrPath; try { - inputAttrPath = flake::parseInputAttrPath(inputToUpdate); + inputAttrPath = flake::NonEmptyInputAttrPath::parse(inputToUpdate); + if (!inputAttrPath) + throw UsageError( + "input path to be updated cannot be zero-length; it would refer to the flake itself, not an input"); } catch (Error & e) { warn( "Invalid flake input '%s'. To update a specific flake, use 'nix flake update --flake %s' instead.", @@ -104,11 +106,11 @@ struct CmdFlakeUpdate : FlakeCommand inputToUpdate); throw e; } - if (lockFlags.inputUpdates.contains(inputAttrPath)) + if (lockFlags.inputUpdates.contains(*inputAttrPath)) warn( "Input '%s' was specified multiple times. You may have done this by accident.", - printInputAttrPath(inputAttrPath)); - lockFlags.inputUpdates.insert(inputAttrPath); + printInputAttrPath(*inputAttrPath)); + lockFlags.inputUpdates.insert(*inputAttrPath); } }}, .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) { @@ -130,7 +132,7 @@ struct CmdFlakeUpdate : FlakeCommand void run(nix::ref store) override { - settings.tarballTtl = 0; + fetchSettings.tarballTtl = 0; auto updateAll = lockFlags.inputUpdates.empty(); lockFlags.recreateLockFile = updateAll; @@ -164,7 +166,7 @@ struct CmdFlakeLock : FlakeCommand void run(nix::ref store) override { - settings.tarballTtl = 0; + fetchSettings.tarballTtl = 0; lockFlags.writeLockFile = true; lockFlags.failOnUnlocked = true; @@ -437,7 +439,7 @@ struct CmdFlakeCheck : FlakeCommand, MixFlakeSchemas throw; } catch (Error & e) { printError("❌ " ANSI_RED "%s" ANSI_NORMAL, leaf.node->getAttrPathStr()); - if (settings.keepGoing) { + if (settings.getWorkerSettings().keepGoing) { logEvalError(); hasErrors = true; } else @@ -530,7 +532,7 @@ struct CmdFlakeCheck : FlakeCommand, MixFlakeSchemas else printError("❌ " ANSI_RED "%s" ANSI_NORMAL, attrPath.to_string(*state)); if (failure->status != BuildResult::Failure::Cancelled) - failure->rethrow(); + throw *failure; } catch (Error & e) { logError(e.info()); } @@ -556,7 +558,7 @@ struct CmdFlakeCheck : FlakeCommand, MixFlakeSchemas struct CmdFlakeInitCommon : virtual Args, EvalCommand, MixFlakeSchemas { std::string templateUrl = "https://flakehub.com/f/DeterminateSystems/flake-templates/0.1"; - Path destDir; + std::filesystem::path destDir; const LockFlags lockFlags{.writeLockFile = false}; @@ -616,11 +618,11 @@ struct CmdFlakeInitCommon : virtual Args, EvalCommand, MixFlakeSchemas else if (st.type == SourceAccessor::tRegular) { auto contents = from2.readFile(); if (std::filesystem::exists(to_st)) { - auto contents2 = readFile(to2.string()); + auto contents2 = readFile(to2); if (contents != contents2) { printError( - "refusing to overwrite existing file '%s'\n please merge it manually with '%s'", - to2.string(), + "refusing to overwrite existing file %s\n please merge it manually with '%s'", + PathFmt(to2), from2); conflictedFiles.push_back(to2); } else { @@ -634,8 +636,8 @@ struct CmdFlakeInitCommon : virtual Args, EvalCommand, MixFlakeSchemas if (std::filesystem::exists(to_st)) { if (std::filesystem::read_symlink(to2) != target) { printError( - "refusing to overwrite existing file '%s'\n please merge it manually with '%s'", - to2.string(), + "refusing to overwrite existing file %s\n please merge it manually with '%s'", + PathFmt(to2), from2); conflictedFiles.push_back(to2); } else { @@ -643,21 +645,28 @@ struct CmdFlakeInitCommon : virtual Args, EvalCommand, MixFlakeSchemas } continue; } else - createSymlink(target, os_string_to_string(PathViewNG{to2})); + createSymlink(target, to2); } else throw Error( "path '%s' needs to be a symlink, file, or directory but instead is a %s", from2, st.typeString()); changedFiles.push_back(to2); - notice("wrote: %s", to2); + notice("wrote: %s", PathFmt(to2)); } }(templateDir, flakeDir); if (!changedFiles.empty() && std::filesystem::exists(std::filesystem::path{flakeDir} / ".git")) { - Strings args = {"-C", flakeDir, "add", "--intent-to-add", "--force", "--"}; + OsStrings args = { + OS_STR("-C"), + flakeDir.native(), + OS_STR("add"), + OS_STR("--intent-to-add"), + OS_STR("--force"), + OS_STR("--"), + }; for (auto & s : changedFiles) - args.emplace_back(s.string()); + args.emplace_back(s.native()); runProgram("git", true, args); } @@ -749,7 +758,7 @@ struct CmdFlakeClone : FlakeCommand struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun, MixNoCheckSigs { - std::string dstUri; + std::optional dstUri; SubstituteFlag substitute = NoSubstitute; @@ -759,7 +768,7 @@ struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun, MixNoCheckSigs .longName = "to", .description = "URI of the destination Nix store", .labels = {"store-uri"}, - .handler = {&dstUri}, + .handler = {[this](std::string s) { dstUri = StoreReference::parse(s); }}, }); } @@ -821,8 +830,8 @@ struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun, MixNoCheckSigs traverse(*flake.lockFile.root); } - if (!dryRun && !dstUri.empty()) { - ref dstStore = dstUri.empty() ? openStore() : openStore(dstUri); + if (!dryRun && dstUri) { + ref dstStore = openStore(StoreReference{*dstUri}); copyPaths(*store, *dstStore, sources, NoRepair, checkSigs, substitute); } diff --git a/src/nix/formatter.cc b/src/nix/formatter.cc index dfb77d87b5f4..8921fcfd8503 100644 --- a/src/nix/formatter.cc +++ b/src/nix/formatter.cc @@ -140,7 +140,7 @@ struct CmdFormatterBuild : MixFormatter, MixOutLinkByDefault auto buildables = unresolvedApp.build(evalStore, store); createOutLinksMaybe(buildables, store); - logger->cout("%s", app.program); + logger->cout("%s", app.program.string()); }; }; diff --git a/src/nix/hash.cc b/src/nix/hash.cc index 2945c672c2cb..43ace0d36cb3 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -9,6 +9,7 @@ #include "nix/util/posix-source-accessor.hh" #include "nix/cmd/misc-store-flags.hh" #include "man-pages.hh" +#include "nix/util/fun.hh" using namespace nix; @@ -112,8 +113,7 @@ struct CmdHashBase : Command } case FileIngestionMethod::Git: { auto sourcePath = makeSourcePath(); - std::function hook; - hook = [&](const SourcePath & path) -> git::TreeEntry { + fun hook = [&](const SourcePath & path) -> git::TreeEntry { auto hashSink = makeSink(); auto mode = dump(path, *hashSink, hook); auto hash = hashSink->finish().hash; diff --git a/src/nix/log.cc b/src/nix/log.cc index 150b4b3711a8..8f251ddab14d 100644 --- a/src/nix/log.cc +++ b/src/nix/log.cc @@ -1,9 +1,8 @@ #include "nix/cmd/command.hh" +#include "nix/cmd/get-build-log.hh" #include "nix/main/common-args.hh" #include "nix/main/shared.hh" #include "nix/store/globals.hh" -#include "nix/store/store-open.hh" -#include "nix/store/log-store.hh" using namespace nix; @@ -30,10 +29,6 @@ struct CmdLog : InstallableCommand { settings.readOnlyMode = true; - auto subs = getDefaultSubstituters(); - - subs.push_front(store); - auto b = installable->toDerivedPath(); // For compat with CLI today, TODO revisit @@ -46,25 +41,9 @@ struct CmdLog : InstallableCommand auto path = resolveDerivedPath(*store, *oneUp); RunPager pager; - for (auto & sub : subs) { - auto * logSubP = dynamic_cast(&*sub); - if (!logSubP) { - printInfo( - "Skipped '%s' which does not support retrieving build logs", sub->config.getHumanReadableURI()); - continue; - } - auto & logSub = *logSubP; - - auto log = logSub.getBuildLog(path); - if (!log) - continue; - logger->stop(); - printInfo("got build log for '%s' from '%s'", installable->what(), logSub.config.getHumanReadableURI()); - writeFull(getStandardOutput(), *log); - return; - } - - throw Error("build log of '%s' is not available", installable->what()); + auto log = fetchBuildLog(store, path, installable->what()); + logger->stop(); + writeFull(getStandardOutput(), log); } }; diff --git a/src/nix/ls.cc b/src/nix/ls.cc index 012850cc05db..7c1627446787 100644 --- a/src/nix/ls.cc +++ b/src/nix/ls.cc @@ -114,13 +114,13 @@ struct CmdLsStore : StoreCommand, MixLs void run(ref store) override { auto [storePath, rest] = store->toStorePath(path); - list(store->requireStoreObjectAccessor(storePath), CanonPath{rest}); + list(store->requireStoreObjectAccessor(storePath), rest); } }; struct CmdLsNar : Command, MixLs { - Path narPath; + std::filesystem::path narPath; std::string path; @@ -144,11 +144,11 @@ struct CmdLsNar : Command, MixLs void run() override { - AutoCloseFD fd = toDescriptor(open(narPath.c_str(), O_RDONLY)); + auto fd = openFileReadonly(narPath); if (!fd) - throw SysError("opening NAR file '%s'", narPath); + throw NativeSysError("opening NAR file %s", PathFmt(narPath)); auto source = FdSource{fd.get()}; - list(makeLazyNarAccessor(source, seekableGetNarBytes(fd.get())), CanonPath{path}); + list(makeLazyNarAccessor(parseNarListing(source), seekableGetNarBytes(fd.get())), CanonPath{path}); } }; diff --git a/src/nix/main.cc b/src/nix/main.cc index 0711804a5447..49aa3f7dd8f8 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -1,3 +1,5 @@ +#include "nix/cmd/common-eval-args.hh" +#include "nix/fetchers/fetch-settings.hh" #include "nix/util/args/root.hh" #include "nix/util/current-process.hh" #include "nix/cmd/command.hh" @@ -27,7 +29,6 @@ #include "cli-config-private.hh" #include -#include #include #ifndef _WIN32 @@ -89,13 +90,13 @@ static bool haveInternet() static void disableNet() { // FIXME: should check for command line overrides only. - if (!settings.useSubstitutes.overridden) + if (!settings.getWorkerSettings().useSubstitutes.overridden) // FIXME: should not disable local substituters (like file:///). - settings.useSubstitutes = false; - if (!settings.tarballTtl.overridden) - settings.tarballTtl = std::numeric_limits::max(); - if (!settings.ttlNarInfoCacheMeta.overridden) - settings.ttlNarInfoCacheMeta = std::numeric_limits::max(); + settings.getWorkerSettings().useSubstitutes = false; + if (!fetchSettings.tarballTtl.overridden) + fetchSettings.tarballTtl = std::numeric_limits::max(); + if (!settings.getNarInfoDiskCacheSettings().ttlMeta.overridden) + settings.getNarInfoDiskCacheSettings().ttlMeta = std::numeric_limits::max(); if (!fileTransferSettings.tries.overridden) fileTransferSettings.tries = 0; if (!fileTransferSettings.connectTimeout.overridden) @@ -259,7 +260,12 @@ static void showHelp(std::vector subcommand, NixArgs & toplevel) evalSettings.restrictEval = true; evalSettings.pureEval = true; - EvalState state({}, openStore("dummy://"), fetchSettings, evalSettings); + auto statePtr = std::make_shared( + LookupPath{}, + openStore(StoreReference{.variant = StoreReference::Specified{.scheme = "dummy"}}), + fetchSettings, + evalSettings); + auto & state = *statePtr; auto vGenerateManpage = state.allocValue(); state.eval( @@ -388,22 +394,22 @@ void mainWrapped(int argc, char ** argv) } #endif - initNix(); - initGC(); - flakeSettings.configureEvalSettings(evalSettings); - /* Set the build hook location For builds we perform a self-invocation, so Nix has to be self-aware. That is, it has to know where it is installed. We don't think it's sentient. */ - settings.buildHook.setDefault( + settings.getWorkerSettings().buildHook.setDefault( Strings{ getNixBin({}).string(), "__build-remote", }); + initNix(); + initGC(); + flakeSettings.configureEvalSettings(evalSettings); + #ifdef __linux__ if (isRootUser()) { try { @@ -428,14 +434,15 @@ void mainWrapped(int argc, char ** argv) } { - auto legacy = RegisterLegacyCommand::commands()[programName]; - if (legacy) - return legacy(argc, argv); + if (auto legacy = get(RegisterLegacyCommand::commands(), programName)) + return (*legacy)(argc, argv); } evalSettings.pureEval = true; +#ifndef _WIN32 setLogFormat("bar"); +#endif settings.verboseBuild = false; // If on a terminal, progress will be displayed via progress bars etc. (thus verbosity=notice) @@ -459,7 +466,12 @@ void mainWrapped(int argc, char ** argv) Xp::FetchTree, }; evalSettings.pureEval = false; - EvalState state({}, openStore("dummy://"), fetchSettings, evalSettings); + auto statePtr = std::make_shared( + LookupPath{}, + openStore(StoreReference{.variant = StoreReference::Specified{.scheme = "dummy"}}), + fetchSettings, + evalSettings); + auto & state = *statePtr; auto builtinsJson = nlohmann::json::object(); for (auto & builtinPtr : state.getBuiltins().attrs()->lexicographicOrder(state.symbols)) { auto & builtin = *builtinPtr; @@ -519,7 +531,7 @@ void mainWrapped(int argc, char ** argv) disableNet(); try { - auto isNixCommand = std::regex_search(programName, std::regex("nix$")); + auto isNixCommand = programName.ends_with("nix"); auto allowShebang = isNixCommand && argc > 1; args.parseCmdline(argvToStrings(argc, argv), allowShebang); } catch (UsageError &) { @@ -567,10 +579,10 @@ void mainWrapped(int argc, char ** argv) disableNet(); if (args.refresh) { - settings.tarballTtl = 0; - settings.ttlNegativeNarInfoCache = 0; - settings.ttlPositiveNarInfoCache = 0; - settings.ttlNarInfoCacheMeta = 0; + fetchSettings.tarballTtl = 0; + settings.getNarInfoDiskCacheSettings().ttlNegative = 0; + settings.getNarInfoDiskCacheSettings().ttlPositive = 0; + settings.getNarInfoDiskCacheSettings().ttlMeta = 0; } if (args.command->second->forceImpureByDefault() && !evalSettings.pureEval.overridden) { diff --git a/src/nix/make-content-addressed.cc b/src/nix/make-content-addressed.cc index a54729c45420..cd0f3f59f610 100644 --- a/src/nix/make-content-addressed.cc +++ b/src/nix/make-content-addressed.cc @@ -30,7 +30,7 @@ struct CmdMakeContentAddressed : virtual CopyCommand, virtual StorePathsCommand, void run(ref srcStore, StorePaths && storePaths) override { - auto dstStore = dstUri.empty() ? openStore() : openStore(dstUri); + auto dstStore = !dstUri ? openStore() : openStore(StoreReference{*dstUri}); auto remappings = makeContentAddressed(*srcStore, *dstStore, StorePathSet(storePaths.begin(), storePaths.end())); diff --git a/src/nix/man-pages.cc b/src/nix/man-pages.cc index 7ab8a0eeb5bd..9b01307a13c9 100644 --- a/src/nix/man-pages.cc +++ b/src/nix/man-pages.cc @@ -8,7 +8,7 @@ namespace nix { std::filesystem::path getNixManDir() { - return canonPath(NIX_MAN_DIR); + return canonPath(std::filesystem::path{NIX_MAN_DIR}); } void showManPage(const std::string & name) diff --git a/src/nix/meson.build b/src/nix/meson.build index 3b343614e421..7252687670c9 100644 --- a/src/nix/meson.build +++ b/src/nix/meson.build @@ -117,6 +117,7 @@ nix_sources = [ config_priv_h ] + files( if host_machine.system() != 'windows' nix_sources += files( 'unix/daemon.cc', + 'unix/store-roots-daemon.cc', ) endif diff --git a/src/nix/nario.cc b/src/nix/nario.cc index 452c8c9ffaa8..9e83e6e0709f 100644 --- a/src/nix/nario.cc +++ b/src/nix/nario.cc @@ -127,7 +127,7 @@ nlohmann::json listNar(Source & source) j["entries"] = nlohmann::json::object(); } - void createRegularFile(const CanonPath & path, std::function func) override + void createRegularFile(const CanonPath & path, fun func) override { struct : CreateRegularFileSink { diff --git a/src/nix/nix-build/nix-build.cc b/src/nix/nix-build/nix-build.cc index 217382ef8ee2..27ac286bd968 100644 --- a/src/nix/nix-build/nix-build.cc +++ b/src/nix/nix-build/nix-build.cc @@ -28,6 +28,7 @@ #include "nix/util/users.hh" #include "nix/cmd/network-proxy.hh" #include "nix/cmd/compatibility-settings.hh" +#include "nix/util/fun.hh" #include "man-pages.hh" using namespace nix; @@ -140,7 +141,7 @@ static void main_nix_build(int argc, char ** argv) auto myName = isNixShell ? "nix-shell" : "nix-build"; auto inShebang = false; - std::string script; + std::filesystem::path script; std::vector savedArgs; AutoDelete tmpDir(createTempDir("", myName)); @@ -200,9 +201,9 @@ static void main_nix_build(int argc, char ** argv) { using LegacyArgs::LegacyArgs; - void setBaseDir(Path baseDir) + void setBaseDir(std::filesystem::path baseDir) { - commandBaseDir = baseDir; + commandBaseDir = baseDir.string(); } }; @@ -284,11 +285,15 @@ static void main_nix_build(int argc, char ** argv) fmt("exec %1% %2% -e 'load(ARGV.shift)' -- %3% %4%", execArgs, interpreter, - escapeShellArgAlways(script), + escapeShellArgAlways(script.string()), joined.view()); } else { envCommand = - fmt("exec %1% %2% %3% %4%", execArgs, interpreter, escapeShellArgAlways(script), joined.view()); + fmt("exec %1% %2% %3% %4%", + execArgs, + interpreter, + escapeShellArgAlways(script.string()), + joined.view()); } } @@ -313,15 +318,15 @@ static void main_nix_build(int argc, char ** argv) throw UsageError("'-p' and '-E' are mutually exclusive"); auto store = openStore(); - auto evalStore = myArgs.evalStoreUrl ? openStore(*myArgs.evalStoreUrl) : store; + auto evalStore = myArgs.evalStoreUrl ? openStore(StoreReference{*myArgs.evalStoreUrl}) : store; - auto state = std::make_unique(myArgs.lookupPath, evalStore, fetchSettings, evalSettings, store); + auto state = std::make_shared(myArgs.lookupPath, evalStore, fetchSettings, evalSettings, store); state->repair = myArgs.repair; if (myArgs.repair) buildMode = bmRepair; if (inShebang && compatibilitySettings.nixShellShebangArgumentsRelativeToScript) { - myArgs.setBaseDir(absPath(dirOf(script))); + myArgs.setBaseDir(absPath(script.parent_path())); } auto autoArgs = myArgs.getAutoArgs(*state); @@ -373,17 +378,17 @@ static void main_nix_build(int argc, char ** argv) exprs = {state->parseStdin()}; else for (auto i : remainingArgs) { + auto shebangBaseDir = absPath(script.parent_path()); if (fromArgs) { - auto shebangBaseDir = absPath(dirOf(script)); exprs.push_back(state->parseExprFromString( std::move(i), (inShebang && compatibilitySettings.nixShellShebangArgumentsRelativeToScript) - ? lookupFileArg(*state, shebangBaseDir) + ? lookupFileArg(*state, shebangBaseDir.string()) : state->rootPath("."))); } else { auto absolute = i; try { - absolute = canonPath(absPath(i), true); + absolute = canonPath(absPath(std::filesystem::path{i}), true).string(); } catch (Error & e) { }; auto [path, outputNames] = parsePathWithOutputs(absolute); @@ -392,9 +397,10 @@ static void main_nix_build(int argc, char ** argv) else { /* If we're in a #! script, interpret filenames relative to the script. */ - auto baseDir = inShebang && !packages ? absPath(i, absPath(dirOf(script))) : i; + std::filesystem::path iPath{i}; + auto baseDir = inShebang && !packages ? absPath(iPath, &shebangBaseDir) : iPath; - auto sourcePath = lookupFileArg(*state, baseDir); + auto sourcePath = lookupFileArg(*state, baseDir.string()); auto resolvedPath = isNixShell ? resolveShellExprPath(sourcePath) : resolveExprPath(sourcePath); exprs.push_back(state->parseExprFromFile(resolvedPath)); @@ -534,7 +540,7 @@ static void main_nix_build(int argc, char ** argv) shell = store->printStorePath(shellDrvOutputs.at("out").value()) + "/bin/bash"; } - if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { + if (drv.shouldResolve()) { auto resolvedDrv = drv.tryResolve(*store); assert(resolvedDrv && "Successfully resolved the derivation"); drv = *resolvedDrv; @@ -555,7 +561,10 @@ static void main_nix_build(int argc, char ** argv) env["NIX_BUILD_TOP"] = env["TMPDIR"] = env["TEMPDIR"] = env["TMP"] = env["TEMP"] = tmpDir.path().string(); env["NIX_STORE"] = store->storeDir; - env["NIX_BUILD_CORES"] = fmt("%d", settings.buildCores ? settings.buildCores : settings.getDefaultCores()); + env["NIX_BUILD_CORES"] = + fmt("%d", + settings.getLocalSettings().buildCores ? settings.getLocalSettings().buildCores + : settings.getDefaultCores()); DerivationOptions drvOptions; try { @@ -570,7 +579,7 @@ static void main_nix_build(int argc, char ** argv) for (auto & var : drv.env) if (drvOptions.passAsFile.count(var.first)) { auto fn = ".attr-" + std::to_string(fileNr++); - Path p = (tmpDir.path() / fn).string(); + auto p = (tmpDir.path() / fn).string(); writeFile(p, var.second); env[var.first + "Path"] = p; } else @@ -581,18 +590,16 @@ static void main_nix_build(int argc, char ** argv) if (drv.structuredAttrs) { StorePathSet inputs; - std::function::ChildNode &)> accumInputClosure; - - accumInputClosure = [&](const StorePath & inputDrv, - const DerivedPathMap::ChildNode & inputNode) { - auto outputs = store->queryPartialDerivationOutputMap(inputDrv, &*evalStore); - for (auto & i : inputNode.value) { - auto o = outputs.at(i); - store->computeFSClosure(*o, inputs); - } - for (const auto & [outputName, childNode] : inputNode.childMap) - accumInputClosure(*outputs.at(outputName), childNode); - }; + fun::ChildNode &)> accumInputClosure = + [&](const StorePath & inputDrv, const DerivedPathMap::ChildNode & inputNode) { + auto outputs = store->queryPartialDerivationOutputMap(inputDrv, &*evalStore); + for (auto & i : inputNode.value) { + auto o = outputs.at(i); + store->computeFSClosure(*o, inputs); + } + for (const auto & [outputName, childNode] : inputNode.childMap) + accumInputClosure(*outputs.at(outputName), childNode); + }; for (const auto & [inputDrv, inputNode] : drv.inputDrvs.map) accumInputClosure(inputDrv, inputNode); @@ -616,6 +623,8 @@ static void main_nix_build(int argc, char ** argv) environment variables and shell functions. Also don't lose the current $PATH directories. */ auto rcfile = (tmpDir.path() / "rc").string(); + auto tz = getEnv("TZ"); + auto tzExport = tz ? "export TZ=" + escapeShellArgAlways(*tz) + "; " : ""; std::string rc = fmt( (R"(_nix_shell_clean_tmpdir() { command rm -rf %1%; };)"s "trap _nix_shell_clean_tmpdir EXIT; " @@ -647,9 +656,9 @@ static void main_nix_build(int argc, char ** argv) escapeShellArgAlways(tmpDir.path().string()), (pure ? "" : "p=$PATH; "), (pure ? "" : "PATH=$PATH:$p; unset p; "), - escapeShellArgAlways(dirOf(*shell)), + escapeShellArgAlways(std::filesystem::path(*shell).parent_path().string()), escapeShellArgAlways(*shell), - (getenv("TZ") ? (std::string("export TZ=") + escapeShellArgAlways(getenv("TZ")) + "; ") : ""), + tzExport, envCommand); vomit("Sourcing nix-shell with file %s and contents:\n%s", rcfile, rc); writeFile(rcfile, rc); diff --git a/src/nix/nix-channel/nix-channel.cc b/src/nix/nix-channel/nix-channel.cc index 00723ba2b09b..e02426f13188 100644 --- a/src/nix/nix-channel/nix-channel.cc +++ b/src/nix/nix-channel/nix-channel.cc @@ -6,6 +6,7 @@ #include "nix/cmd/legacy.hh" #include "nix/cmd/common-eval-args.hh" #include "nix/expr/eval-settings.hh" // for defexpr +#include "nix/util/os-string.hh" #include "nix/util/users.hh" #include "nix/fetchers/tarball.hh" #include "nix/fetchers/fetch-settings.hh" @@ -44,7 +45,13 @@ static void readChannels() // Writes the list of channels. static void writeChannels() { - auto channelsFD = AutoCloseFD{open(channelsList.c_str(), O_WRONLY | O_CLOEXEC | O_CREAT | O_TRUNC, 0644)}; + auto channelsFD = openNewFileForWrite( + channelsList, + 0644, + { + .truncateExisting = true, + .followSymlinksOnTruncate = true, /* Back-compat. */ + }); if (!channelsFD) throw SysError("opening '%1%' for writing", channelsList.string()); for (const auto & channel : channels) @@ -63,7 +70,7 @@ static void addChannel(const std::string & url, const std::string & name) writeChannels(); } -static Path profile; +static std::filesystem::path profile; // Remove a channel. static void removeChannel(const std::string & name) @@ -72,10 +79,18 @@ static void removeChannel(const std::string & name) channels.erase(name); writeChannels(); - runProgram(getNixBin("nix-env").string(), true, {"--profile", profile, "--uninstall", name}); + runProgram( + getNixBin("nix-env"), + true, + { + OS_STR("--profile"), + profile.native(), + OS_STR("--uninstall"), + string_to_os_string(name), + }); } -static Path nixDefExpr; +static std::filesystem::path nixDefExpr; // Fetch Nix expressions and binary cache URLs from the subscribed channels. static void update(const StringSet & channelNames) @@ -111,12 +126,12 @@ static void update(const StringSet & channelNames) if (!(channelNames.empty() || channelNames.count(name))) { // no need to update this channel, reuse the existing store path - Path symlink = profile + "/" + name; - Path storepath = dirOf(readLink(symlink)); + std::filesystem::path symlink = profile / name; + std::filesystem::path storepath = readLink(symlink).parent_path(); exprs.push_back( "f: rec { name = \"" + cname + "\"; type = \"derivation\"; outputs = [\"out\"]; system = \"builtin\"; outPath = builtins.storePath \"" - + storepath + "\"; out = { inherit outPath; };}"); + + storepath.string() + "\"; out = { inherit outPath; };}"); } else { // We want to download the url to a file to see if it's a tarball while also checking if we // got redirected in the process, so that we can grab the various parts of a nix channel @@ -127,12 +142,16 @@ static void update(const StringSet & channelNames) bool unpacked = false; if (std::regex_search(std::string{result.storePath.to_string()}, std::regex("\\.tar\\.(gz|bz2|xz)$"))) { runProgram( - getNixBin("nix-build").string(), + getNixBin("nix-build"), false, - {"--no-out-link", - "--expr", - "import " + unpackChannelPath + "{ name = \"" + cname + "\"; channelName = \"" + name - + "\"; src = builtins.storePath \"" + store->printStorePath(result.storePath) + "\"; }"}); + { + OS_STR("--no-out-link"), + OS_STR("--expr"), + string_to_os_string( + "import " + unpackChannelPath.string() + "{ name = \"" + cname + "\"; channelName = \"" + + name + "\"; src = builtins.storePath \"" + store->printStorePath(result.storePath) + + "\"; }"), + }); unpacked = true; } @@ -155,25 +174,30 @@ static void update(const StringSet & channelNames) // Unpack the channel tarballs into the Nix store and install them // into the channels profile. std::cerr << "unpacking " << exprs.size() << " channels...\n"; - Strings envArgs{ - "--profile", profile, "--file", unpackChannelPath, "--install", "--remove-all", "--from-expression"}; + OsStrings envArgs{ + OS_STR("--profile"), + profile.native(), + OS_STR("--file"), + string_to_os_string(unpackChannelPath), + OS_STR("--install"), + OS_STR("--remove-all"), + OS_STR("--from-expression")}; for (auto & expr : exprs) - envArgs.push_back(std::move(expr)); - envArgs.push_back("--quiet"); - runProgram(getNixBin("nix-env").string(), false, envArgs); + envArgs.push_back(string_to_os_string(std::move(expr))); + envArgs.push_back(OS_STR("--quiet")); + runProgram(getNixBin("nix-env"), false, envArgs); // Make the channels appear in nix-env. - struct stat st; + PosixStat st; if (lstat(nixDefExpr.c_str(), &st) == 0) { if (S_ISLNK(st.st_mode)) // old-skool ~/.nix-defexpr - if (unlink(nixDefExpr.c_str()) == -1) - throw SysError("unlinking %1%", nixDefExpr); + unlink(nixDefExpr); } else if (errno != ENOENT) { - throw SysError("getting status of %1%", nixDefExpr); + throw SysError("getting status of %1%", PathFmt(nixDefExpr)); } createDirs(nixDefExpr); - auto channelLink = nixDefExpr + "/channels"; + auto channelLink = nixDefExpr / "channels"; replaceSymlink(profile, channelLink); } @@ -187,12 +211,13 @@ For details and to offer feedback on the deprecation process, see: https://githu { // Figure out the name of the `.nix-channels' file to use auto home = getHome(); - channelsList = settings.useXDGBaseDirectories ? createNixStateDir() + "/channels" : home + "/.nix-channels"; + channelsList = + settings.useXDGBaseDirectories ? (createNixStateDir() / "channels").string() : home + "/.nix-channels"; nixDefExpr = getNixDefExpr(); // Figure out the name of the channels profile. - profile = profilesDir() + "/channels"; - createDirs(dirOf(profile)); + profile = profilesDir(settings.getProfileDirsOptions()) / "channels"; + createDirs(profile.parent_path()); enum { cNone, cAdd, cRemove, cList, cUpdate, cListGenerations, cRollback } cmd = cNone; @@ -259,20 +284,26 @@ For details and to offer feedback on the deprecation process, see: https://githu case cListGenerations: if (!args.empty()) throw UsageError("'--list-generations' expects no arguments"); - std::cout << runProgram(getNixBin("nix-env").string(), false, {"--profile", profile, "--list-generations"}) - << std::flush; + std::cout << runProgram( + getNixBin("nix-env"), + false, + { + OS_STR("--profile"), + profile.native(), + OS_STR("--list-generations"), + }) << std::flush; break; case cRollback: if (args.size() > 1) throw UsageError("'--rollback' has at most one argument"); - Strings envArgs{"--profile", profile}; + OsStrings envArgs{OS_STR("--profile"), profile.native()}; if (args.size() == 1) { - envArgs.push_back("--switch-generation"); - envArgs.push_back(args[0]); + envArgs.push_back(OS_STR("--switch-generation")); + envArgs.push_back(string_to_os_string(args[0])); } else { - envArgs.push_back("--rollback"); + envArgs.push_back(OS_STR("--rollback")); } - runProgram(getNixBin("nix-env").string(), false, envArgs); + runProgram(getNixBin("nix-env"), false, envArgs); break; } diff --git a/src/nix/nix-collect-garbage/nix-collect-garbage.cc b/src/nix/nix-collect-garbage/nix-collect-garbage.cc index 29ca17a5de25..3d84ca13cd99 100644 --- a/src/nix/nix-collect-garbage/nix-collect-garbage.cc +++ b/src/nix/nix-collect-garbage/nix-collect-garbage.cc @@ -1,5 +1,6 @@ #include "nix/util/file-system.hh" #include "nix/util/signals.hh" +#include "nix/util/error.hh" #include "nix/store/store-open.hh" #include "nix/store/store-cast.hh" #include "nix/store/gc-store.hh" @@ -12,10 +13,6 @@ #include #include -namespace nix::fs { -using namespace std::filesystem; -} - using namespace nix; std::string deleteOlderThan; @@ -41,9 +38,9 @@ void removeOldGenerations(std::filesystem::path dir) if (type == std::filesystem::file_type::symlink && canWrite) { std::string link; try { - link = readLink(path); - } catch (std::filesystem::filesystem_error & e) { - if (e.code() == std::errc::no_such_file_or_directory) + link = readLink(path).string(); + } catch (SystemError & e) { + if (e.is(std::errc::no_such_file_or_directory)) continue; throw; } @@ -87,25 +84,26 @@ static int main_nix_collect_garbage(int argc, char ** argv) return true; }); + if (options.maxFreed != std::numeric_limits::max() && dryRun) + throw UsageError("options --max-freed and --dry-run cannot be combined"); + if (removeOld) { + auto profilesDirOpts = settings.getProfileDirsOptions(); std::set dirsToClean = { - profilesDir(), + profilesDir(profilesDirOpts), std::filesystem::path{settings.nixStateDir} / "profiles", - getDefaultProfile().parent_path(), + getDefaultProfile(profilesDirOpts).parent_path(), }; for (auto & dir : dirsToClean) removeOldGenerations(dir); } - // Run the actual garbage collector. - if (!dryRun) { - auto store = openStore(); - auto & gcStore = require(*store); - options.action = GCOptions::gcDeleteDead; - GCResults results; - PrintFreed freed(true, results); - gcStore.collectGarbage(options, results); - } + auto store = openStore(); + auto & gcStore = require(*store); + options.action = dryRun ? GCOptions::gcReturnDead : GCOptions::gcDeleteDead; + GCResults results; + Finally printer([&] { printFreed(dryRun, results); }); + gcStore.collectGarbage(options, results); return 0; } diff --git a/src/nix/nix-copy-closure/nix-copy-closure.cc b/src/nix/nix-copy-closure/nix-copy-closure.cc index 87d0f65905bf..33aec6c3e659 100644 --- a/src/nix/nix-copy-closure/nix-copy-closure.cc +++ b/src/nix/nix-copy-closure/nix-copy-closure.cc @@ -1,5 +1,6 @@ #include "nix/main/shared.hh" #include "nix/store/realisation.hh" +#include "nix/store/legacy-ssh-store.hh" #include "nix/store/store-open.hh" #include "nix/cmd/legacy.hh" #include "man-pages.hh" @@ -15,7 +16,7 @@ static int main_nix_copy_closure(int argc, char ** argv) auto dryRun = false; auto useSubstitutes = NoSubstitute; std::string sshHost; - PathSet storePaths; + StringSet storePaths; parseCmdLine(argc, argv, [&](Strings::iterator & arg, const Strings::iterator & end) { if (*arg == "--help") @@ -48,9 +49,14 @@ static int main_nix_copy_closure(int argc, char ** argv) if (sshHost.empty()) throw UsageError("no host name specified"); - auto remoteUri = "ssh://" + sshHost + (gzip ? "?compress=true" : ""); - auto to = toMode ? openStore(remoteUri) : openStore(); - auto from = toMode ? openStore() : openStore(remoteUri); + auto remoteConfig = + /* FIXME: This doesn't go through the back-compat machinery for IPv6 unbracketed URLs that + is in StoreReference::parse. TODO: Maybe add a authority parsing function specifically + for SSH reference parsing? */ + make_ref(ParsedURL::Authority::parse(sshHost), LegacySSHStoreConfig::Params{}); + remoteConfig->compress |= gzip; + auto to = toMode ? remoteConfig->openStore() : openStore(); + auto from = toMode ? openStore() : remoteConfig->openStore(); RealisedPath::Set storePaths2; for (auto & path : storePaths) diff --git a/src/nix/nix-env/nix-env.cc b/src/nix/nix-env/nix-env.cc index c006de7f8003..d890a74837a9 100644 --- a/src/nix/nix-env/nix-env.cc +++ b/src/nix/nix-env/nix-env.cc @@ -5,6 +5,7 @@ #include "nix/expr/eval.hh" #include "nix/expr/get-drvs.hh" #include "nix/store/globals.hh" +#include "nix/util/config-global.hh" #include "nix/store/names.hh" #include "nix/store/profiles.hh" #include "nix/store/path-with-outputs.hh" @@ -34,13 +35,46 @@ using namespace nix; using std::cout; +/** + * Settings related to Nix user environments. + */ +struct EnvSettings : Config +{ + Setting keepDerivations{ + this, + false, + "keep-env-derivations", + R"( + If `false` (default), derivations are not stored in Nix user + environments. That is, the derivations of any build-time-only + dependencies may be garbage-collected. + + If `true`, when you add a Nix derivation to a user environment, the + path of the derivation is stored in the user environment. Thus, the + derivation isn't garbage-collected until the user environment + generation is deleted (`nix-env --delete-generations`). To prevent + build-time-only dependencies from being collected, you should also + turn on `keep-outputs`. + + The difference between this option and `keep-derivations` is that + this one is "sticky": it applies to any user environment created + while this option was enabled, while `keep-derivations` only applies + at the moment the garbage collector is run. + )", + {"env-keep-derivations"}}; +}; + +EnvSettings envSettings; + +static GlobalConfig::Register rSettings(&envSettings); + typedef enum { srcNixExprDrvs, srcNixExprs, srcStorePaths, srcProfile, srcAttrPath, srcUnknown } InstallSourceType; struct InstallSourceInfo { InstallSourceType type; std::shared_ptr nixExprPath; /* for srcNixExprDrvs, srcNixExprs */ - Path profile; /* for srcProfile */ + std::filesystem::path profile; /* for srcProfile */ std::string systemFilter; /* for srcNixExprDrvs */ Bindings * autoArgs; }; @@ -48,7 +82,7 @@ struct InstallSourceInfo struct Globals { InstallSourceInfo instSource; - Path profile; + std::filesystem::path profile; std::shared_ptr state; bool dryRun; bool preserveInstalled; @@ -489,8 +523,8 @@ static void setMetaFlag(EvalState & state, PackageInfo & drv, const std::string drv.setMeta(name, v); } -static void -installDerivations(Globals & globals, const Strings & args, const Path & profile, std::optional priority) +static void installDerivations( + Globals & globals, const Strings & args, const std::filesystem::path & profile, std::optional priority) { debug("installing derivations"); @@ -547,7 +581,7 @@ installDerivations(Globals & globals, const Strings & args, const Path & profile if (globals.dryRun) return; - if (createUserEnv(*globals.state, allElems, profile, settings.envKeepDerivations, lockToken)) + if (createUserEnv(*globals.state, allElems, profile, envSettings.keepDerivations, lockToken)) break; } } @@ -658,7 +692,7 @@ static void upgradeDerivations(Globals & globals, const Strings & args, UpgradeT if (globals.dryRun) return; - if (createUserEnv(*globals.state, newElems, globals.profile, settings.envKeepDerivations, lockToken)) + if (createUserEnv(*globals.state, newElems, globals.profile, envSettings.keepDerivations, lockToken)) break; } } @@ -717,7 +751,7 @@ static void opSetFlag(Globals & globals, Strings opFlags, Strings opArgs) checkSelectorUse(selectors); /* Write the new user environment. */ - if (createUserEnv(*globals.state, installedElems, globals.profile, settings.envKeepDerivations, lockToken)) + if (createUserEnv(*globals.state, installedElems, globals.profile, envSettings.keepDerivations, lockToken)) break; } } @@ -767,7 +801,7 @@ static void opSet(Globals & globals, Strings opFlags, Strings opArgs) switchLink(globals.profile, generation); } -static void uninstallDerivations(Globals & globals, Strings & selectors, Path & profile) +static void uninstallDerivations(Globals & globals, Strings & selectors, const std::filesystem::path & profile) { while (true) { auto lockToken = optimisticLockProfile(profile); @@ -800,7 +834,7 @@ static void uninstallDerivations(Globals & globals, Strings & selectors, Path & if (globals.dryRun) return; - if (createUserEnv(*globals.state, workingElems, profile, settings.envKeepDerivations, lockToken)) + if (createUserEnv(*globals.state, workingElems, profile, envSettings.keepDerivations, lockToken)) break; } } @@ -1261,7 +1295,7 @@ static void opSwitchProfile(Globals & globals, Strings opFlags, Strings opArgs) if (opArgs.size() != 1) throw UsageError("exactly one argument expected"); - Path profile = absPath(opArgs.front()); + auto profile = absPath(std::filesystem::path{opArgs.front()}); auto profileLink = settings.useXDGBaseDirectories ? createNixStateDir() / "profile" : getHome() / ".nix-profile"; switchLink(profileLink, profile); @@ -1381,11 +1415,11 @@ static int main_nix_env(int argc, char ** argv) if (!pathExists(nixExprPath)) { try { + auto profilesDirOpts = settings.getProfileDirsOptions(); createDirs(nixExprPath); - replaceSymlink(defaultChannelsDir(), nixExprPath / "channels"); + replaceSymlink(defaultChannelsDir(profilesDirOpts), nixExprPath / "channels"); if (!isRootUser()) - replaceSymlink(rootChannelsDir(), nixExprPath / "channels_root"); - } catch (std::filesystem::filesystem_error &) { + replaceSymlink(rootChannelsDir(profilesDirOpts), nixExprPath / "channels_root"); } catch (Error &) { } } @@ -1491,7 +1525,7 @@ static int main_nix_env(int argc, char ** argv) globals.profile = getEnv("NIX_PROFILE").value_or(""); if (globals.profile == "") - globals.profile = getDefaultProfile().string(); + globals.profile = getDefaultProfile(settings.getProfileDirsOptions()).string(); op(globals, std::move(opFlags), std::move(opArgs)); diff --git a/src/nix/nix-env/user-env.cc b/src/nix/nix-env/user-env.cc index ac36bf97011d..eab668a9206a 100644 --- a/src/nix/nix-env/user-env.cc +++ b/src/nix/nix-env/user-env.cc @@ -16,15 +16,15 @@ namespace nix { -PackageInfos queryInstalled(EvalState & state, const Path & userEnv) +PackageInfos queryInstalled(EvalState & state, const std::filesystem::path & userEnv) { PackageInfos elems; - if (pathExists(userEnv + "/manifest.json")) - throw Error("profile '%s' is incompatible with 'nix-env'; please use 'nix profile' instead", userEnv); - auto manifestFile = userEnv + "/manifest.nix"; + if (pathExists(userEnv / "manifest.json")) + throw Error("profile %s is incompatible with 'nix-env'; please use 'nix profile' instead", PathFmt(userEnv)); + auto manifestFile = userEnv / "manifest.nix"; if (pathExists(manifestFile)) { Value v; - state.evalFile(state.rootPath(CanonPath(manifestFile)).resolveSymlinks(), v); + state.evalFile(state.rootPath(CanonPath(manifestFile.string())).resolveSymlinks(), v); Bindings & bindings = Bindings::emptyBindings; getDerivations(state, v, "", bindings, elems, false); } @@ -32,7 +32,11 @@ PackageInfos queryInstalled(EvalState & state, const Path & userEnv) } bool createUserEnv( - EvalState & state, PackageInfos & elems, const Path & profile, bool keepDerivations, const std::string & lockToken) + EvalState & state, + PackageInfos & elems, + const std::filesystem::path & profile, + bool keepDerivations, + const std::string & lockToken) { /* Build the components in the user environment, if they don't exist already. */ @@ -110,7 +114,7 @@ bool createUserEnv( environment. */ auto manifestFile = ({ std::ostringstream str; - printAmbiguous(state, manifest, str, nullptr, std::numeric_limits::max()); + printAmbiguous(state, manifest, str, nullptr); StringSource source{str.view()}; state.store->addToStoreFromDump( source, @@ -166,7 +170,7 @@ bool createUserEnv( std::filesystem::path lockTokenCur = optimisticLockProfile(profile); if (lockToken != lockTokenCur) { - printInfo("profile '%1%' changed while we were busy; restarting", profile); + printInfo("profile %s changed while we were busy; restarting", PathFmt(profile)); return false; } diff --git a/src/nix/nix-env/user-env.hh b/src/nix/nix-env/user-env.hh index abe25af65fe3..2c0ce9da0dab 100644 --- a/src/nix/nix-env/user-env.hh +++ b/src/nix/nix-env/user-env.hh @@ -5,9 +5,13 @@ namespace nix { -PackageInfos queryInstalled(EvalState & state, const Path & userEnv); +PackageInfos queryInstalled(EvalState & state, const std::filesystem::path & userEnv); bool createUserEnv( - EvalState & state, PackageInfos & elems, const Path & profile, bool keepDerivations, const std::string & lockToken); + EvalState & state, + PackageInfos & elems, + const std::filesystem::path & profile, + bool keepDerivations, + const std::string & lockToken); } // namespace nix diff --git a/src/nix/nix-instantiate/nix-instantiate.cc b/src/nix/nix-instantiate/nix-instantiate.cc index f09b4078a245..9861b52c9ad2 100644 --- a/src/nix/nix-instantiate/nix-instantiate.cc +++ b/src/nix/nix-instantiate/nix-instantiate.cc @@ -21,7 +21,7 @@ using namespace nix; -static Path gcRoot; +std::filesystem::path gcRoot; static int rootNr = 0; enum OutputKind { okPlain, okRaw, okXML, okJSON }; @@ -74,7 +74,7 @@ void processExpr( if (strict) state.forceValueDeep(vRes); std::set seen; - printAmbiguous(state, vRes, std::cout, &seen, std::numeric_limits::max()); + printAmbiguous(state, vRes, std::cout, &seen); std::cout << std::endl; } } else { @@ -89,15 +89,15 @@ void processExpr( if (outputName == "") throw Error("derivation '%1%' lacks an 'outputName' attribute", drvPathS); - if (gcRoot == "") + if (gcRoot.empty()) printGCWarning(); else { - Path rootName = absPath(gcRoot); + auto rootName = absPath(gcRoot); if (++rootNr > 1) rootName += "-" + std::to_string(rootNr); auto store2 = state.store.dynamic_pointer_cast(); if (store2) - drvPathS = store2->addPermRoot(drvPath, rootName); + drvPathS = store2->addPermRoot(drvPath, rootName).string(); } std::cout << fmt("%s%s\n", drvPathS, (outputName != "out" ? "!" + outputName : "")); } @@ -173,9 +173,9 @@ static int main_nix_instantiate(int argc, char ** argv) settings.readOnlyMode = true; auto store = openStore(); - auto evalStore = myArgs.evalStoreUrl ? openStore(*myArgs.evalStoreUrl) : store; + auto evalStore = myArgs.evalStoreUrl ? openStore(StoreReference{*myArgs.evalStoreUrl}) : store; - auto state = std::make_unique(myArgs.lookupPath, evalStore, fetchSettings, evalSettings, store); + auto state = std::make_shared(myArgs.lookupPath, evalStore, fetchSettings, evalSettings, store); state->repair = myArgs.repair; Bindings & autoArgs = *myArgs.getAutoArgs(*state); diff --git a/src/nix/nix-store/nix-store.cc b/src/nix/nix-store/nix-store.cc index 74697ade110f..d6649d3e96dd 100644 --- a/src/nix/nix-store/nix-store.cc +++ b/src/nix/nix-store/nix-store.cc @@ -6,6 +6,7 @@ #include "nix/store/store-cast.hh" #include "nix/store/local-fs-store.hh" #include "nix/store/log-store.hh" +#include "nix/store/local-store.hh" #include "nix/store/serve-protocol.hh" #include "nix/store/serve-protocol-connection.hh" #include "nix/main/shared.hh" @@ -15,13 +16,15 @@ #include "nix/store/globals.hh" #include "nix/store/path-with-outputs.hh" #include "nix/store/export-import.hh" +#include "nix/util/strings.hh" +#include "nix/store/posix-fs-canonicalise.hh" +#include "nix/util/error.hh" +#include "nix/store/gc-store.hh" #include "man-pages.hh" #ifndef _WIN32 // TODO implement on Windows or provide allowed-to-noop interface -# include "nix/store/local-store.hh" # include "nix/util/monitor-fd.hh" -# include "nix/store/posix-fs-canonicalise.hh" #endif #include @@ -43,12 +46,11 @@ using std::cout; typedef void (*Operation)(Strings opFlags, Strings opArgs); -static Path gcRoot; +static std::filesystem::path gcRoot; static int rootNr = 0; static bool noOutput = false; static std::shared_ptr store; -#ifndef _WIN32 // TODO reenable on Windows once we have `LocalStore` there ref ensureLocalStore() { auto store2 = std::dynamic_pointer_cast(store); @@ -56,7 +58,6 @@ ref ensureLocalStore() throw Error("you don't have sufficient rights to use this command"); return ref(store2); } -#endif static StorePath useDeriver(const StorePath & path) { @@ -68,9 +69,10 @@ static StorePath useDeriver(const StorePath & path) return *info->deriver; } -/* Realise the given path. For a derivation that means build it; for - other paths it means ensure their validity. */ -static PathSet realisePath(StorePathWithOutputs path, bool build = true) +/** + * Because we are downcasting first thing to a `LocalFSStore`, we know it is OK to return local paths. + */ +static std::set realisePath(StorePathWithOutputs path, bool build = true) { auto store2 = std::dynamic_pointer_cast(store); @@ -86,19 +88,19 @@ static PathSet realisePath(StorePathWithOutputs path, bool build = true) for (auto & i : drv.outputs) path.outputs.insert(i.first); - PathSet outputs; + std::set outputs; for (auto & j : path.outputs) { /* Match outputs of a store path with outputs of the derivation that produces it. */ DerivationOutputs::iterator i = drv.outputs.find(j); if (i == drv.outputs.end()) throw Error("derivation '%s' does not have an output named '%s'", store2->printStorePath(path.path), j); auto outPath = outputPaths.at(i->first); - auto retPath = store->printStorePath(outPath); + std::filesystem::path retPath = store->printStorePath(outPath); if (store2) { if (gcRoot == "") printGCWarning(); else { - Path rootName = gcRoot; + std::filesystem::path rootName = gcRoot; if (rootNr > 1) rootName += "-" + std::to_string(rootNr); if (i->first != "out") @@ -120,14 +122,14 @@ static PathSet realisePath(StorePathWithOutputs path, bool build = true) if (gcRoot == "") printGCWarning(); else { - Path rootName = gcRoot; + std::filesystem::path rootName = gcRoot; rootNr++; if (rootNr > 1) rootName += "-" + std::to_string(rootNr); return {store2->addPermRoot(path.path, rootName)}; } } - return {store->printStorePath(path.path)}; + return {std::filesystem::path{store->printStorePath(path.path)}}; } } @@ -180,7 +182,7 @@ static void opRealise(Strings opFlags, Strings opArgs) auto paths2 = realisePath(i, false); if (!noOutput) for (auto & j : paths2) - cout << fmt("%1%\n", j); + cout << fmt("%s\n", j.string()); } } @@ -503,7 +505,8 @@ static void opQuery(Strings opFlags, Strings opArgs) args.insert(p); StorePathSet referrers; - store->computeFSClosure(args, referrers, true, settings.gcKeepOutputs, settings.gcKeepDerivations); + auto & gcSettings = settings.getLocalSettings().getGCSettings(); + store->computeFSClosure(args, referrers, true, gcSettings.keepOutputs, gcSettings.keepDerivations); auto & gcStore = require(*store); Roots roots = gcStore.findRoots(false); @@ -526,25 +529,17 @@ static void opPrintEnv(Strings opFlags, Strings opArgs) if (opArgs.size() != 1) throw UsageError("'--print-env' requires one derivation store path"); - Path drvPath = opArgs.front(); - Derivation drv = store->derivationFromPath(store->parseStorePath(drvPath)); + StorePath drvPath = store->parseStorePath(opArgs.front()); + Derivation drv = store->derivationFromPath(drvPath); /* Print each environment variable in the derivation in a format * that can be sourced by the shell. */ for (auto & i : drv.env) logger->cout("export %1%; %1%=%2%\n", i.first, escapeShellArgAlways(i.second)); - /* Also output the arguments. This doesn't preserve whitespace in - arguments. */ - cout << "export _args; _args='"; - bool first = true; - for (auto & i : drv.args) { - if (!first) - cout << ' '; - first = false; - cout << escapeShellArgAlways(i); - } - cout << "'\n"; + /* Also output the arguments. */ + std::string argsStr = concatStringsSep(" ", drv.args); + cout << "export _args; _args=" << escapeShellArgAlways(argsStr) << "\n"; } static void opReadLog(Strings opFlags, Strings opArgs) @@ -596,11 +591,9 @@ static void registerValidity(bool reregister, bool hashGiven, bool canonicalise) if (!store->isValidPath(info->path) || reregister) { /* !!! races */ if (canonicalise) -#ifdef _WIN32 // TODO implement on Windows - throw UnimplementedError("file attribute canonicalisation Is not implemented on Windows"); -#else - canonicalisePathMetaData(store->printStorePath(info->path), {}); -#endif + canonicalisePathMetaData( + store->printStorePath(info->path), + {NIX_WHEN_SUPPORT_ACLS(settings.getLocalSettings().ignoredAcls)}); if (!hashGiven) { HashResult hash = hashPath( {store->requireStoreObjectAccessor(info->path, /*requireValidPath=*/false)}, @@ -613,9 +606,7 @@ static void registerValidity(bool reregister, bool hashGiven, bool canonicalise) } } -#ifndef _WIN32 // TODO reenable on Windows once we have `LocalStore` there ensureLocalStore()->registerValidPaths(infos); -#endif } static void opLoadDB(Strings opFlags, Strings opArgs) @@ -691,11 +682,15 @@ static void opGC(Strings opFlags, Strings opArgs) if (!opArgs.empty()) throw UsageError("no arguments expected"); + if (options.maxFreed != std::numeric_limits::max() + && (options.action == GCOptions::gcReturnDead || options.action == GCOptions::gcReturnLive || printRoots)) + throw UsageError("option --max-freed cannot be combined with --print-live, --print-dead, or --print-roots"); + auto & gcStore = require(*store); if (printRoots) { Roots roots = gcStore.findRoots(false); - std::set> roots2; + std::set> roots2; // Transpose and sort the roots. for (auto & [target, links] : roots) for (auto & link : links) @@ -705,12 +700,14 @@ static void opGC(Strings opFlags, Strings opArgs) } else { - PrintFreed freed(options.action == GCOptions::gcDeleteDead, results); + Finally printer([&] { + if (options.action != GCOptions::gcDeleteDead) + for (auto & i : results.paths) + cout << i << std::endl; + else + printFreed(false, results); + }); gcStore.collectGarbage(options, results); - - if (options.action != GCOptions::gcDeleteDead) - for (auto & i : results.paths) - cout << i << std::endl; } } @@ -734,7 +731,7 @@ static void opDelete(Strings opFlags, Strings opArgs) auto & gcStore = require(*store); GCResults results; - PrintFreed freed(true, results); + Finally printer([&] { printFreed(false, results); }); gcStore.collectGarbage(options, results); } @@ -894,7 +891,7 @@ static void opServe(Strings opFlags, Strings opArgs) FdSink out(getStandardOutput()); /* Exchange the greeting. */ - ServeProto::Version clientVersion = ServeProto::BasicServerConnection::handshake(out, in, SERVE_PROTOCOL_VERSION); + ServeProto::Version clientVersion = ServeProto::BasicServerConnection::handshake(out, in, ServeProto::latest); ServeProto::ReadConn rconn{ .from = in, @@ -909,8 +906,8 @@ static void opServe(Strings opFlags, Strings opArgs) // FIXME: changing options here doesn't work if we're // building through the daemon. verbosity = lvlError; - settings.keepLog = false; - settings.useSubstitutes = false; + settings.getLogFileSettings().keepLog = false; + settings.getWorkerSettings().useSubstitutes = false; auto options = ServeProto::Serialise::read(*store, rconn); @@ -919,11 +916,11 @@ static void opServe(Strings opFlags, Strings opArgs) // See how the serialization logic in // `ServeProto::Serialise` matches // these conditions. - settings.maxSilentTime = options.maxSilentTime; - settings.buildTimeout = options.buildTimeout; - if (GET_PROTOCOL_MINOR(clientVersion) >= 2) - settings.maxLogSize = options.maxLogSize; - if (GET_PROTOCOL_MINOR(clientVersion) >= 3) { + settings.getWorkerSettings().maxSilentTime = options.maxSilentTime; + settings.getWorkerSettings().buildTimeout = options.buildTimeout; + if (clientVersion >= ServeProto::Version{2, 2}) + settings.getWorkerSettings().maxLogSize = options.maxLogSize; + if (clientVersion >= ServeProto::Version{2, 3}) { if (options.nrRepeats != 0) { throw Error("client requested repeating builds, but this is not currently implemented"); } @@ -934,9 +931,9 @@ static void opServe(Strings opFlags, Strings opArgs) // checked that `nrRepeats` in fact is 0, so we can safely // ignore this without doing something other than what the // client asked for. - settings.runDiffHook = true; + settings.getLocalSettings().runDiffHook = true; } - if (GET_PROTOCOL_MINOR(clientVersion) >= 7) { + if (clientVersion >= ServeProto::Version{2, 7}) { settings.keepFailed = options.keepFailed; } }; @@ -1066,7 +1063,7 @@ static void opServe(Strings opFlags, Strings opArgs) info.deriver = store->parseStorePath(deriver); info.references = ServeProto::Serialise::read(*store, rconn); in >> info.registrationTime >> info.narSize >> info.ultimate; - info.sigs = readStrings(in); + info.sigs = ServeProto::Serialise>::read(*store, rconn); info.ca = ContentAddress::parseOpt(readString(in)); if (info.narSize == 0) @@ -1106,9 +1103,8 @@ static void opGenerateBinaryCacheKey(Strings opFlags, Strings opArgs) auto secretKey = SecretKey::generate(keyName); - writeFile(publicKeyFile, secretKey.toPublicKey().to_string()); - umask(0077); - writeFile(secretKeyFile, secretKey.to_string()); + writeFile(publicKeyFile, secretKey.toPublicKey().to_string(), 0666, FsSync::Yes); + writeFile(secretKeyFile, secretKey.to_string(), 0600, FsSync::Yes); } static void opVersion(Strings opFlags, Strings opArgs) diff --git a/src/nix/package.nix b/src/nix/package.nix index 9c223bceb2ba..fa52f2c5c91e 100644 --- a/src/nix/package.nix +++ b/src/nix/package.nix @@ -90,18 +90,12 @@ mkMesonExecutable (finalAttrs: { # For some reason that is not clear, it is wanting to use libgcc_eh which is not available. # Force this to be built with compiler-rt & libunwind over libgcc_eh works. # Issue: https://github.com/NixOS/nixpkgs/issues/177129 - NIX_CFLAGS_COMPILE = - lib.optionals - ( - stdenv.cc.isClang - && stdenv.hostPlatform.isStatic - && stdenv.cc.libcxx != null - && stdenv.cc.libcxx.isLLVM - ) - [ - "-rtlib=compiler-rt" - "-unwindlib=libunwind" - ]; + NIX_CFLAGS_COMPILE = lib.optionalString ( + stdenv.cc.isClang + && stdenv.hostPlatform.isStatic + && stdenv.cc.libcxx != null + && stdenv.cc.libcxx.isLLVM + ) "-rtlib=compiler-rt -unwindlib=libunwind"; meta = { mainProgram = "nix"; diff --git a/src/nix/path-info.cc b/src/nix/path-info.cc index 6bffe2424e42..3e36197b958d 100644 --- a/src/nix/path-info.cc +++ b/src/nix/path-info.cc @@ -227,7 +227,7 @@ struct CmdPathInfo : StorePathsCommand, MixJSON if (info->ca) ss.push_back("ca:" + renderContentAddress(*info->ca)); for (auto & sig : info->sigs) - ss.push_back(sig); + ss.push_back(sig.to_string()); str << concatStringsSep(" ", ss); } diff --git a/src/nix/prefetch.cc b/src/nix/prefetch.cc index 781677cb4fe5..be7aacf3103a 100644 --- a/src/nix/prefetch.cc +++ b/src/nix/prefetch.cc @@ -114,9 +114,9 @@ std::tuple prefetchFile( if (executable) mode = 0700; - AutoCloseFD fd = toDescriptor(open(tmpFile.string().c_str(), O_WRONLY | O_CREAT | O_EXCL, mode)); + auto fd = openNewFileForWrite(tmpFile, mode, {.truncateExisting = false}); if (!fd) - throw SysError("creating temporary file '%s'", tmpFile); + throw SysError("creating temporary file %s", PathFmt(tmpFile)); FdSink sink(fd.get()); @@ -128,11 +128,13 @@ std::tuple prefetchFile( /* Optionally unpack the file. */ if (unpack) { Activity act(*logger, lvlChatty, actUnknown, fmt("unpacking '%s'", url.to_string())); - auto unpacked = (tmpDir.path() / "unpacked").string(); + auto unpacked = tmpDir.path() / "unpacked"; createDirs(unpacked); - unpackTarfile(tmpFile.string(), unpacked); + unpackTarfile(tmpFile, unpacked); auto entries = DirectoryIterator{unpacked}; + if (entries == DirectoryIterator{}) + throw Error("archive '%s' is empty", url.to_string()); /* If the archive unpacks to a single file/directory, then use that as the top-level. */ tmpFile = entries->path(); @@ -213,7 +215,7 @@ static int main_nix_prefetch_url(int argc, char ** argv) setLogFormat("bar"); auto store = openStore(); - auto state = std::make_unique(myArgs.lookupPath, store, fetchSettings, evalSettings); + auto state = std::make_shared(myArgs.lookupPath, store, fetchSettings, evalSettings); Bindings & autoArgs = *myArgs.getAutoArgs(*state); diff --git a/src/nix/profile.cc b/src/nix/profile.cc index 25a4110b89cc..5a856e5e60ce 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -140,7 +140,7 @@ struct ProfileManifest sOriginalUrl = "originalUrl"; break; default: - throw Error("profile manifest '%s' has unsupported version %d", manifestPath, version); + throw Error("profile manifest %s has unsupported version %d", PathFmt(manifestPath), version); } auto elems = json["elements"]; @@ -415,7 +415,7 @@ struct CmdProfileAdd : InstallablesCommand, MixDefaultProfile } try { - updateProfile(manifest.build(store)); + updateProfile(*store, manifest.build(store)); } catch (BuildEnvFileConflictError & conflictError) { // FIXME use C++20 std::ranges once macOS has it // See @@ -424,10 +424,10 @@ struct CmdProfileAdd : InstallablesCommand, MixDefaultProfile for (auto it = begin; it != end; it++) { auto & [name, profileElement] = *it; for (auto & storePath : profileElement.storePaths) { - if (conflictError.fileA.starts_with(store->printStorePath(storePath))) { + if (conflictError.fileA.string().starts_with(store->printStorePath(storePath))) { return std::tuple(conflictError.fileA, name, profileElement.toInstallables(*store)); } - if (conflictError.fileB.starts_with(store->printStorePath(storePath))) { + if (conflictError.fileB.string().starts_with(store->printStorePath(storePath))) { return std::tuple(conflictError.fileB, name, profileElement.toInstallables(*store)); } } @@ -465,8 +465,8 @@ struct CmdProfileAdd : InstallablesCommand, MixDefaultProfile "To prioritise the existing package:\n" "\n" " nix profile add %4% --priority %7%\n", - originalConflictingFilePath, - newConflictingFilePath, + PathFmt(originalConflictingFilePath), + PathFmt(newConflictingFilePath), originalEntryName, concatStringsSep(" ", newConflictingRefs), conflictError.priority, @@ -688,7 +688,7 @@ struct CmdProfileRemove : virtual EvalCommand, MixProfileElementMatchers auto removedCount = oldManifest.elements.size() - newManifest.elements.size(); printInfo("removed %d packages, kept %d packages", removedCount, newManifest.elements.size()); - updateProfile(newManifest.build(store)); + updateProfile(*store, newManifest.build(store)); } }; @@ -797,7 +797,7 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixProfileElementMatchers element.updateStorePaths(getEvalStore(), store, builtPaths.find(&*installable)->second.first); } - updateProfile(manifest.build(store)); + updateProfile(*store, manifest.build(store)); } }; diff --git a/src/nix/provenance.cc b/src/nix/provenance.cc index b863cc07d685..fddda617e2ac 100644 --- a/src/nix/provenance.cc +++ b/src/nix/provenance.cc @@ -97,7 +97,7 @@ struct CmdProvenanceShow : StorePathsCommand if (auto tree = std::dynamic_pointer_cast(next)) { FlakeRef flakeRef( fetchers::Input::fromAttrs(fetchSettings, fetchers::jsonToAttrs(*tree->attrs)), - Path(flakePath.parent().value_or(CanonPath::root).rel())); + std::string(flakePath.parent().value_or(CanonPath::root).rel())); logger->cout( "← %sinstantiated from %sflake output " ANSI_BOLD "%s#%s" ANSI_NORMAL, flake->pure ? "" : ANSI_RED "impurely" ANSI_NORMAL " ", diff --git a/src/nix/registry.cc b/src/nix/registry.cc index 6d913adcd6cb..c943e80f9e1c 100644 --- a/src/nix/registry.cc +++ b/src/nix/registry.cc @@ -40,7 +40,7 @@ class RegistryCommand : virtual Args return registry; } - Path getRegistryPath() + std::filesystem::path getRegistryPath() { if (registry_path.empty()) { return fetchers::getUserRegistryPath().string(); @@ -118,7 +118,7 @@ struct CmdRegistryAdd : MixEvalArgs, Command, RegistryCommand extraAttrs["dir"] = toRef.subdir; registry->remove(fromRef.input); registry->add(fromRef.input, toRef.input, extraAttrs); - registry->write(getRegistryPath()); + registry->write(getRegistryPath().string()); } }; @@ -147,7 +147,7 @@ struct CmdRegistryRemove : RegistryCommand, Command { auto registry = getRegistry(); registry->remove(parseFlakeRef(fetchSettings, url).input); - registry->write(getRegistryPath()); + registry->write(getRegistryPath().string()); } }; @@ -198,7 +198,7 @@ struct CmdRegistryPin : RegistryCommand, EvalCommand extraAttrs["dir"] = ref.subdir; registry->remove(ref.input); registry->add(ref.input, resolved, extraAttrs); - registry->write(getRegistryPath()); + registry->write(getRegistryPath().string()); } }; diff --git a/src/nix/repl.cc b/src/nix/repl.cc index c5787166ab64..cd790c0b09d2 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -6,20 +6,22 @@ #include "nix/cmd/command.hh" #include "nix/cmd/installable-value.hh" #include "nix/cmd/repl.hh" +#include "nix/util/os-string.hh" #include "nix/util/processes.hh" +#include "nix/util/environment-variables.hh" #include "self-exe.hh" namespace nix { -void runNix(const std::string & program, const Strings & args, const std::optional & input = {}) +void runNix(const std::string & program, OsStrings args, const std::optional & input = {}) { - auto subprocessEnv = getEnv(); - subprocessEnv["NIX_CONFIG"] = globalConfig.toKeyValue(); + auto subprocessEnv = getEnvOs(); + subprocessEnv[OS_STR("NIX_CONFIG")] = string_to_os_string(globalConfig.toKeyValue()); // isInteractive avoid grabling interactive commands runProgram2( RunOptions{ .program = getNixBin(program).string(), - .args = args, + .args = std::move(args), .environment = subprocessEnv, .input = input, .isInteractive = true, @@ -99,7 +101,7 @@ struct CmdRepl : RawInstallablesCommand } return values; }; - auto repl = AbstractNixRepl::create(lookupPath, openStore(), state, getValues, runNix); + auto repl = AbstractNixRepl::create(lookupPath, state, getValues, runNix); repl->autoArgs = getAutoArgs(*repl->state); repl->initEnv(); repl->mainLoop(); diff --git a/src/nix/run.cc b/src/nix/run.cc index 8b7a518c9a69..29c66a242a51 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -24,10 +24,6 @@ extern char ** environ __attribute__((weak)); -namespace nix::fs { -using namespace std::filesystem; -} - using namespace nix; std::string chrootHelperName = "__run_in_chroot"; @@ -85,18 +81,25 @@ void execProgramInStore( if (store->storeDir != store2->getRealStoreDir()) { Strings helperArgs = { - chrootHelperName, store->storeDir, store2->getRealStoreDir(), std::string(system.value_or("")), program}; + chrootHelperName, + store->storeDir, + store2->getRealStoreDir().string(), + std::string(system.value_or("")), + program}; for (auto & arg : args) helperArgs.push_back(arg); - execve(getSelfExe().value_or("nix").c_str(), stringsToCharPtrs(helperArgs).data(), envp); + execve(getSelfExe().value_or("nix").string().c_str(), stringsToCharPtrs(helperArgs).data(), envp); throw SysError("could not execute chroot helper"); } #ifdef __linux__ if (system) - linux::setPersonality(*system); + linux::setPersonality({ + .system = *system, + .impersonateLinux26 = settings.getLocalSettings().impersonateLinux26, + }); #endif if (useLookupPath == UseLookupPath::Use) { @@ -208,9 +211,9 @@ void chrootHelper(int argc, char ** argv) auto st = entry.symlink_status(); if (std::filesystem::is_directory(st)) { if (mkdir(dst.c_str(), 0700) == -1) - throw SysError("creating directory '%s'", dst); + throw SysError("creating directory %s", PathFmt(dst)); if (mount(src.c_str(), dst.c_str(), "", MS_BIND | MS_REC, 0) == -1) - throw SysError("mounting '%s' on '%s'", src, dst); + throw SysError("mounting %s on %s", PathFmt(src), PathFmt(dst)); } else if (std::filesystem::is_symlink(st)) createSymlink(readLink(src), dst); } @@ -221,7 +224,7 @@ void chrootHelper(int argc, char ** argv) Finally freeCwd([&]() { free(cwd); }); if (chroot(tmpDir.c_str()) == -1) - throw SysError("chrooting into '%s'", tmpDir); + throw SysError("chrooting into %s", PathFmt(tmpDir)); if (chdir(cwd) == -1) throw SysError("chdir to '%s' in chroot", cwd); @@ -237,7 +240,10 @@ void chrootHelper(int argc, char ** argv) # ifdef __linux__ if (system != "") - linux::setPersonality(system); + linux::setPersonality({ + .system = system, + .impersonateLinux26 = settings.getLocalSettings().impersonateLinux26, + }); # endif execvp(cmd.c_str(), stringsToCharPtrs(args).data()); diff --git a/src/nix/sigs.cc b/src/nix/sigs.cc index e82f0d284b95..c72204cea3d4 100644 --- a/src/nix/sigs.cc +++ b/src/nix/sigs.cc @@ -11,7 +11,7 @@ using namespace nix; struct CmdCopySigs : StorePathsCommand { - Strings substituterUris; + std::vector substituterUris; CmdCopySigs() { @@ -20,7 +20,7 @@ struct CmdCopySigs : StorePathsCommand .shortName = 's', .description = "Copy signatures from the specified store.", .labels = {"store-uri"}, - .handler = {[&](std::string s) { substituterUris.push_back(s); }}, + .handler = {[&](std::string s) { substituterUris.push_back(StoreReference::parse(s)); }}, }); } @@ -44,7 +44,7 @@ struct CmdCopySigs : StorePathsCommand // FIXME: factor out commonality with MixVerify. std::vector> substituters; for (auto & s : substituterUris) - substituters.push_back(openStore(s)); + substituters.push_back(openStore(StoreReference{s})); ThreadPool pool{fileTransferSettings.httpConnections}; @@ -52,16 +52,14 @@ struct CmdCopySigs : StorePathsCommand // logger->setExpected(doneLabel, storePaths.size()); - auto doPath = [&](const Path & storePathS) { + auto doPath = [&](const StorePath & storePath) { // Activity act(*logger, lvlInfo, "getting signatures for '%s'", storePath); checkInterrupt(); - auto storePath = store->parseStorePath(storePathS); - auto info = store->queryPathInfo(storePath); - StringSet newSigs; + std::set newSigs; for (auto & store2 : substituters) { try { @@ -89,7 +87,7 @@ struct CmdCopySigs : StorePathsCommand }; for (auto & storePath : storePaths) - pool.enqueue(std::bind(doPath, store->printStorePath(storePath))); + pool.enqueue(std::bind(doPath, storePath)); pool.process(); @@ -101,7 +99,7 @@ static auto rCmdCopySigs = registerCommand2({"store", "copy-sigs"}) struct CmdSign : StorePathsCommand { - Path secretKeyFile; + std::filesystem::path secretKeyFile; CmdSign() { diff --git a/src/nix/store-delete.cc b/src/nix/store-delete.cc index 42517c8828ed..3e2e0f2ba517 100644 --- a/src/nix/store-delete.cc +++ b/src/nix/store-delete.cc @@ -40,7 +40,7 @@ struct CmdStoreDelete : StorePathsCommand options.pathsToDelete.insert(path); GCResults results; - PrintFreed freed(true, results); + Finally printer([&] { printFreed(false, results); }); gcStore.collectGarbage(options, results); } }; diff --git a/src/nix/store-gc.cc b/src/nix/store-gc.cc index b0a627837ce6..971be26be5b3 100644 --- a/src/nix/store-gc.cc +++ b/src/nix/store-gc.cc @@ -4,6 +4,8 @@ #include "nix/store/store-api.hh" #include "nix/store/store-cast.hh" #include "nix/store/gc-store.hh" +#include "nix/util/error.hh" +#include "nix/util/logging.hh" using namespace nix; @@ -15,7 +17,7 @@ struct CmdStoreGC : StoreCommand, MixDryRun { addFlag({ .longName = "max", - .description = "Stop after freeing *n* bytes of disk space.", + .description = "Stop after freeing *n* bytes of disk space. Cannot be combined with --dry-run.", .labels = {"n"}, .handler = {&options.maxFreed}, }); @@ -35,11 +37,14 @@ struct CmdStoreGC : StoreCommand, MixDryRun void run(ref store) override { + if (options.maxFreed != std::numeric_limits::max() && dryRun) + throw UsageError("options --max and --dry-run cannot be combined"); + auto & gcStore = require(*store); options.action = dryRun ? GCOptions::gcReturnDead : GCOptions::gcDeleteDead; GCResults results; - PrintFreed freed(options.action == GCOptions::gcDeleteDead, results); + Finally printer([&] { printFreed(dryRun, results); }); gcStore.collectGarbage(options, results); } }; diff --git a/src/nix/unix/daemon.cc b/src/nix/unix/daemon.cc index 661488c56ef9..7fce2e8f6752 100644 --- a/src/nix/unix/daemon.cc +++ b/src/nix/unix/daemon.cc @@ -15,6 +15,7 @@ #include "nix/store/derivations.hh" #include "nix/util/finally.hh" #include "nix/cmd/legacy.hh" +#include "nix/cmd/unix-socket-server.hh" #include "nix/store/daemon.hh" #include "man-pages.hh" @@ -27,8 +28,6 @@ #include #include #include -#include -#include #include #include #include @@ -39,10 +38,6 @@ # include "nix/util/cgroup.hh" #endif -#if defined(__APPLE__) || defined(__FreeBSD__) -# include -#endif - using namespace nix; using namespace nix::daemon; @@ -197,63 +192,6 @@ matchUser(const std::optional & user, const std::optional pid; - std::optional uid; - std::optional gid; -}; - -/** - * Get the identity of the caller, if possible. - */ -static PeerInfo getPeerInfo(int remote) -{ - PeerInfo peer; - -#if defined(SO_PEERCRED) - -# if defined(__OpenBSD__) - struct sockpeercred cred; -# else - ucred cred; -# endif - socklen_t credLen = sizeof(cred); - if (getsockopt(remote, SOL_SOCKET, SO_PEERCRED, &cred, &credLen) == 0) { - peer.pid = cred.pid; - peer.uid = cred.uid; - peer.gid = cred.gid; - } - -#elif defined(LOCAL_PEERCRED) - -# if !defined(SOL_LOCAL) -# define SOL_LOCAL 0 -# endif - - xucred cred; - socklen_t credLen = sizeof(cred); - if (getsockopt(remote, SOL_LOCAL, LOCAL_PEERCRED, &cred, &credLen) == 0) - peer.uid = cred.cr_uid; - -#endif - - return peer; -} - -#define SD_LISTEN_FDS_START 3 - -/** - * Open a store without a path info cache. - */ -static ref openUncachedStore() -{ - Store::Config::Params params; // FIXME: get params from somewhere - // Disable caching since the client already does that. - params["path-info-cache-size"] = "0"; - return openStore(settings.storeUri, params); -} - /** * Authenticate a potential client * @@ -265,7 +203,7 @@ static ref openUncachedStore() * * If the potential client is not allowed to talk to us, we throw an `Error`. */ -static std::pair> authPeer(const PeerInfo & peer) +static std::pair> authPeer(const unix::PeerInfo & peer) { TrustedFlag trusted = NotTrusted; @@ -285,7 +223,7 @@ static std::pair> authPeer(const PeerInf if (matchUser(user, group, trustedUsers)) trusted = Trusted; - if ((!trusted && !matchUser(user, group, allowedUsers)) || group == settings.buildUsersGroup) + if ((!trusted && !matchUser(user, group, allowedUsers)) || group == settings.getLocalSettings().buildUsersGroup) throw Error("user '%1%' is not allowed to connect to the Nix daemon", user.value_or("")); return {trusted, std::move(user)}; @@ -295,37 +233,21 @@ static std::pair> authPeer(const PeerInf * Run a server. The loop opens a socket and accepts new connections from that * socket. * + * @param storeConfig The store configuration to use for opening stores. * @param forceTrustClientOpt If present, force trusting or not trusted * the client. Otherwise, decide based on the authentication settings * and user credentials (from the unix domain socket). */ -static void daemonLoop(std::optional forceTrustClientOpt) +static void daemonLoop(ref storeConfig, std::optional forceTrustClientOpt) { if (chdir("/") == -1) throw SysError("cannot change current directory"); - AutoCloseFD fdSocket; - - // Handle socket-based activation by systemd. - auto listenFds = getEnv("LISTEN_FDS"); - if (listenFds) { - if (getEnv("LISTEN_PID") != std::to_string(getpid()) || listenFds != "1") - throw Error("unexpected systemd environment variables"); - fdSocket = SD_LISTEN_FDS_START; - unix::closeOnExec(fdSocket.get()); - } - - // Otherwise, create and bind to a Unix domain socket. - else { - createDirs(dirOf(settings.nixDaemonSocketFile)); - fdSocket = createUnixDomainSocket(settings.nixDaemonSocketFile, 0666); - } - // Get rid of children automatically; don't let them become zombies. setSigChldAction(true); #ifdef __linux__ - if (settings.useCgroups) { + if (settings.getLocalSettings().useCgroups) { experimentalFeatureSettings.require(Xp::Cgroups); // This also sets the root cgroup to the current one. @@ -333,9 +255,9 @@ static void daemonLoop(std::optional forceTrustClientOpt) auto cgroupFS = getCgroupFS(); if (!cgroupFS) throw Error("cannot determine the cgroups file system"); - auto rootCgroupPath = canonPath(*cgroupFS + "/" + rootCgroup); + auto rootCgroupPath = *cgroupFS / rootCgroup.rel(); if (!pathExists(rootCgroupPath)) - throw Error("expected cgroup directory '%s'", rootCgroupPath); + throw Error("expected cgroup directory %s", PathFmt(rootCgroupPath)); auto daemonCgroupPath = rootCgroupPath + "/nix-daemon"; // Create new sub-cgroup for the daemon. if (mkdir(daemonCgroupPath.c_str(), 0755) != 0 && errno != EEXIST) @@ -345,81 +267,67 @@ static void daemonLoop(std::optional forceTrustClientOpt) } #endif - // Loop accepting connections. - while (1) { - - try { - // Accept a connection. - struct sockaddr_un remoteAddr; - socklen_t remoteAddrLen = sizeof(remoteAddr); - - AutoCloseFD remote = accept(fdSocket.get(), (struct sockaddr *) &remoteAddr, &remoteAddrLen); - checkInterrupt(); - if (!remote) { - if (errno == EINTR) - continue; - throw SysError("accepting connection"); - } - - unix::closeOnExec(remote.get()); - - PeerInfo peer; - TrustedFlag trusted; - std::optional userName; - - if (forceTrustClientOpt) - trusted = *forceTrustClientOpt; - else { - peer = getPeerInfo(remote.get()); - auto [_trusted, _userName] = authPeer(peer); - trusted = _trusted; - userName = _userName; - }; - - printInfo( - (std::string) "accepted connection from pid %1%, user %2%" + (trusted ? " (trusted)" : ""), - peer.pid ? std::to_string(*peer.pid) : "", - userName.value_or("")); - - // Fork a child to handle the connection. - ProcessOptions options; - options.errorPrefix = "unexpected Nix daemon error: "; - options.dieWithParent = false; - options.runExitHandlers = true; - options.allowVfork = false; - startProcess( - [&]() { - fdSocket = -1; - - // Background the daemon. - if (setsid() == -1) - throw SysError("creating a new session"); - - // Restore normal handling of SIGCHLD. - setSigChldAction(false); - - // For debugging, stuff the pid into argv[1]. - if (peer.pid && savedArgv[1]) { - auto processName = std::to_string(*peer.pid); - strncpy(savedArgv[1], processName.c_str(), strlen(savedArgv[1])); - } - - // Handle the connection. - processConnection( - openUncachedStore(), FdSource(remote.get()), FdSink(remote.get()), trusted, NotRecursive); - - exit(0); - }, - options); - - } catch (Interrupted & e) { - return; - } catch (Error & error) { - auto ei = error.info(); - // FIXME: add to trace? - ei.msg = HintFmt("while processing connection: %1%", ei.msg.str()); - logError(ei); - } + try { + unix::serveUnixSocket( + { + .socketPath = settings.nixDaemonSocketFile, + .socketMode = 0666, + }, + [&](AutoCloseFD remote, std::function closeListeners) { + unix::closeOnExec(remote.get()); + + unix::PeerInfo peer; + TrustedFlag trusted; + std::optional userName; + + if (forceTrustClientOpt) + trusted = *forceTrustClientOpt; + else { + peer = unix::getPeerInfo(remote.get()); + auto [_trusted, _userName] = authPeer(peer); + trusted = _trusted; + userName = _userName; + }; + + printInfo( + (std::string) "accepted connection from pid %1%, user %2%" + (trusted ? " (trusted)" : ""), + peer.pid ? std::to_string(*peer.pid) : "", + userName.value_or("")); + + // Fork a child to handle the connection. + ProcessOptions options; + options.errorPrefix = "unexpected Nix daemon error: "; + options.dieWithParent = false; + options.runExitHandlers = true; + options.allowVfork = false; + startProcess( + [&, storeConfig, closeListeners = std::move(closeListeners)]() { + closeListeners(); + + // Background the daemon. + if (setsid() == -1) + throw SysError("creating a new session"); + + // Restore normal handling of SIGCHLD. + setSigChldAction(false); + + // For debugging, stuff the pid into argv[1]. + if (peer.pid && savedArgv[1]) { + auto processName = std::to_string(*peer.pid); + strncpy(savedArgv[1], processName.c_str(), strlen(savedArgv[1])); + } + + // Handle the connection. + auto store = storeConfig->openStore(); + store->init(); + processConnection(store, FdSource(remote.get()), FdSink(remote.get()), trusted, NotRecursive); + + exit(0); + }, + options); + }); + } catch (Interrupted & e) { + return; } } @@ -481,16 +389,22 @@ static void processStdioConnection(ref store, TrustedFlag trustClient) * Entry point shared between the new CLI `nix daemon` and old CLI * `nix-daemon`. * + * @param storeConfig The store configuration to use for opening stores. * @param forceTrustClientOpt See `daemonLoop()` and the parameter with * the same name over there for details. * * @param processOps Whether to force processing ops even if the next * store also is a remote store and could process it directly. */ -static void runDaemon(bool stdio, std::optional forceTrustClientOpt, bool processOps) +static void +runDaemon(ref storeConfig, bool stdio, std::optional forceTrustClientOpt, bool processOps) { + // Disable caching since the client already does that. + storeConfig->pathInfoCacheSize = 0; + if (stdio) { - auto store = openUncachedStore(); + auto store = storeConfig->openStore(); + store->init(); std::shared_ptr remoteStore; @@ -507,7 +421,7 @@ static void runDaemon(bool stdio, std::optional forceTrustClientOpt // access to those is explicitly not `nix-daemon`'s responsibility. processStdioConnection(store, forceTrustClientOpt.value_or(Trusted)); } else - daemonLoop(forceTrustClientOpt); + daemonLoop(storeConfig, forceTrustClientOpt); } static int main_nix_daemon(int argc, char ** argv) @@ -543,7 +457,7 @@ static int main_nix_daemon(int argc, char ** argv) return true; }); - runDaemon(stdio, isTrustedOpt, processOps); + runDaemon(resolveStoreConfig(StoreReference{settings.storeUri.get()}), stdio, isTrustedOpt, processOps); return 0; } @@ -551,7 +465,7 @@ static int main_nix_daemon(int argc, char ** argv) static RegisterLegacyCommand r_nix_daemon("nix-daemon", main_nix_daemon); -struct CmdDaemon : Command +struct CmdDaemon : StoreConfigCommand { bool stdio = false; std::optional isTrustedOpt = std::nullopt; @@ -561,7 +475,7 @@ struct CmdDaemon : Command { addFlag({ .longName = "stdio", - .description = "Attach to standard I/O, instead of trying to bind to a UNIX socket.", + .description = "Attach to standard I/O, instead of using UNIX socket(s).", .handler = {&stdio, true}, }); @@ -616,9 +530,9 @@ struct CmdDaemon : Command ; } - void run() override + void run(ref storeConfig) override { - runDaemon(stdio, isTrustedOpt, processOps); + runDaemon(std::move(storeConfig), stdio, isTrustedOpt, processOps); } }; diff --git a/src/nix/unix/daemon.md b/src/nix/unix/daemon.md index b1ea850ede25..423627ef5bb0 100644 --- a/src/nix/unix/daemon.md +++ b/src/nix/unix/daemon.md @@ -42,4 +42,18 @@ management framework such as `systemd` on Linux, or `launchctl` on Darwin. Note that this daemon does not fork into the background. +# Systemd socket activation + +`nix daemon` supports systemd socket-based activation using the +`nix-daemon.socket` unit in the Nix distribution. It supports +listening on multiple addresses; for example, the following stanza in +`nix-daemon.socket` makes the daemon listen on two Unix domain +sockets: + +``` +[Socket] +ListenStream=/nix/var/nix/daemon-socket/socket +ListenStream=/nix/var/nix/daemon-socket/socket-2 +``` + )"" diff --git a/src/nix/unix/store-roots-daemon.cc b/src/nix/unix/store-roots-daemon.cc new file mode 100644 index 000000000000..c9728a2efe98 --- /dev/null +++ b/src/nix/unix/store-roots-daemon.cc @@ -0,0 +1,66 @@ +#include "nix/cmd/command.hh" +#include "nix/cmd/unix-socket-server.hh" +#include "nix/store/local-store.hh" +#include "nix/store/store-api.hh" +#include "nix/store/local-gc.hh" +#include "nix/util/file-descriptor.hh" + +#include + +using namespace nix; + +struct CmdRootsDaemon : StoreConfigCommand +{ + CmdRootsDaemon() {} + + std::string description() override + { + return "run a daemon that returns garbage collector roots on request"; + } + + std::string doc() override + { + return +#include "store-roots-daemon.md" + ; + } + + std::optional experimentalFeature() override + { + return Xp::LocalOverlayStore; + } + + void run(ref storeConfig) override + { + auto localStoreConfig = dynamic_cast(&*storeConfig); + if (!localStoreConfig) { + throw UsageError( + "Roots daemon only functions with a local store, not '%s'", storeConfig->getHumanReadableURI()); + } + + auto gcSocketPath = localStoreConfig->getRootsSocketPath(); + + unix::serveUnixSocket( + { + .socketPath = gcSocketPath, + .socketMode = 0666, + }, + [&](AutoCloseFD remote, std::function closeListeners) { + std::thread([&, remote = std::move(remote)]() mutable { + auto roots = findRuntimeRootsUnchecked(*localStoreConfig); + + FdSink sink(remote.get()); + + for (auto & [key, _] : roots) { + sink(localStoreConfig->printStorePath(key)); + sink(std::string_view("\0", 1)); + } + + sink.flush(); + remote.close(); + }).detach(); + }); + } +}; + +static auto rCmdStoreRootsDaemon = registerCommand2({"store", "roots-daemon"}); diff --git a/src/nix/unix/store-roots-daemon.md b/src/nix/unix/store-roots-daemon.md new file mode 100644 index 000000000000..5f4f778ed4d7 --- /dev/null +++ b/src/nix/unix/store-roots-daemon.md @@ -0,0 +1,46 @@ +R""( + +# Examples + +* Run the daemon: + + ```console + # nix store roots-daemon + ``` + +# Description + +This command runs a daemon that serves garbage collector roots from a Unix domain socket. +It is not required in all Nix installations, but is useful when the main Nix daemon +is not running as root and therefore cannot find runtime roots by scanning `/proc`. + +When the garbage collector runs with [`use-roots-daemon`](@docroot@/store/types/local-store.md#store-experimental-option-use-roots-daemon) +enabled, it connects to this daemon to discover additional roots that should not be collected. + +The daemon listens on [``](@docroot@/store/types/local-store.md#store-option-state)`/gc-roots-socket/socket` (typically `/nix/var/nix/gc-roots-socket/socket`). + +# Protocol + +The protocol is simple. +For each client-initiated Unix socket connection, the server: + +1. Sends zero or more [store paths](@docroot@/store/store-path.md) as NUL-terminated (`\0`) strings. +2. Closes the connection. + +Example (with `\0` shown as newlines for clarity): +``` +/nix/store/s66mzxpvicwk07gjbjfw9izjfa797vsw-hello-2.12.1 +/nix/store/fvpr7x8l3illdnziggvkhdpf6vikg65w-git-2.44.0 +``` + +# Security + +No information is provided as to which processes are opening which store paths. +While only the main Nix daemon needs to use this daemon, any user able to talk to the main Nix daemon can already obtain the same information with [`nix-store --gc --print-roots`](@docroot@/command-ref/nix-store/gc.md). + +Therefore, restricting this daemon to only accept the Nix daemon as a client is, while recommended for defense-in-depth reasons, strictly speaking not reducing what information can be extracted versus merely restricting this daemon to accept connections from any [allowed user](@docroot@/command-ref/conf-file.md#conf-allowed-users). + +# Systemd socket activation + +`nix store roots-daemon` supports systemd socket-based activation, [just like `nix-daemon`](@docroot@/command-ref/nix-daemon.md#systemd-socket-activation). +)"" diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc index f5ca094c6af7..e6f1b010a893 100644 --- a/src/nix/upgrade-nix.cc +++ b/src/nix/upgrade-nix.cc @@ -1,3 +1,4 @@ +#include "nix/util/os-string.hh" #include "nix/util/processes.hh" #include "nix/cmd/command.hh" #include "nix/main/common-args.hh" @@ -9,10 +10,42 @@ #include "nix/store/names.hh" #include "nix/util/executable-path.hh" #include "nix/store/globals.hh" +#include "nix/util/config-global.hh" #include "self-exe.hh" using namespace nix; +/** + * Check whether a path has a "profiles" component. + */ +static bool hasProfilesComponent(const std::filesystem::path & path) +{ + return std::ranges::contains(path, OS_STR("profiles")); +} + +/** + * Settings related to upgrading Nix itself. + */ +struct UpgradeSettings : Config +{ + /** + * The URL of the file that contains the store paths of the latest Nix release. + */ + Setting storePathUrl{ + this, + "", + "upgrade-nix-store-path-url", + R"( + Deprecated. This option was used to configure how `nix upgrade-nix` operated. + + Using this setting has no effect. It will be removed in a future release of Determinate Nix. + )"}; +}; + +UpgradeSettings upgradeSettings; + +static GlobalConfig::Register rSettings(&upgradeSettings); + struct CmdUpgradeNix : MixDryRun, StoreCommand { /** diff --git a/src/nix/verify.cc b/src/nix/verify.cc index 309d19a1d4ea..6fb10bf3e897 100644 --- a/src/nix/verify.cc +++ b/src/nix/verify.cc @@ -15,7 +15,7 @@ struct CmdVerify : StorePathsCommand { bool noContents = false; bool noTrust = false; - Strings substituterUris; + std::vector substituterUris; size_t sigsNeeded = 0; CmdVerify() @@ -37,7 +37,7 @@ struct CmdVerify : StorePathsCommand .shortName = 's', .description = "Use signatures from the specified store.", .labels = {"store-uri"}, - .handler = {[&](std::string s) { substituterUris.push_back(s); }}, + .handler = {[&](std::string s) { substituterUris.push_back(StoreReference::parse(s)); }}, }); addFlag({ @@ -65,7 +65,7 @@ struct CmdVerify : StorePathsCommand { std::vector> substituters; for (auto & s : substituterUris) - substituters.push_back(openStore(s)); + substituters.push_back(openStore(StoreReference{s})); auto publicKeys = getDefaultPublicKeys(); @@ -123,11 +123,11 @@ struct CmdVerify : StorePathsCommand else { - StringSet sigsSeen; + std::set sigsSeen; size_t actualSigsNeeded = std::max(sigsNeeded, (size_t) 1); size_t validSigs = 0; - auto doSigs = [&](StringSet sigs) { + auto doSigs = [&](std::set sigs) { for (const auto & sig : sigs) { if (!sigsSeen.insert(sig).second) continue; diff --git a/src/nix/why-depends.cc b/src/nix/why-depends.cc index 29da9e953e84..e7224f6cc999 100644 --- a/src/nix/why-depends.cc +++ b/src/nix/why-depends.cc @@ -3,6 +3,7 @@ #include "nix/store/path-references.hh" #include "nix/util/source-accessor.hh" #include "nix/main/shared.hh" +#include "nix/util/fun.hh" #include @@ -165,12 +166,12 @@ struct CmdWhyDepends : SourceExprCommand, MixOperateOnOptions closure (i.e., that have a non-infinite distance to 'dependency'). Print every edge on a path between `package` and `dependency`. */ - std::function printNode; - struct BailOut {}; - printNode = [&](Node & node, const std::string & firstPad, const std::string & tailPad) { + fun printNode = [&](Node & node, + const std::string & firstPad, + const std::string & tailPad) { assert(node.dist != inf); if (precise) { logger->cout( diff --git a/src/kaitai-struct-checks/.version b/src/nswrapper/.version similarity index 100% rename from src/kaitai-struct-checks/.version rename to src/nswrapper/.version diff --git a/src/nswrapper/meson.build b/src/nswrapper/meson.build new file mode 100644 index 000000000000..77b96d677e72 --- /dev/null +++ b/src/nswrapper/meson.build @@ -0,0 +1,46 @@ +project( + 'nix-nswrapper', + 'cpp', + version : files('.version'), + default_options : [ + 'cpp_std=c++23', + # TODO(Qyriad): increase the warning level + 'warning_level=1', + 'errorlogs=true', # Please print logs for tests that fail + 'localstatedir=/nix/var', + ], + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +cxx = meson.get_compiler('cpp') + +subdir('nix-meson-build-support/deps-lists') + +deps_private_maybe_subproject = [ + dependency('nix-util'), +] +deps_public_maybe_subproject = [] +subdir('nix-meson-build-support/subprojects') + +subdir('nix-meson-build-support/export-all-symbols') +subdir('nix-meson-build-support/windows-version') + +subdir('nix-meson-build-support/common') + +sources = files('nswrapper.cc') + +include_dirs = [ include_directories('.') ] + +this_exe = executable( + meson.project_name(), + sources, + dependencies : deps_private_subproject + deps_private + deps_other, + include_directories : include_dirs, + link_args : linker_export_flags, + install : true, + install_dir : get_option('libexecdir'), + cpp_pch : do_pch ? [ 'pch/precompiled-headers.hh' ] : [], +) + +meson.override_find_program('nix-nswrapper', this_exe) diff --git a/src/kaitai-struct-checks/nix-meson-build-support b/src/nswrapper/nix-meson-build-support similarity index 100% rename from src/kaitai-struct-checks/nix-meson-build-support rename to src/nswrapper/nix-meson-build-support diff --git a/src/nswrapper/nswrapper.cc b/src/nswrapper/nswrapper.cc new file mode 100644 index 000000000000..c7a659aa0e39 --- /dev/null +++ b/src/nswrapper/nswrapper.cc @@ -0,0 +1,129 @@ +#include "nix/util/file-descriptor.hh" +#include "nix/util/os-string.hh" +#include "nix/util/processes.hh" +#include "nix/util/util.hh" +#include +#include +#include +#include + +namespace nix { + +static uid_t parseUid(std::string_view value) +{ + auto parsed = string2Int(value); + if (!parsed.has_value()) + throw UsageError("Not a valid integer"); + return parsed.value(); +} + +void mainWrapped(int argc, char ** argv) +{ + if (argc < 4) + throw UsageError("Usage: %1% first_uid num_uids command [args...]", argv[0]); + + auto baseExtra = parseUid(argv[1]); + auto numExtra = parseUid(argv[2]); + + auto currentUid = geteuid(); + auto currentGid = getegid(); + + if (numExtra == 0) + throw UsageError("Must have at least 1 extra UID"); + + if (baseExtra >= baseExtra + numExtra) + throw UsageError("Extra UIDs must not wrap"); + + if (baseExtra <= currentUid && currentUid < baseExtra + numExtra) + throw UsageError("Extra UIDs must not include current UID"); + + if (baseExtra <= currentGid && currentGid < baseExtra + numExtra) + throw UsageError("Extra GIDs must not include current GID"); + + // Unfortunately we can't call `newuidmap` on ourselves. + // We have to be in a new user namespace or `newuidmap` + // will refuse to add new users, but it won't work after + // entering a new namespace. + // Therefore, make a new process to call newuidmap/newgidmap from, + // then call unshare on the parent + + auto parentPid = getpid(); + if (parentPid < 0) + throw SysError("getpid on parent"); + + Pipe toHelper; + toHelper.create(); + + Pid helper = startProcess([&]() { + toHelper.writeSide.close(); + // Wait for the host to unshare, + readLine(toHelper.readSide.get()); + + runProgram( + "newuidmap", + true, + toOsStrings({ + std::to_string(parentPid), + // UID 0 in namespace is euid of parent + "0", + std::to_string(currentUid), + "1", + // numExtra starting from baseExtra are mapped 1:1 to outside namespace + std::to_string(baseExtra), + std::to_string(baseExtra), + std::to_string(numExtra), + })); + + runProgram( + "newgidmap", + true, + toOsStrings({ + std::to_string(parentPid), + // GID 0 in namespace is egid of parent + "0", + std::to_string(currentGid), + "1", + // numExtra starting from baseExtra are mapped 1:1 to outside namespace + std::to_string(baseExtra), + std::to_string(baseExtra), + std::to_string(numExtra), + })); + + exit(0); + }); + + toHelper.readSide.close(); + + // CLONE_NEWUSER (user namespace) so we can remap users + // CLONE_NEWNS (mount namespace) so we can remount devpts + if (unshare(CLONE_NEWUSER | CLONE_NEWNS) < 0) + throw SysError("creating new namespace"); + + writeFull(toHelper.writeSide.get(), "0\n"); + + if (!statusOk(helper.wait())) + throw Error("adding uids/gids to namespace"); + + if (setresuid(0, 0, 0) < 0) + throw SysError("setting uid"); + + if (setresgid(0, 0, 0) < 0) + throw SysError("setting gid"); + + if (setgroups(0, nullptr) < 0) + throw SysError("dropping groups"); + + // We have to mount a new devpts, otherwise we won't be able to chown ptses for build users + if (mount("none", "/dev/pts", "devpts", 0, "mode=0620") < 0) + throw SysError("mounting /dev/pts"); + + if (execvp(argv[3], argv + 3) < 0) + throw SysError("executing wrapped program"); +} + +} // namespace nix + +int main(int argc, char ** argv) +{ + return nix::handleExceptions(argv[0], [&]() { nix::mainWrapped(argc, argv); }); +} diff --git a/src/nswrapper/package.nix b/src/nswrapper/package.nix new file mode 100644 index 000000000000..50e25aa54110 --- /dev/null +++ b/src/nswrapper/package.nix @@ -0,0 +1,43 @@ +{ + lib, + mkMesonExecutable, + + nix-util, + # Configuration Options + + version, +}: + +let + inherit (lib) fileset; +in + +mkMesonExecutable (finalAttrs: { + pname = "nix-nswrapper"; + inherit version; + + workDir = ./.; + fileset = fileset.unions [ + ../../nix-meson-build-support + ./nix-meson-build-support + ../../.version + ./.version + ./meson.build + + (fileset.fileFilter (file: file.hasExt "cc") ./.) + (fileset.fileFilter (file: file.hasExt "hh") ./.) + ]; + + buildInputs = [ + nix-util + ]; + + mesonFlags = [ + ]; + + meta = { + mainProgram = "nix-nswrapper"; + platforms = lib.platforms.linux; + }; + +}) diff --git a/src/nswrapper/pch/precompiled-headers.hh b/src/nswrapper/pch/precompiled-headers.hh new file mode 100644 index 000000000000..9554509854c9 --- /dev/null +++ b/src/nswrapper/pch/precompiled-headers.hh @@ -0,0 +1 @@ +#include "nix/util/util.hh" diff --git a/src/perl/lib/Nix/Store.xs b/src/perl/lib/Nix/Store.xs index 6de26f0d2840..505faf0279cb 100644 --- a/src/perl/lib/Nix/Store.xs +++ b/src/perl/lib/Nix/Store.xs @@ -156,7 +156,7 @@ StoreWrapper::queryPathInfo(char * path, int base32) XPUSHs(sv_2mortal(newRV((SV *) refs))); AV * sigs = newAV(); for (auto & i : info->sigs) - av_push(sigs, newSVpv(i.c_str(), 0)); + av_push(sigs, newSVpv(i.to_string().c_str(), 0)); XPUSHs(sv_2mortal(newRV((SV *) sigs))); } catch (Error & e) { croak("%s", e.what()); @@ -301,7 +301,7 @@ SV * convertHash(char * algo, char * s, int toBase32) SV * signString(char * secretKey_, char * msg) PPCODE: try { - auto sig = SecretKey(secretKey_).signDetached(msg); + auto sig = SecretKey(secretKey_).signDetached(msg).to_string(); XPUSHs(sv_2mortal(newSVpv(sig.c_str(), sig.size()))); } catch (Error & e) { croak("%s", e.what()); @@ -424,4 +424,4 @@ StoreWrapper::addTempRoot(char * storePath) SV * getStoreDir() PPCODE: - XPUSHs(sv_2mortal(newSVpv(settings.nixStore.c_str(), 0))); + XPUSHs(sv_2mortal(newSVpv(resolveStoreConfig(StoreReference{settings.storeUri.get()})->storeDir.c_str(), 0))); diff --git a/src/perl/package.nix b/src/perl/package.nix index b2a1f6975836..21485091fdd9 100644 --- a/src/perl/package.nix +++ b/src/perl/package.nix @@ -47,6 +47,8 @@ perl.pkgs.toPerlModule ( nix-store bzip2 libsodium + perlPackages.DBI + perlPackages.DBDSQLite ]; # `perlPackages.Test2Harness` is marked broken for Darwin @@ -64,8 +66,6 @@ perl.pkgs.toPerlModule ( ''; mesonFlags = [ - (lib.mesonOption "dbi_path" "${perlPackages.DBI}/${perl.libPrefix}") - (lib.mesonOption "dbd_sqlite_path" "${perlPackages.DBDSQLite}/${perl.libPrefix}") (lib.mesonEnable "tests" finalAttrs.finalPackage.doCheck) ]; diff --git a/tests/functional/absolute-path-literals.sh b/tests/functional/absolute-path-literals.sh new file mode 100755 index 000000000000..8cb31ffa4666 --- /dev/null +++ b/tests/functional/absolute-path-literals.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +source common.sh + +# Tests for absolute path literals that require NIX_CONFIG or grepQuietInverse. +# Basic warn/fatal/default behavior tests are in lang/eval-*-abs-path-*.nix + +clearStoreIfPossible + +# Test: Setting via NIX_CONFIG +NIX_CONFIG='lint-absolute-path-literals = warn' nix eval --expr '/tmp/bar' 2>"$TEST_ROOT"/stderr +grepQuiet "absolute path literals are not portable" "$TEST_ROOT/stderr" + +# Test: Command line overrides config +NIX_CONFIG='lint-absolute-path-literals = warn' nix eval --lint-absolute-path-literals ignore --expr '/tmp/bar' 2>"$TEST_ROOT"/stderr +grepQuietInverse "absolute path literal" "$TEST_ROOT/stderr" + +echo "absolute-path-literals test passed!" diff --git a/tests/functional/binary-cache.sh b/tests/functional/binary-cache.sh index 200af1034c30..68263459337e 100755 --- a/tests/functional/binary-cache.sh +++ b/tests/functional/binary-cache.sh @@ -39,6 +39,10 @@ nix log --substituters "file://$cacheDir" "$outPath" | grep FOO nix store copy-log --from "file://$cacheDir" "$(nix-store -qd "$outPath")"^'*' nix log "$outPath" | grep FOO +# Test that plus sign in the URL path is handled correctly. +cacheDir2="$TEST_ROOT/binary+cache" +nix copy --to "file://$cacheDir2" "$outPath" && [[ -d "$cacheDir2" ]] + basicDownloadTests() { # No uploading tests bcause upload with force HTTP doesn't work. diff --git a/tests/functional/build-remote.sh b/tests/functional/build-remote.sh index f396bc72e8f1..6776d912af2e 100644 --- a/tests/functional/build-remote.sh +++ b/tests/functional/build-remote.sh @@ -86,5 +86,5 @@ out="$(nix-build 2>&1 failing.nix \ [[ "$out" =~ .*"note: keeping build directory".* ]] [[ "$out" =~ .*"The failed build directory was kept on the remote builder due to".* ]] -build_dir="$(grep "note: keeping build" <<< "$out" | sed -E "s/^(.*)note: keeping build directory '(.*)'(.*)$/\2/")" +build_dir="$(grep "note: keeping build" <<< "$out" | sed -E "s/^(.*)note: keeping build directory \"(.*)\"(.*)$/\2/")" [[ "foo" = $(<"$build_dir"/bar) ]] diff --git a/tests/functional/build.sh b/tests/functional/build.sh index 6a4c90a43781..fdef4d5f7cee 100755 --- a/tests/functional/build.sh +++ b/tests/functional/build.sh @@ -180,7 +180,10 @@ test "$status" = 1 # Precise number of errors depends on daemon version / goal refactorings (( "$(<<<"$out" grep -cE '^error:')" >= 2 )) -if isDaemonNewer "2.29pre"; then +if isDaemonNewer "2.31"; then + <<<"$out" grepQuiet -E "error: Cannot build '.*-x4\\.drv'" + <<<"$out" grepQuiet -E "Reason: 1 dependency failed." +elif isDaemonNewer "2.29pre"; then <<<"$out" grepQuiet -E "error: Cannot build '.*-x4\\.drv'" <<<"$out" grepQuiet -E "Reason: 1 dependency failed." <<<"$out" grepQuiet -E "Build failed due to failed dependency" @@ -202,3 +205,80 @@ else fi <<<"$out" grepQuiet -vE "hash mismatch in fixed-output derivation '.*-x3\\.drv'" <<<"$out" grepQuiet -vE "hash mismatch in fixed-output derivation '.*-x2\\.drv'" + +# Regression test: cancelled builds should not be reported as failures +# When fast-fail fails, slow and depends-on-slow are cancelled (not failed). +# Only fast-fail should be reported as a failure. +# Uses fifo for synchronization to ensure deterministic behavior. +# Requires -j2 so slow and fast-fail run concurrently (fifo deadlocks if serialized). +if isDaemonNewer "2.34pre" && canUseSandbox; then + fifoDir="$TEST_ROOT/cancelled-builds-fifo" + mkdir -p "$fifoDir" + mkfifo "$fifoDir/fifo" + chmod a+rw "$fifoDir/fifo" + # When using a separate test store, we need sandbox-paths to access + # the system store (where bash/coreutils live). On NixOS, the test + # uses the system store directly, so this isn't needed (and would + # conflict with input paths). + sandboxPathsArg=() + if ! isTestOnNixOS; then + sandboxPathsArg=(--option sandbox-paths "/nix/store") + fi + out="$(nix flake check ./cancelled-builds --impure -L -j2 \ + --option sandbox true \ + "${sandboxPathsArg[@]}" \ + --option sandbox-build-dir /build-tmp \ + --option extra-sandbox-paths "/cancelled-builds-fifo=$fifoDir" \ + 2>&1)" && status=0 || status=$? + rm -rf "$fifoDir" + test "$status" = 1 + # The error should be for fast-fail, not for cancelled goals + <<<"$out" grepQuiet -E "Cannot build.*fast-fail" + # Cancelled goals should NOT appear in error messages (but may appear in "will be built" list) + <<<"$out" grepQuietInverse -E "^error:.*slow" + <<<"$out" grepQuietInverse -E "^error:.*depends-on-slow" + <<<"$out" grepQuietInverse -E "^error:.*depends-on-fail" + # Error messages should not be empty (end with just "failed:") + <<<"$out" grepQuietInverse -E "^error:.*failed: *$" + + # Test that nix build follows the same rules (uses a slightly different code path) + mkdir -p "$fifoDir" + mkfifo "$fifoDir/fifo" + chmod a+rw "$fifoDir/fifo" + sandboxPathsArg=() + if ! isTestOnNixOS; then + sandboxPathsArg=(--option sandbox-paths "/nix/store") + fi + system=$(nix eval --raw --impure --expr builtins.currentSystem) + out="$(nix build --impure -L -j2 \ + --option sandbox true \ + "${sandboxPathsArg[@]}" \ + --option sandbox-build-dir /build-tmp \ + --option extra-sandbox-paths "/cancelled-builds-fifo=$fifoDir" \ + "./cancelled-builds#checks.$system.slow" \ + "./cancelled-builds#checks.$system.depends-on-slow" \ + "./cancelled-builds#checks.$system.fast-fail" \ + "./cancelled-builds#checks.$system.depends-on-fail" \ + 2>&1)" && status=0 || status=$? + rm -rf "$fifoDir" + test "$status" = 1 + # The error should be for fast-fail, not for cancelled goals + <<<"$out" grepQuiet -E "Cannot build.*fast-fail" + # Cancelled goals should NOT appear in error messages + <<<"$out" grepQuietInverse -E "^error:.*slow" + <<<"$out" grepQuietInverse -E "^error:.*depends-on-slow" + <<<"$out" grepQuietInverse -E "^error:.*depends-on-fail" + # Error messages should not be empty (end with just "failed:") + <<<"$out" grepQuietInverse -E "^error:.*failed: *$" +fi + +# https://github.com/NixOS/nix/issues/14883 +# When max-jobs=0 and no remote builders, the error should say +# "local builds are disabled" instead of the misleading +# "required system or feature not available". +if isDaemonNewer "2.34pre"; then + expectStderr 1 nix build --impure --max-jobs 0 --expr \ + 'derivation { name = "test-maxjobs"; builder = "/bin/sh"; args = ["-c" "exit 0"]; system = builtins.currentSystem; }' \ + --no-link \ + | grepQuiet "local builds are disabled" +fi diff --git a/tests/functional/ca/duplicate-realisation-in-closure.sh b/tests/functional/ca/duplicate-realisation-in-closure.sh index 4a5e8c042b76..032fb6164d10 100644 --- a/tests/functional/ca/duplicate-realisation-in-closure.sh +++ b/tests/functional/ca/duplicate-realisation-in-closure.sh @@ -25,4 +25,9 @@ nix build -f nondeterministic.nix dep2 --no-link # If everything goes right, we should rebuild dep2 rather than fetch it from # the cache (because that would mean duplicating `current-time` in the closure), # and have `dep1 == dep2`. + +# FIXME: Force the use of small-step resolutions only to fix this in a +# better way (#11896, #11928). +skipTest "temporarily broken because dependent realisations are removed" + nix build --substituters "$REMOTE_STORE" -f nondeterministic.nix toplevel --no-require-sigs --no-link diff --git a/tests/functional/ca/issue-13247.sh b/tests/functional/ca/issue-13247.sh index 70591951329b..29bc2f912014 100755 --- a/tests/functional/ca/issue-13247.sh +++ b/tests/functional/ca/issue-13247.sh @@ -42,7 +42,7 @@ buildViaSubstitute () { nix build -f issue-13247.nix "$1" --no-link --max-jobs 0 --substituters "$cache" --no-require-sigs --offline --substitute } -# Substitue just the first output +# Substitute just the first output buildViaSubstitute use-a-more-outputs^first # Should only fetch the output we asked for @@ -52,10 +52,10 @@ buildViaSubstitute use-a-more-outputs^first delete -# Failure with 2.28 encountered in CI -requireDaemonNewerThan "2.29" +# Failure with 2.33 encountered in CI +requireDaemonNewerThan "2.34pre" -# Substitue just the first output +# Substitute just the first output # # This derivation is the same after normalization, so we should get # early cut-off, and thus a chance to download just the output we want diff --git a/tests/functional/ca/substitute.sh b/tests/functional/ca/substitute.sh index 9728470f0b83..2f6ebcef5c11 100644 --- a/tests/functional/ca/substitute.sh +++ b/tests/functional/ca/substitute.sh @@ -22,7 +22,10 @@ nix copy --to "$REMOTE_STORE" --file ./content-addressed.nix # Restart the build on an empty store, ensuring that we don't build clearStore -buildDrvs --substitute --substituters "$REMOTE_STORE" --no-require-sigs -j0 transitivelyDependentCA +# FIXME: `dependentCA` should not need to be explicitly mentioned in +# this. Force the use of small-step resolutions only to allow not +# mentioning it explicitly again. (#11896, #11928). +buildDrvs --substitute --substituters "$REMOTE_STORE" --no-require-sigs -j0 transitivelyDependentCA dependentCA # Check that the thing we’ve just substituted has its realisation stored nix realisation info --file ./content-addressed.nix transitivelyDependentCA # Check that its dependencies have it too diff --git a/tests/functional/cancelled-builds/flake.nix b/tests/functional/cancelled-builds/flake.nix new file mode 100644 index 000000000000..0b8bf1ca5d86 --- /dev/null +++ b/tests/functional/cancelled-builds/flake.nix @@ -0,0 +1,64 @@ +# Regression test for cancelled builds not being reported as failures. +# +# Scenario: When a build fails while other builds are running, those other +# builds (and their dependents) get cancelled. Previously, cancelled builds +# were incorrectly reported as failures with empty error messages. +# +# Uses a fifo for synchronization: fast-fail waits for slow to start before +# failing, ensuring slow is actually running when it gets cancelled. +# +# See: tests/functional/build.sh (search for "cancelled-builds") +{ + outputs = + { self }: + let + config = import "${builtins.getEnv "_NIX_TEST_BUILD_DIR"}/config.nix"; + in + with config; + { + checks.${system} = { + # A derivation that signals it started via fifo, then waits + slow = mkDerivation { + name = "slow"; + buildCommand = '' + echo "slow: started, signaling via fifo" + echo started > /cancelled-builds-fifo/fifo + echo "slow: waiting..." + sleep 10 + touch $out + ''; + }; + + # Depends on slow - will be cancelled when fast-fail fails + depends-on-slow = mkDerivation { + name = "depends-on-slow"; + slow = self.checks.${system}.slow; + buildCommand = '' + echo "depends-on-slow: slow finished at $slow" + touch $out + ''; + }; + + # Waits for slow to start via fifo, then fails + fast-fail = mkDerivation { + name = "fast-fail"; + buildCommand = '' + echo "fast-fail: waiting for slow to start..." + read line < /cancelled-builds-fifo/fifo + echo "fast-fail: slow started, now failing" >&2 + exit 1 + ''; + }; + + # Depends on fast-fail - will fail with DependencyFailed + depends-on-fail = mkDerivation { + name = "depends-on-fail"; + fast-fail = self.checks.${system}.fast-fail; + buildCommand = '' + echo "depends-on-fail: fast-fail finished (should never get here)" + touch $out + ''; + }; + }; + }; +} diff --git a/tests/functional/chroot-store.sh b/tests/functional/chroot-store.sh index 7300f04ba75f..cbb80c8710ad 100755 --- a/tests/functional/chroot-store.sh +++ b/tests/functional/chroot-store.sh @@ -42,6 +42,24 @@ PATH2=$(nix path-info --store "$TEST_ROOT/x" "$CORRECT_PATH") PATH3=$(nix path-info --store "local?root=$TEST_ROOT/x" "$CORRECT_PATH") [ "$CORRECT_PATH" == "$PATH3" ] +# Test chroot store path with + symbols in it to exercise pct-encoding issues. +cp -r "$TEST_ROOT/x" "$TEST_ROOT/x+chroot" + +PATH4=$(nix path-info --store "local://$TEST_ROOT/x+chroot" "$CORRECT_PATH") +[ "$CORRECT_PATH" == "$PATH4" ] + +PATH5=$(nix path-info --store "$TEST_ROOT/x+chroot" "$CORRECT_PATH") +[ "$CORRECT_PATH" == "$PATH5" ] + +# Params are pct-encoded. +PATH6=$(nix path-info --store "local?root=$TEST_ROOT/x%2Bchroot" "$CORRECT_PATH") +[ "$CORRECT_PATH" == "$PATH6" ] + +PATH7=$(nix path-info --store "local://$TEST_ROOT/x%2Bchroot" "$CORRECT_PATH") +[ "$CORRECT_PATH" == "$PATH7" ] +# Path gets decoded. +[[ ! -d "$TEST_ROOT/x%2Bchroot" ]] + # Ensure store info trusted works with local store nix --store "$TEST_ROOT/x" store info --json | jq -e '.trusted' diff --git a/tests/functional/cli-characterisation.sh b/tests/functional/cli-characterisation.sh new file mode 100644 index 000000000000..a795b14e25e4 --- /dev/null +++ b/tests/functional/cli-characterisation.sh @@ -0,0 +1,104 @@ +#!/usr/bin/env bash + +# Characterization tests for nix-env, nix-build, and nix-instantiate. +# Captures error messages, output formats, and edge-case behavior. +# +# Expected output files in: cli-characterisation/ +# Regenerate with: _NIX_TEST_ACCEPT=1 meson test -C build cli-characterisation + +source common.sh + +source characterisation/framework.sh + +set +x + +badDiff=0 +badExitCode=0 + +# Normalize store paths, hashes, source paths, and system +normalize() { + sed -i \ + -e "s|${NIX_STORE_DIR:-/nix/store}/[a-z0-9]\{32\}|/test-root/store/xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx|g" \ + -e "s|$TEST_ROOT|/test-root|g" \ + -e "s|$(pwd)|/pwd|g" \ + -e "s|$system|SYSTEM|g" \ + -e "s|/nix/store/[a-z0-9]\{32\}|/nix/store/xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx|g" \ + -e "s|/test-root/store/[a-z0-9]\{32\}|/test-root/store/xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx|g" \ + -e "s|'[a-z0-9]\{32\}-|'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx-|g" \ + "$@" +} + +diffAndAccept() { + local testName="$1" + local got="cli-characterisation/$testName.$2" + local expected="cli-characterisation/$testName.$3" + diffAndAcceptInner "$testName" "$got" "$expected" +} + +clearProfiles + +# Each test case has a descriptor: cli-characterisation/.cmd +# containing: +# +# stdout -> .out, stderr -> .err +# Both are compared against .out.exp / .err.exp + +for cmdFile in cli-characterisation/*.cmd; do + testName=$(basename "$cmdFile" .cmd) + echo "testing $testName" + + # read returns 1 on EOF without trailing newline; handle it + read -r expectedExit cmd < "$cmdFile" || [[ -n "$expectedExit" ]] + + if + # shellcheck disable=SC2086 # word splitting of cmd is intended + expect "$expectedExit" $cmd \ + 1> "cli-characterisation/$testName.out" \ + 2> "cli-characterisation/$testName.err" + then + normalize "cli-characterisation/$testName.out" "cli-characterisation/$testName.err" + diffAndAccept "$testName" out out.exp + diffAndAccept "$testName" err err.exp + else + echo "FAIL: $testName exited with wrong code" + badExitCode=1 + fi +done + +# --- Multi-step tests that need a real .drv path in the store --- + +drvPath=$(nix-instantiate ./cli-characterisation/simple.nix) + +# Constructor: "building more than one derivation output is not supported" +testName=nix-build-multi-output +echo "testing $testName" +if + expect 1 nix-build "$drvPath!out,bin" \ + 1> "cli-characterisation/$testName.out" \ + 2> "cli-characterisation/$testName.err" +then + normalize "cli-characterisation/$testName.out" "cli-characterisation/$testName.err" + diffAndAccept "$testName" out out.exp + diffAndAccept "$testName" err err.exp +else + echo "FAIL: $testName exited with wrong code" + badExitCode=1 +fi + +# Constructor: "derivation does not have output" +testName=nix-build-bad-output +echo "testing $testName" +if + expect 1 nix-build "$drvPath!nonexistent" \ + 1> "cli-characterisation/$testName.out" \ + 2> "cli-characterisation/$testName.err" +then + normalize "cli-characterisation/$testName.out" "cli-characterisation/$testName.err" + diffAndAccept "$testName" out out.exp + diffAndAccept "$testName" err err.exp +else + echo "FAIL: $testName exited with wrong code" + badExitCode=1 +fi + +characterisationTestExit diff --git a/tests/functional/cli-characterisation/assert-fail.nix b/tests/functional/cli-characterisation/assert-fail.nix new file mode 100644 index 000000000000..03a161dedfce --- /dev/null +++ b/tests/functional/cli-characterisation/assert-fail.nix @@ -0,0 +1,7 @@ +with import ../config.nix; +mkDerivation { + name = + assert false; + "assert-fail-1.0"; + buildCommand = "mkdir -p $out"; +} diff --git a/tests/functional/cli-characterisation/bad-drvpath.nix b/tests/functional/cli-characterisation/bad-drvpath.nix new file mode 100644 index 000000000000..4f77127342d3 --- /dev/null +++ b/tests/functional/cli-characterisation/bad-drvpath.nix @@ -0,0 +1,6 @@ +{ + type = "derivation"; + name = "bad-drvpath-1.0"; + outPath = builtins.toFile "out" ""; + drvPath = builtins.toFile "not-a-drv" ""; +} diff --git a/tests/functional/cli-characterisation/bad-output-specified-type.nix b/tests/functional/cli-characterisation/bad-output-specified-type.nix new file mode 100644 index 000000000000..7958ea09a2d5 --- /dev/null +++ b/tests/functional/cli-characterisation/bad-output-specified-type.nix @@ -0,0 +1,10 @@ +{ + type = "derivation"; + name = "bad-output-specified-type-1.0"; + outPath = builtins.toFile "out" ""; + outputs = [ "out" ]; + out = { + outPath = builtins.toFile "out" ""; + }; + outputSpecified = "yes"; +} diff --git a/tests/functional/cli-characterisation/bad-output-specified.nix b/tests/functional/cli-characterisation/bad-output-specified.nix new file mode 100644 index 000000000000..174d1aee0d6b --- /dev/null +++ b/tests/functional/cli-characterisation/bad-output-specified.nix @@ -0,0 +1,8 @@ +{ + type = "derivation"; + name = "bad-output-specified-1.0"; + outPath = builtins.toFile "out" ""; + outputs = [ "out" ]; + outputSpecified = true; + outputName = "nonexistent"; +} diff --git a/tests/functional/cli-characterisation/bad-outputs-to-install-elem.nix b/tests/functional/cli-characterisation/bad-outputs-to-install-elem.nix new file mode 100644 index 000000000000..906326df8aba --- /dev/null +++ b/tests/functional/cli-characterisation/bad-outputs-to-install-elem.nix @@ -0,0 +1,8 @@ +with import ../config.nix; +mkDerivation { + name = "bad-outputs-to-install-elem-1.0"; + buildCommand = "mkdir -p $out"; + meta = { + outputsToInstall = [ 42 ]; # elements should be strings + }; +} diff --git a/tests/functional/cli-characterisation/bad-outputs-to-install-nosuch.nix b/tests/functional/cli-characterisation/bad-outputs-to-install-nosuch.nix new file mode 100644 index 000000000000..0a39e149c8d8 --- /dev/null +++ b/tests/functional/cli-characterisation/bad-outputs-to-install-nosuch.nix @@ -0,0 +1,8 @@ +with import ../config.nix; +mkDerivation { + name = "bad-outputs-to-install-nosuch-1.0"; + buildCommand = "mkdir -p $out"; + meta = { + outputsToInstall = [ "nonexistent" ]; + }; +} diff --git a/tests/functional/cli-characterisation/bad-outputs-to-install-type.nix b/tests/functional/cli-characterisation/bad-outputs-to-install-type.nix new file mode 100644 index 000000000000..4f7e22b71069 --- /dev/null +++ b/tests/functional/cli-characterisation/bad-outputs-to-install-type.nix @@ -0,0 +1,8 @@ +with import ../config.nix; +mkDerivation { + name = "bad-outputs-to-install-type-1.0"; + buildCommand = "mkdir -p $out"; + meta = { + outputsToInstall = "out"; # should be a list + }; +} diff --git a/tests/functional/cli-characterisation/context-built.nix b/tests/functional/cli-characterisation/context-built.nix new file mode 100644 index 000000000000..c44008307f76 --- /dev/null +++ b/tests/functional/cli-characterisation/context-built.nix @@ -0,0 +1,13 @@ +# Built context: derivation output reference +with import ../config.nix; +let + drv = mkDerivation { + name = "helper-1.0"; + buildCommand = "mkdir -p $out"; + }; +in +{ + type = "derivation"; + name = "${drv}"; + outPath = "/nix/store/fake"; +} diff --git a/tests/functional/cli-characterisation/context-drvdeep.nix b/tests/functional/cli-characterisation/context-drvdeep.nix new file mode 100644 index 000000000000..0c73d091a764 --- /dev/null +++ b/tests/functional/cli-characterisation/context-drvdeep.nix @@ -0,0 +1,13 @@ +# DrvDeep context: derivation path with full closure dependency +with import ../config.nix; +let + drv = mkDerivation { + name = "helper-1.0"; + buildCommand = "mkdir -p $out"; + }; +in +{ + type = "derivation"; + name = builtins.addDrvOutputDependencies drv.drvPath; + outPath = "/nix/store/fake"; +} diff --git a/tests/functional/cli-characterisation/context-opaque-drv.nix b/tests/functional/cli-characterisation/context-opaque-drv.nix new file mode 100644 index 000000000000..5ff5f73d00cf --- /dev/null +++ b/tests/functional/cli-characterisation/context-opaque-drv.nix @@ -0,0 +1,13 @@ +# Opaque context with a .drv store path (via unsafeDiscardOutputDependency roundtrip) +with import ../config.nix; +let + drv = mkDerivation { + name = "helper-1.0"; + buildCommand = "mkdir -p $out"; + }; +in +{ + type = "derivation"; + name = builtins.unsafeDiscardOutputDependency (builtins.addDrvOutputDependencies drv.drvPath); + outPath = "/nix/store/fake"; +} diff --git a/tests/functional/cli-characterisation/context-opaque.nix b/tests/functional/cli-characterisation/context-opaque.nix new file mode 100644 index 000000000000..109992dbbdf1 --- /dev/null +++ b/tests/functional/cli-characterisation/context-opaque.nix @@ -0,0 +1,6 @@ +# Opaque context: plain store path reference from file interpolation +{ + type = "derivation"; + name = "${builtins.toFile "x" ""}"; + outPath = "/nix/store/fake"; +} diff --git a/tests/functional/cli-characterisation/deep-meta.nix b/tests/functional/cli-characterisation/deep-meta.nix new file mode 100644 index 000000000000..06cf82f43f8f --- /dev/null +++ b/tests/functional/cli-characterisation/deep-meta.nix @@ -0,0 +1,15 @@ +with import ../config.nix; +mkDerivation { + name = "deep-meta-1.0"; + buildCommand = "mkdir -p $out"; + meta = { + description = "Has deeply nested meta"; + nested = { + a = { + b = { + c = "deep value"; + }; + }; + }; + }; +} diff --git a/tests/functional/cli-characterisation/ghost-outpath.nix b/tests/functional/cli-characterisation/ghost-outpath.nix new file mode 100644 index 000000000000..f1fca8117c35 --- /dev/null +++ b/tests/functional/cli-characterisation/ghost-outpath.nix @@ -0,0 +1,15 @@ +{ + type = "derivation"; + name = "ghost-outpath-1.0"; + outPath = builtins.toFile "out" ""; + out = { + outPath = builtins.toFile "out" ""; + }; + ghost = { + # no outPath + }; + outputs = [ + "out" + "ghost" + ]; +} diff --git a/tests/functional/cli-characterisation/ghost-output.nix b/tests/functional/cli-characterisation/ghost-output.nix new file mode 100644 index 000000000000..0fac35abd4d3 --- /dev/null +++ b/tests/functional/cli-characterisation/ghost-output.nix @@ -0,0 +1,13 @@ +{ + type = "derivation"; + name = "ghost-output-1.0"; + outPath = builtins.toFile "out" ""; + out = { + outPath = builtins.toFile "out" ""; + }; + outputs = [ + "out" + "ghost" + ]; + # no "ghost" attr -> silently skipped +} diff --git a/tests/functional/cli-characterisation/infinite-meta.nix b/tests/functional/cli-characterisation/infinite-meta.nix new file mode 100644 index 000000000000..22d559647355 --- /dev/null +++ b/tests/functional/cli-characterisation/infinite-meta.nix @@ -0,0 +1,12 @@ +with import ../config.nix; +let + x = { inherit x; }; +in +mkDerivation { + name = "infinite-meta-1.0"; + buildCommand = "mkdir -p $out"; + meta = { + description = "Has infinite recursion in meta"; + bad = x; + }; +} diff --git a/tests/functional/cli-characterisation/meta-types.nix b/tests/functional/cli-characterisation/meta-types.nix new file mode 100644 index 000000000000..60094ce3b868 --- /dev/null +++ b/tests/functional/cli-characterisation/meta-types.nix @@ -0,0 +1,21 @@ +with import ../config.nix; +mkDerivation { + name = "meta-types-1.0"; + buildCommand = "mkdir -p $out"; + meta = { + # Proper types + anInt = 42; + aBool = true; + aFloat = 3.14; + aString = "hello"; + aList = [ + 1 + 2 + 3 + ]; + # String-encoded values (backwards compatibility) + stringInt = "123"; + stringBool = "true"; + stringFloat = "2.72"; + }; +} diff --git a/tests/functional/cli-characterisation/meta-with-bad-list.nix b/tests/functional/cli-characterisation/meta-with-bad-list.nix new file mode 100644 index 000000000000..22039852043a --- /dev/null +++ b/tests/functional/cli-characterisation/meta-with-bad-list.nix @@ -0,0 +1,14 @@ +with import ../config.nix; +let + normal = mkDerivation { + name = "normal-1.0"; + buildCommand = "mkdir -p $out"; + }; +in +mkDerivation { + name = "meta-with-bad-list-1.0"; + buildCommand = "mkdir -p $out"; + meta = { + bad = [ normal ]; + }; +} diff --git a/tests/functional/cli-characterisation/meta-with-drv.nix b/tests/functional/cli-characterisation/meta-with-drv.nix new file mode 100644 index 000000000000..f22a7055ef22 --- /dev/null +++ b/tests/functional/cli-characterisation/meta-with-drv.nix @@ -0,0 +1,14 @@ +with import ../config.nix; +let + normal = mkDerivation { + name = "normal-1.0"; + buildCommand = "mkdir -p $out"; + }; +in +mkDerivation { + name = "meta-with-drv-1.0"; + buildCommand = "mkdir -p $out"; + meta = { + someDrv = normal; + }; +} diff --git a/tests/functional/cli-characterisation/meta-with-function.nix b/tests/functional/cli-characterisation/meta-with-function.nix new file mode 100644 index 000000000000..05fee50041a6 --- /dev/null +++ b/tests/functional/cli-characterisation/meta-with-function.nix @@ -0,0 +1,8 @@ +with import ../config.nix; +mkDerivation { + name = "meta-with-function-1.0"; + buildCommand = "mkdir -p $out"; + meta = { + bad = x: x; + }; +} diff --git a/tests/functional/cli-characterisation/meta-with-outpath.nix b/tests/functional/cli-characterisation/meta-with-outpath.nix new file mode 100644 index 000000000000..d6dc3e3abb85 --- /dev/null +++ b/tests/functional/cli-characterisation/meta-with-outpath.nix @@ -0,0 +1,11 @@ +with import ../config.nix; +mkDerivation { + name = "meta-with-outpath-1.0"; + buildCommand = "mkdir -p $out"; + meta = { + description = "Has outPath in meta"; + bad = { + outPath = "/nix/store/fake"; + }; + }; +} diff --git a/tests/functional/cli-characterisation/name-with-context.nix b/tests/functional/cli-characterisation/name-with-context.nix new file mode 100644 index 000000000000..aac4352519d1 --- /dev/null +++ b/tests/functional/cli-characterisation/name-with-context.nix @@ -0,0 +1,12 @@ +with import ../config.nix; +let + normal = mkDerivation { + name = "normal-1.0"; + buildCommand = "mkdir -p $out"; + }; +in +{ + type = "derivation"; + name = "${normal}"; + outPath = "/nix/store/fake"; +} diff --git a/tests/functional/cli-characterisation/nix-build-bad-output.err.exp b/tests/functional/cli-characterisation/nix-build-bad-output.err.exp new file mode 100644 index 000000000000..02e2cec88b63 --- /dev/null +++ b/tests/functional/cli-characterisation/nix-build-bad-output.err.exp @@ -0,0 +1 @@ +error: derivation '/test-root/store/xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx-simple-1.0.drv' does not have output 'nonexistent' diff --git a/tests/functional/cli-characterisation/nix-build-multi-output.err.exp b/tests/functional/cli-characterisation/nix-build-multi-output.err.exp new file mode 100644 index 000000000000..2e3669ce6687 --- /dev/null +++ b/tests/functional/cli-characterisation/nix-build-multi-output.err.exp @@ -0,0 +1 @@ +error: building more than one derivation output is not supported, in '/test-root/store/xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx-simple-1.0.drv!out,bin' diff --git a/tests/functional/cli-characterisation/nix-env-install-bad-oti-elem.cmd b/tests/functional/cli-characterisation/nix-env-install-bad-oti-elem.cmd new file mode 100644 index 000000000000..18e4d5aeea29 --- /dev/null +++ b/tests/functional/cli-characterisation/nix-env-install-bad-oti-elem.cmd @@ -0,0 +1 @@ +1 nix-env -f ./cli-characterisation/bad-outputs-to-install-elem.nix -i bad-outputs-to-install-elem \ No newline at end of file diff --git a/tests/functional/cli-characterisation/nix-env-install-bad-oti-elem.err.exp b/tests/functional/cli-characterisation/nix-env-install-bad-oti-elem.err.exp new file mode 100644 index 000000000000..5759634cac7d --- /dev/null +++ b/tests/functional/cli-characterisation/nix-env-install-bad-oti-elem.err.exp @@ -0,0 +1,5 @@ +installing 'bad-outputs-to-install-elem-1.0' +this derivation will be built: + /test-root/store/xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx-bad-outputs-to-install-elem-1.0.drv +building '/test-root/store/xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx-bad-outputs-to-install-elem-1.0.drv'... +error: this derivation has bad 'meta.outputsToInstall' diff --git a/tests/functional/cli-characterisation/nix-env-install-bad-oti-nosuch.cmd b/tests/functional/cli-characterisation/nix-env-install-bad-oti-nosuch.cmd new file mode 100644 index 000000000000..8a3960b94a91 --- /dev/null +++ b/tests/functional/cli-characterisation/nix-env-install-bad-oti-nosuch.cmd @@ -0,0 +1 @@ +1 nix-env -f ./cli-characterisation/bad-outputs-to-install-nosuch.nix -i bad-outputs-to-install-nosuch \ No newline at end of file diff --git a/tests/functional/cli-characterisation/nix-env-install-bad-oti-nosuch.err.exp b/tests/functional/cli-characterisation/nix-env-install-bad-oti-nosuch.err.exp new file mode 100644 index 000000000000..eb3821ddcf0b --- /dev/null +++ b/tests/functional/cli-characterisation/nix-env-install-bad-oti-nosuch.err.exp @@ -0,0 +1,5 @@ +installing 'bad-outputs-to-install-nosuch-1.0' +this derivation will be built: + /test-root/store/xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx-bad-outputs-to-install-nosuch-1.0.drv +building '/test-root/store/xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx-bad-outputs-to-install-nosuch-1.0.drv'... +error: this derivation has bad 'meta.outputsToInstall' diff --git a/tests/functional/cli-characterisation/nix-env-install-bad-oti-type.cmd b/tests/functional/cli-characterisation/nix-env-install-bad-oti-type.cmd new file mode 100644 index 000000000000..8fa78f25a213 --- /dev/null +++ b/tests/functional/cli-characterisation/nix-env-install-bad-oti-type.cmd @@ -0,0 +1 @@ +1 nix-env -f ./cli-characterisation/bad-outputs-to-install-type.nix -i bad-outputs-to-install-type \ No newline at end of file diff --git a/tests/functional/cli-characterisation/nix-env-install-bad-oti-type.err.exp b/tests/functional/cli-characterisation/nix-env-install-bad-oti-type.err.exp new file mode 100644 index 000000000000..9ca78807f6ee --- /dev/null +++ b/tests/functional/cli-characterisation/nix-env-install-bad-oti-type.err.exp @@ -0,0 +1,5 @@ +installing 'bad-outputs-to-install-type-1.0' +this derivation will be built: + /test-root/store/xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx-bad-outputs-to-install-type-1.0.drv +building '/test-root/store/xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx-bad-outputs-to-install-type-1.0.drv'... +error: this derivation has bad 'meta.outputsToInstall' diff --git a/tests/functional/cli-characterisation/nix-env-install-bad-output-specified-type.cmd b/tests/functional/cli-characterisation/nix-env-install-bad-output-specified-type.cmd new file mode 100644 index 000000000000..713006d754a7 --- /dev/null +++ b/tests/functional/cli-characterisation/nix-env-install-bad-output-specified-type.cmd @@ -0,0 +1 @@ +1 nix-env -f ./cli-characterisation/bad-output-specified-type.nix -i bad-output-specified-type \ No newline at end of file diff --git a/tests/functional/cli-characterisation/nix-env-install-bad-output-specified-type.err.exp b/tests/functional/cli-characterisation/nix-env-install-bad-output-specified-type.err.exp new file mode 100644 index 000000000000..b65a1f3da494 --- /dev/null +++ b/tests/functional/cli-characterisation/nix-env-install-bad-output-specified-type.err.exp @@ -0,0 +1,15 @@ +installing 'bad-output-specified-type-1.0' +error: + … while evaluating the 'outputSpecified' attribute of a derivation + at /pwd/cli-characterisation/bad-output-specified-type.nix:9:3: + 8| }; + 9| outputSpecified = "yes"; + | ^ + 10| } + + error: expected a Boolean but found a string: "yes" + at /pwd/cli-characterisation/bad-output-specified-type.nix:9:3: + 8| }; + 9| outputSpecified = "yes"; + | ^ + 10| } diff --git a/tests/functional/cli-characterisation/nix-env-install-bad-output-specified.cmd b/tests/functional/cli-characterisation/nix-env-install-bad-output-specified.cmd new file mode 100644 index 000000000000..181b53d4919e --- /dev/null +++ b/tests/functional/cli-characterisation/nix-env-install-bad-output-specified.cmd @@ -0,0 +1 @@ +1 nix-env -f ./cli-characterisation/bad-output-specified.nix -i bad-output-specified \ No newline at end of file diff --git a/tests/functional/cli-characterisation/nix-env-install-bad-output-specified.err.exp b/tests/functional/cli-characterisation/nix-env-install-bad-output-specified.err.exp new file mode 100644 index 000000000000..0e3e4b8fc4d5 --- /dev/null +++ b/tests/functional/cli-characterisation/nix-env-install-bad-output-specified.err.exp @@ -0,0 +1,2 @@ +installing 'bad-output-specified-1.0' +error: derivation does not have output 'nonexistent' diff --git a/tests/functional/cli-characterisation/nix-env-qa-attrpath.cmd b/tests/functional/cli-characterisation/nix-env-qa-attrpath.cmd new file mode 100644 index 000000000000..d35fa1306cc2 --- /dev/null +++ b/tests/functional/cli-characterisation/nix-env-qa-attrpath.cmd @@ -0,0 +1 @@ +0 nix-env -f ./cli-characterisation/sample-package-set.nix -qa -P --description -a \ No newline at end of file diff --git a/tests/functional/cli-characterisation/nix-env-qa-attrpath.out.exp b/tests/functional/cli-characterisation/nix-env-qa-attrpath.out.exp new file mode 100644 index 000000000000..04099657256f --- /dev/null +++ b/tests/functional/cli-characterisation/nix-env-qa-attrpath.out.exp @@ -0,0 +1,9 @@ +bad-drvpath bad-drvpath-1.0 +bad-output-specified bad-output-specified-1.0 +deep-meta deep-meta-1.0 Has deeply nested meta +ghost-outpath ghost-outpath-1.0 +ghost-output ghost-output-1.0 +infinite-meta infinite-meta-1.0 Has infinite recursion in meta +no-system no-system-1.0 +normal normal-1.0 A normal package +simple simple-1.0 diff --git a/tests/functional/cli-characterisation/nix-env-qa-bad-drvpath.cmd b/tests/functional/cli-characterisation/nix-env-qa-bad-drvpath.cmd new file mode 100644 index 000000000000..482b01403486 --- /dev/null +++ b/tests/functional/cli-characterisation/nix-env-qa-bad-drvpath.cmd @@ -0,0 +1 @@ +1 nix-env -f ./cli-characterisation/bad-drvpath.nix -qa --json --drv-path \ No newline at end of file diff --git a/tests/functional/cli-characterisation/nix-env-qa-bad-drvpath.err.exp b/tests/functional/cli-characterisation/nix-env-qa-bad-drvpath.err.exp new file mode 100644 index 000000000000..e05ff77298e7 --- /dev/null +++ b/tests/functional/cli-characterisation/nix-env-qa-bad-drvpath.err.exp @@ -0,0 +1,11 @@ +error: + … while querying the derivation named 'bad-drvpath-1.0' + + … while evaluating the 'drvPath' attribute of a derivation + at /pwd/cli-characterisation/bad-drvpath.nix:5:3: + 4| outPath = builtins.toFile "out" ""; + 5| drvPath = builtins.toFile "not-a-drv" ""; + | ^ + 6| } + + error: store path 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx-not-a-drv' is not a valid derivation path diff --git a/tests/functional/cli-characterisation/nix-env-qa-context-built.cmd b/tests/functional/cli-characterisation/nix-env-qa-context-built.cmd new file mode 100644 index 000000000000..372123378add --- /dev/null +++ b/tests/functional/cli-characterisation/nix-env-qa-context-built.cmd @@ -0,0 +1 @@ +1 nix-env -f ./cli-characterisation/context-built.nix -qa --json \ No newline at end of file diff --git a/tests/functional/cli-characterisation/nix-env-qa-context-built.err.exp b/tests/functional/cli-characterisation/nix-env-qa-context-built.err.exp new file mode 100644 index 000000000000..a9bcbadb3adf --- /dev/null +++ b/tests/functional/cli-characterisation/nix-env-qa-context-built.err.exp @@ -0,0 +1,4 @@ +error: + … while evaluating the 'name' attribute of a derivation + + error: the string '/test-root/store/xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx-helper-1.0' is not allowed to refer to a store path (such as '/test-root/store/xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx-helper-1.0.drv^out') diff --git a/tests/functional/cli-characterisation/nix-env-qa-context-drvdeep.cmd b/tests/functional/cli-characterisation/nix-env-qa-context-drvdeep.cmd new file mode 100644 index 000000000000..e09be3aa4e43 --- /dev/null +++ b/tests/functional/cli-characterisation/nix-env-qa-context-drvdeep.cmd @@ -0,0 +1 @@ +1 nix-env -f ./cli-characterisation/context-drvdeep.nix -qa --json \ No newline at end of file diff --git a/tests/functional/cli-characterisation/nix-env-qa-context-drvdeep.err.exp b/tests/functional/cli-characterisation/nix-env-qa-context-drvdeep.err.exp new file mode 100644 index 000000000000..a74a21cf312e --- /dev/null +++ b/tests/functional/cli-characterisation/nix-env-qa-context-drvdeep.err.exp @@ -0,0 +1,4 @@ +error: + … while evaluating the 'name' attribute of a derivation + + error: the string '/test-root/store/xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx-helper-1.0.drv' is not allowed to refer to a store path (such as '/test-root/store/xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx-helper-1.0.drv (deep)') diff --git a/tests/functional/cli-characterisation/nix-env-qa-context-opaque-drv.cmd b/tests/functional/cli-characterisation/nix-env-qa-context-opaque-drv.cmd new file mode 100644 index 000000000000..14ccfd7f119f --- /dev/null +++ b/tests/functional/cli-characterisation/nix-env-qa-context-opaque-drv.cmd @@ -0,0 +1 @@ +1 nix-env -f ./cli-characterisation/context-opaque-drv.nix -qa --json \ No newline at end of file diff --git a/tests/functional/cli-characterisation/nix-env-qa-context-opaque-drv.err.exp b/tests/functional/cli-characterisation/nix-env-qa-context-opaque-drv.err.exp new file mode 100644 index 000000000000..4e8b54094305 --- /dev/null +++ b/tests/functional/cli-characterisation/nix-env-qa-context-opaque-drv.err.exp @@ -0,0 +1,4 @@ +error: + … while evaluating the 'name' attribute of a derivation + + error: the string '/test-root/store/xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx-helper-1.0.drv' is not allowed to refer to a store path (such as '/test-root/store/xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx-helper-1.0.drv') diff --git a/tests/functional/cli-characterisation/nix-env-qa-context-opaque.cmd b/tests/functional/cli-characterisation/nix-env-qa-context-opaque.cmd new file mode 100644 index 000000000000..c3de8b91da8d --- /dev/null +++ b/tests/functional/cli-characterisation/nix-env-qa-context-opaque.cmd @@ -0,0 +1 @@ +1 nix-env -f ./cli-characterisation/context-opaque.nix -qa --json \ No newline at end of file diff --git a/tests/functional/cli-characterisation/nix-env-qa-context-opaque.err.exp b/tests/functional/cli-characterisation/nix-env-qa-context-opaque.err.exp new file mode 100644 index 000000000000..9b8280170a6c --- /dev/null +++ b/tests/functional/cli-characterisation/nix-env-qa-context-opaque.err.exp @@ -0,0 +1,4 @@ +error: + … while evaluating the 'name' attribute of a derivation + + error: the string '/test-root/store/xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx-x' is not allowed to refer to a store path (such as '/test-root/store/xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx-x') diff --git a/tests/functional/cli-characterisation/nix-env-qa-deep-meta.cmd b/tests/functional/cli-characterisation/nix-env-qa-deep-meta.cmd new file mode 100644 index 000000000000..c4ca8e6ccab0 --- /dev/null +++ b/tests/functional/cli-characterisation/nix-env-qa-deep-meta.cmd @@ -0,0 +1 @@ +0 nix-env -f ./cli-characterisation/deep-meta.nix -qa --json --meta \ No newline at end of file diff --git a/tests/functional/cli-characterisation/nix-env-qa-deep-meta.out.exp b/tests/functional/cli-characterisation/nix-env-qa-deep-meta.out.exp new file mode 100644 index 000000000000..6a9c8f3f4af3 --- /dev/null +++ b/tests/functional/cli-characterisation/nix-env-qa-deep-meta.out.exp @@ -0,0 +1,22 @@ +{ + "": { + "meta": { + "description": "Has deeply nested meta", + "nested": { + "a": { + "b": { + "c": "deep value" + } + } + } + }, + "name": "deep-meta-1.0", + "outputName": "out", + "outputs": { + "out": null + }, + "pname": "deep-meta", + "system": "SYSTEM", + "version": "1.0" + } +} diff --git a/tests/functional/cli-characterisation/nix-env-qa-description.cmd b/tests/functional/cli-characterisation/nix-env-qa-description.cmd new file mode 100644 index 000000000000..b393ee1c7fc8 --- /dev/null +++ b/tests/functional/cli-characterisation/nix-env-qa-description.cmd @@ -0,0 +1 @@ +0 nix-env -f ./cli-characterisation/sample-package-set.nix -qa --description \ No newline at end of file diff --git a/tests/functional/cli-characterisation/nix-env-qa-description.out.exp b/tests/functional/cli-characterisation/nix-env-qa-description.out.exp new file mode 100644 index 000000000000..4de8601169e6 --- /dev/null +++ b/tests/functional/cli-characterisation/nix-env-qa-description.out.exp @@ -0,0 +1,9 @@ +bad-drvpath-1.0 +bad-output-specified-1.0 +deep-meta-1.0 Has deeply nested meta +ghost-outpath-1.0 +ghost-output-1.0 +infinite-meta-1.0 Has infinite recursion in meta +no-system-1.0 +normal-1.0 A normal package +simple-1.0 diff --git a/tests/functional/cli-characterisation/nix-env-qa-ghost-outpath.cmd b/tests/functional/cli-characterisation/nix-env-qa-ghost-outpath.cmd new file mode 100644 index 000000000000..078608362187 --- /dev/null +++ b/tests/functional/cli-characterisation/nix-env-qa-ghost-outpath.cmd @@ -0,0 +1 @@ +0 nix-env -f ./cli-characterisation/ghost-outpath.nix -qa --json --out-path \ No newline at end of file diff --git a/tests/functional/cli-characterisation/nix-env-qa-ghost-outpath.out.exp b/tests/functional/cli-characterisation/nix-env-qa-ghost-outpath.out.exp new file mode 100644 index 000000000000..3c56201cbb70 --- /dev/null +++ b/tests/functional/cli-characterisation/nix-env-qa-ghost-outpath.out.exp @@ -0,0 +1,12 @@ +{ + "": { + "name": "ghost-outpath-1.0", + "outputName": "", + "outputs": { + "out": "/test-root/store/xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx-out" + }, + "pname": "ghost-outpath", + "system": "unknown", + "version": "1.0" + } +} diff --git a/tests/functional/cli-characterisation/nix-env-qa-ghost-output.cmd b/tests/functional/cli-characterisation/nix-env-qa-ghost-output.cmd new file mode 100644 index 000000000000..d8a1e0a12452 --- /dev/null +++ b/tests/functional/cli-characterisation/nix-env-qa-ghost-output.cmd @@ -0,0 +1 @@ +0 nix-env -f ./cli-characterisation/ghost-output.nix -qa --json --out-path \ No newline at end of file diff --git a/tests/functional/cli-characterisation/nix-env-qa-ghost-output.out.exp b/tests/functional/cli-characterisation/nix-env-qa-ghost-output.out.exp new file mode 100644 index 000000000000..c844432a3a99 --- /dev/null +++ b/tests/functional/cli-characterisation/nix-env-qa-ghost-output.out.exp @@ -0,0 +1,12 @@ +{ + "": { + "name": "ghost-output-1.0", + "outputName": "", + "outputs": { + "out": "/test-root/store/xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx-out" + }, + "pname": "ghost-output", + "system": "unknown", + "version": "1.0" + } +} diff --git a/tests/functional/cli-characterisation/nix-env-qa-infinite-meta.cmd b/tests/functional/cli-characterisation/nix-env-qa-infinite-meta.cmd new file mode 100644 index 000000000000..5ea24bd38c5a --- /dev/null +++ b/tests/functional/cli-characterisation/nix-env-qa-infinite-meta.cmd @@ -0,0 +1 @@ +1 nix-env -f ./cli-characterisation/infinite-meta.nix -qa --json --meta \ No newline at end of file diff --git a/tests/functional/cli-characterisation/nix-env-qa-infinite-meta.err.exp b/tests/functional/cli-characterisation/nix-env-qa-infinite-meta.err.exp new file mode 100644 index 000000000000..b75fea85c88f --- /dev/null +++ b/tests/functional/cli-characterisation/nix-env-qa-infinite-meta.err.exp @@ -0,0 +1,9 @@ +error: + … while querying the derivation named 'infinite-meta-1.0' + + error: stack overflow; max-call-depth exceeded + at /pwd/cli-characterisation/infinite-meta.nix:3:7: + 2| let + 3| x = { inherit x; }; + | ^ + 4| in diff --git a/tests/functional/cli-characterisation/nix-env-qa-json-meta.cmd b/tests/functional/cli-characterisation/nix-env-qa-json-meta.cmd new file mode 100644 index 000000000000..02abd306bcf8 --- /dev/null +++ b/tests/functional/cli-characterisation/nix-env-qa-json-meta.cmd @@ -0,0 +1 @@ +1 nix-env -f ./cli-characterisation/sample-package-set.nix -qa --json --meta \ No newline at end of file diff --git a/tests/functional/cli-characterisation/nix-env-qa-json-meta.err.exp b/tests/functional/cli-characterisation/nix-env-qa-json-meta.err.exp new file mode 100644 index 000000000000..b75fea85c88f --- /dev/null +++ b/tests/functional/cli-characterisation/nix-env-qa-json-meta.err.exp @@ -0,0 +1,9 @@ +error: + … while querying the derivation named 'infinite-meta-1.0' + + error: stack overflow; max-call-depth exceeded + at /pwd/cli-characterisation/infinite-meta.nix:3:7: + 2| let + 3| x = { inherit x; }; + | ^ + 4| in diff --git a/tests/functional/cli-characterisation/nix-env-qa-json-paths.cmd b/tests/functional/cli-characterisation/nix-env-qa-json-paths.cmd new file mode 100644 index 000000000000..4a58abc964fe --- /dev/null +++ b/tests/functional/cli-characterisation/nix-env-qa-json-paths.cmd @@ -0,0 +1 @@ +1 nix-env -f ./cli-characterisation/sample-package-set.nix -qa --json --out-path --drv-path \ No newline at end of file diff --git a/tests/functional/cli-characterisation/nix-env-qa-json-paths.err.exp b/tests/functional/cli-characterisation/nix-env-qa-json-paths.err.exp new file mode 100644 index 000000000000..e05ff77298e7 --- /dev/null +++ b/tests/functional/cli-characterisation/nix-env-qa-json-paths.err.exp @@ -0,0 +1,11 @@ +error: + … while querying the derivation named 'bad-drvpath-1.0' + + … while evaluating the 'drvPath' attribute of a derivation + at /pwd/cli-characterisation/bad-drvpath.nix:5:3: + 4| outPath = builtins.toFile "out" ""; + 5| drvPath = builtins.toFile "not-a-drv" ""; + | ^ + 6| } + + error: store path 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx-not-a-drv' is not a valid derivation path diff --git a/tests/functional/cli-characterisation/nix-env-qa-json.cmd b/tests/functional/cli-characterisation/nix-env-qa-json.cmd new file mode 100644 index 000000000000..85030b56cd0b --- /dev/null +++ b/tests/functional/cli-characterisation/nix-env-qa-json.cmd @@ -0,0 +1 @@ +0 nix-env -f ./cli-characterisation/sample-package-set.nix -qa --json \ No newline at end of file diff --git a/tests/functional/cli-characterisation/nix-env-qa-json.out.exp b/tests/functional/cli-characterisation/nix-env-qa-json.out.exp new file mode 100644 index 000000000000..6d0c7ce7c4fb --- /dev/null +++ b/tests/functional/cli-characterisation/nix-env-qa-json.out.exp @@ -0,0 +1,94 @@ +{ + "bad-drvpath": { + "name": "bad-drvpath-1.0", + "outputName": "", + "outputs": { + "out": null + }, + "pname": "bad-drvpath", + "system": "unknown", + "version": "1.0" + }, + "bad-output-specified": { + "name": "bad-output-specified-1.0", + "outputName": "nonexistent", + "outputs": { + "out": null + }, + "pname": "bad-output-specified", + "system": "unknown", + "version": "1.0" + }, + "deep-meta": { + "name": "deep-meta-1.0", + "outputName": "out", + "outputs": { + "out": null + }, + "pname": "deep-meta", + "system": "SYSTEM", + "version": "1.0" + }, + "ghost-outpath": { + "name": "ghost-outpath-1.0", + "outputName": "", + "outputs": { + "ghost": null, + "out": null + }, + "pname": "ghost-outpath", + "system": "unknown", + "version": "1.0" + }, + "ghost-output": { + "name": "ghost-output-1.0", + "outputName": "", + "outputs": { + "ghost": null, + "out": null + }, + "pname": "ghost-output", + "system": "unknown", + "version": "1.0" + }, + "infinite-meta": { + "name": "infinite-meta-1.0", + "outputName": "out", + "outputs": { + "out": null + }, + "pname": "infinite-meta", + "system": "SYSTEM", + "version": "1.0" + }, + "no-system": { + "name": "no-system-1.0", + "outputName": "", + "outputs": { + "out": null + }, + "pname": "no-system", + "system": "unknown", + "version": "1.0" + }, + "normal": { + "name": "normal-1.0", + "outputName": "out", + "outputs": { + "out": null + }, + "pname": "normal", + "system": "SYSTEM", + "version": "1.0" + }, + "simple": { + "name": "simple-1.0", + "outputName": "out", + "outputs": { + "out": null + }, + "pname": "simple", + "system": "SYSTEM", + "version": "1.0" + } +} diff --git a/tests/functional/cli-characterisation/nix-env-qa-meta-types.cmd b/tests/functional/cli-characterisation/nix-env-qa-meta-types.cmd new file mode 100644 index 000000000000..0bf766c9264c --- /dev/null +++ b/tests/functional/cli-characterisation/nix-env-qa-meta-types.cmd @@ -0,0 +1 @@ +0 nix-env -f ./cli-characterisation/meta-types.nix -qa --json --meta \ No newline at end of file diff --git a/tests/functional/cli-characterisation/nix-env-qa-meta-types.out.exp b/tests/functional/cli-characterisation/nix-env-qa-meta-types.out.exp new file mode 100644 index 000000000000..7b8975f366f9 --- /dev/null +++ b/tests/functional/cli-characterisation/nix-env-qa-meta-types.out.exp @@ -0,0 +1,26 @@ +{ + "": { + "meta": { + "aBool": true, + "aFloat": 3.14, + "aList": [ + 1, + 2, + 3 + ], + "aString": "hello", + "anInt": 42, + "stringBool": "true", + "stringFloat": "2.72", + "stringInt": "123" + }, + "name": "meta-types-1.0", + "outputName": "out", + "outputs": { + "out": null + }, + "pname": "meta-types", + "system": "SYSTEM", + "version": "1.0" + } +} diff --git a/tests/functional/cli-characterisation/nix-env-qa-meta-with-bad-list.cmd b/tests/functional/cli-characterisation/nix-env-qa-meta-with-bad-list.cmd new file mode 100644 index 000000000000..d88f3d073bef --- /dev/null +++ b/tests/functional/cli-characterisation/nix-env-qa-meta-with-bad-list.cmd @@ -0,0 +1 @@ +0 nix-env -f ./cli-characterisation/meta-with-bad-list.nix -qa --json --meta \ No newline at end of file diff --git a/tests/functional/cli-characterisation/nix-env-qa-meta-with-bad-list.err.exp b/tests/functional/cli-characterisation/nix-env-qa-meta-with-bad-list.err.exp new file mode 100644 index 000000000000..7d18fb2355eb --- /dev/null +++ b/tests/functional/cli-characterisation/nix-env-qa-meta-with-bad-list.err.exp @@ -0,0 +1 @@ +derivation 'meta-with-bad-list-1.0' has invalid meta attribute 'bad' diff --git a/tests/functional/cli-characterisation/nix-env-qa-meta-with-bad-list.out.exp b/tests/functional/cli-characterisation/nix-env-qa-meta-with-bad-list.out.exp new file mode 100644 index 000000000000..335aedea1989 --- /dev/null +++ b/tests/functional/cli-characterisation/nix-env-qa-meta-with-bad-list.out.exp @@ -0,0 +1,15 @@ +{ + "": { + "meta": { + "bad": null + }, + "name": "meta-with-bad-list-1.0", + "outputName": "out", + "outputs": { + "out": null + }, + "pname": "meta-with-bad-list", + "system": "SYSTEM", + "version": "1.0" + } +} diff --git a/tests/functional/cli-characterisation/nix-env-qa-meta-with-drv.cmd b/tests/functional/cli-characterisation/nix-env-qa-meta-with-drv.cmd new file mode 100644 index 000000000000..89ac06abedde --- /dev/null +++ b/tests/functional/cli-characterisation/nix-env-qa-meta-with-drv.cmd @@ -0,0 +1 @@ +0 nix-env -f ./cli-characterisation/meta-with-drv.nix -qa --json --meta \ No newline at end of file diff --git a/tests/functional/cli-characterisation/nix-env-qa-meta-with-drv.err.exp b/tests/functional/cli-characterisation/nix-env-qa-meta-with-drv.err.exp new file mode 100644 index 000000000000..4aaee78ace18 --- /dev/null +++ b/tests/functional/cli-characterisation/nix-env-qa-meta-with-drv.err.exp @@ -0,0 +1 @@ +derivation 'meta-with-drv-1.0' has invalid meta attribute 'someDrv' diff --git a/tests/functional/cli-characterisation/nix-env-qa-meta-with-drv.out.exp b/tests/functional/cli-characterisation/nix-env-qa-meta-with-drv.out.exp new file mode 100644 index 000000000000..c99a0b9eee7c --- /dev/null +++ b/tests/functional/cli-characterisation/nix-env-qa-meta-with-drv.out.exp @@ -0,0 +1,15 @@ +{ + "": { + "meta": { + "someDrv": null + }, + "name": "meta-with-drv-1.0", + "outputName": "out", + "outputs": { + "out": null + }, + "pname": "meta-with-drv", + "system": "SYSTEM", + "version": "1.0" + } +} diff --git a/tests/functional/cli-characterisation/nix-env-qa-meta-with-function.cmd b/tests/functional/cli-characterisation/nix-env-qa-meta-with-function.cmd new file mode 100644 index 000000000000..c41a448f8b25 --- /dev/null +++ b/tests/functional/cli-characterisation/nix-env-qa-meta-with-function.cmd @@ -0,0 +1 @@ +0 nix-env -f ./cli-characterisation/meta-with-function.nix -qa --json --meta \ No newline at end of file diff --git a/tests/functional/cli-characterisation/nix-env-qa-meta-with-function.err.exp b/tests/functional/cli-characterisation/nix-env-qa-meta-with-function.err.exp new file mode 100644 index 000000000000..54ae42856093 --- /dev/null +++ b/tests/functional/cli-characterisation/nix-env-qa-meta-with-function.err.exp @@ -0,0 +1 @@ +derivation 'meta-with-function-1.0' has invalid meta attribute 'bad' diff --git a/tests/functional/cli-characterisation/nix-env-qa-meta-with-function.out.exp b/tests/functional/cli-characterisation/nix-env-qa-meta-with-function.out.exp new file mode 100644 index 000000000000..42b9a7b3c4a1 --- /dev/null +++ b/tests/functional/cli-characterisation/nix-env-qa-meta-with-function.out.exp @@ -0,0 +1,15 @@ +{ + "": { + "meta": { + "bad": null + }, + "name": "meta-with-function-1.0", + "outputName": "out", + "outputs": { + "out": null + }, + "pname": "meta-with-function", + "system": "SYSTEM", + "version": "1.0" + } +} diff --git a/tests/functional/cli-characterisation/nix-env-qa-meta-with-outpath.cmd b/tests/functional/cli-characterisation/nix-env-qa-meta-with-outpath.cmd new file mode 100644 index 000000000000..8d986fdd8997 --- /dev/null +++ b/tests/functional/cli-characterisation/nix-env-qa-meta-with-outpath.cmd @@ -0,0 +1 @@ +0 nix-env -f ./cli-characterisation/meta-with-outpath.nix -qa --json --meta \ No newline at end of file diff --git a/tests/functional/cli-characterisation/nix-env-qa-meta-with-outpath.err.exp b/tests/functional/cli-characterisation/nix-env-qa-meta-with-outpath.err.exp new file mode 100644 index 000000000000..a701efba7997 --- /dev/null +++ b/tests/functional/cli-characterisation/nix-env-qa-meta-with-outpath.err.exp @@ -0,0 +1 @@ +derivation 'meta-with-outpath-1.0' has invalid meta attribute 'bad' diff --git a/tests/functional/cli-characterisation/nix-env-qa-meta-with-outpath.out.exp b/tests/functional/cli-characterisation/nix-env-qa-meta-with-outpath.out.exp new file mode 100644 index 000000000000..df5043c753ba --- /dev/null +++ b/tests/functional/cli-characterisation/nix-env-qa-meta-with-outpath.out.exp @@ -0,0 +1,16 @@ +{ + "": { + "meta": { + "bad": null, + "description": "Has outPath in meta" + }, + "name": "meta-with-outpath-1.0", + "outputName": "out", + "outputs": { + "out": null + }, + "pname": "meta-with-outpath", + "system": "SYSTEM", + "version": "1.0" + } +} diff --git a/tests/functional/cli-characterisation/nix-env-qa-name-with-context.cmd b/tests/functional/cli-characterisation/nix-env-qa-name-with-context.cmd new file mode 100644 index 000000000000..89f2449425b7 --- /dev/null +++ b/tests/functional/cli-characterisation/nix-env-qa-name-with-context.cmd @@ -0,0 +1 @@ +1 nix-env -f ./cli-characterisation/name-with-context.nix -qa --json \ No newline at end of file diff --git a/tests/functional/cli-characterisation/nix-env-qa-name-with-context.err.exp b/tests/functional/cli-characterisation/nix-env-qa-name-with-context.err.exp new file mode 100644 index 000000000000..c7b5ac32e7dd --- /dev/null +++ b/tests/functional/cli-characterisation/nix-env-qa-name-with-context.err.exp @@ -0,0 +1,4 @@ +error: + … while evaluating the 'name' attribute of a derivation + + error: the string '/test-root/store/xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx-normal-1.0' is not allowed to refer to a store path (such as '/test-root/store/xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx-normal-1.0.drv^out') diff --git a/tests/functional/cli-characterisation/nix-env-qa-no-name.cmd b/tests/functional/cli-characterisation/nix-env-qa-no-name.cmd new file mode 100644 index 000000000000..4e0d6736c208 --- /dev/null +++ b/tests/functional/cli-characterisation/nix-env-qa-no-name.cmd @@ -0,0 +1 @@ +1 nix-env -f ./cli-characterisation/no-name.nix -qa --json \ No newline at end of file diff --git a/tests/functional/cli-characterisation/nix-env-qa-no-name.err.exp b/tests/functional/cli-characterisation/nix-env-qa-no-name.err.exp new file mode 100644 index 000000000000..fb237dd8a797 --- /dev/null +++ b/tests/functional/cli-characterisation/nix-env-qa-no-name.err.exp @@ -0,0 +1 @@ +error: derivation name missing diff --git a/tests/functional/cli-characterisation/nix-env-qa-no-outpath.cmd b/tests/functional/cli-characterisation/nix-env-qa-no-outpath.cmd new file mode 100644 index 000000000000..b119a37c8d99 --- /dev/null +++ b/tests/functional/cli-characterisation/nix-env-qa-no-outpath.cmd @@ -0,0 +1 @@ +1 nix-env -f ./cli-characterisation/no-outpath.nix -qa --json --out-path \ No newline at end of file diff --git a/tests/functional/cli-characterisation/nix-env-qa-no-outpath.err.exp b/tests/functional/cli-characterisation/nix-env-qa-no-outpath.err.exp new file mode 100644 index 000000000000..b296c1bc5ddd --- /dev/null +++ b/tests/functional/cli-characterisation/nix-env-qa-no-outpath.err.exp @@ -0,0 +1,4 @@ +error: + … while querying the derivation named 'no-outpath-1.0' + + error: derivation does not have attribute 'outPath' diff --git a/tests/functional/cli-characterisation/nix-env-qa-no-system.cmd b/tests/functional/cli-characterisation/nix-env-qa-no-system.cmd new file mode 100644 index 000000000000..68085b8ed3b5 --- /dev/null +++ b/tests/functional/cli-characterisation/nix-env-qa-no-system.cmd @@ -0,0 +1 @@ +0 nix-env -f ./cli-characterisation/no-system.nix -qa --json \ No newline at end of file diff --git a/tests/functional/cli-characterisation/nix-env-qa-no-system.out.exp b/tests/functional/cli-characterisation/nix-env-qa-no-system.out.exp new file mode 100644 index 000000000000..06d63ea925b0 --- /dev/null +++ b/tests/functional/cli-characterisation/nix-env-qa-no-system.out.exp @@ -0,0 +1,12 @@ +{ + "": { + "name": "no-system-1.0", + "outputName": "", + "outputs": { + "out": null + }, + "pname": "no-system", + "system": "unknown", + "version": "1.0" + } +} diff --git a/tests/functional/cli-characterisation/nix-env-qa-not-a-drv.cmd b/tests/functional/cli-characterisation/nix-env-qa-not-a-drv.cmd new file mode 100644 index 000000000000..1081a053a73a --- /dev/null +++ b/tests/functional/cli-characterisation/nix-env-qa-not-a-drv.cmd @@ -0,0 +1 @@ +1 nix-env -f ./cli-characterisation/not-a-drv.nix -qa --json \ No newline at end of file diff --git a/tests/functional/cli-characterisation/nix-env-qa-not-a-drv.err.exp b/tests/functional/cli-characterisation/nix-env-qa-not-a-drv.err.exp new file mode 100644 index 000000000000..b8418027d9f1 --- /dev/null +++ b/tests/functional/cli-characterisation/nix-env-qa-not-a-drv.err.exp @@ -0,0 +1 @@ +error: expression does not evaluate to a derivation (or a set or list of those) diff --git a/tests/functional/cli-characterisation/nix-env-qa-outputName-with-context.cmd b/tests/functional/cli-characterisation/nix-env-qa-outputName-with-context.cmd new file mode 100644 index 000000000000..60b4cc838bee --- /dev/null +++ b/tests/functional/cli-characterisation/nix-env-qa-outputName-with-context.cmd @@ -0,0 +1 @@ +1 nix-env -f ./cli-characterisation/outputName-with-context.nix -qa --json \ No newline at end of file diff --git a/tests/functional/cli-characterisation/nix-env-qa-outputName-with-context.err.exp b/tests/functional/cli-characterisation/nix-env-qa-outputName-with-context.err.exp new file mode 100644 index 000000000000..b915365e5ace --- /dev/null +++ b/tests/functional/cli-characterisation/nix-env-qa-outputName-with-context.err.exp @@ -0,0 +1,6 @@ +error: + … while querying the derivation named 'outputName-with-context-1.0' + + … while evaluating the output name of a derivation + + error: the string '/test-root/store/xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx-normal-1.0' is not allowed to refer to a store path (such as '/test-root/store/xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx-normal-1.0.drv^out') diff --git a/tests/functional/cli-characterisation/nix-env-qa-outputsToInstall-with-context.cmd b/tests/functional/cli-characterisation/nix-env-qa-outputsToInstall-with-context.cmd new file mode 100644 index 000000000000..7f8cb8bf5a94 --- /dev/null +++ b/tests/functional/cli-characterisation/nix-env-qa-outputsToInstall-with-context.cmd @@ -0,0 +1 @@ +1 nix-env -f ./cli-characterisation/outputsToInstall-with-context.nix -i outputsToInstall-with-context \ No newline at end of file diff --git a/tests/functional/cli-characterisation/nix-env-qa-outputsToInstall-with-context.err.exp b/tests/functional/cli-characterisation/nix-env-qa-outputsToInstall-with-context.err.exp new file mode 100644 index 000000000000..cc04948eca5e --- /dev/null +++ b/tests/functional/cli-characterisation/nix-env-qa-outputsToInstall-with-context.err.exp @@ -0,0 +1,5 @@ +installing 'outputsToInstall-with-context-1.0' +this derivation will be built: + /test-root/store/xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx-outputsToInstall-with-context-1.0.drv +building '/test-root/store/xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx-outputsToInstall-with-context-1.0.drv'... +error: this derivation has bad 'meta.outputsToInstall' diff --git a/tests/functional/cli-characterisation/nix-env-qa-system-with-context.cmd b/tests/functional/cli-characterisation/nix-env-qa-system-with-context.cmd new file mode 100644 index 000000000000..e93fe0ccedbf --- /dev/null +++ b/tests/functional/cli-characterisation/nix-env-qa-system-with-context.cmd @@ -0,0 +1 @@ +1 nix-env -f ./cli-characterisation/system-with-context.nix -qa --json \ No newline at end of file diff --git a/tests/functional/cli-characterisation/nix-env-qa-system-with-context.err.exp b/tests/functional/cli-characterisation/nix-env-qa-system-with-context.err.exp new file mode 100644 index 000000000000..25986ba4de77 --- /dev/null +++ b/tests/functional/cli-characterisation/nix-env-qa-system-with-context.err.exp @@ -0,0 +1,11 @@ +error: + … while querying the derivation named 'system-with-context-1.0' + + … while evaluating the 'system' attribute of a derivation + at /pwd/cli-characterisation/system-with-context.nix:11:3: + 10| name = "system-with-context-1.0"; + 11| system = "${normal}"; + | ^ + 12| outPath = "/nix/store/fake"; + + error: the string '/test-root/store/xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx-normal-1.0' is not allowed to refer to a store path (such as '/test-root/store/xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx-normal-1.0.drv^out') diff --git a/tests/functional/cli-characterisation/nix-env-qa-xml.cmd b/tests/functional/cli-characterisation/nix-env-qa-xml.cmd new file mode 100644 index 000000000000..1533f0e7f5ed --- /dev/null +++ b/tests/functional/cli-characterisation/nix-env-qa-xml.cmd @@ -0,0 +1 @@ +0 nix-env -f ./cli-characterisation/sample-package-set.nix -qa --xml \ No newline at end of file diff --git a/tests/functional/cli-characterisation/nix-env-qa-xml.out.exp b/tests/functional/cli-characterisation/nix-env-qa-xml.out.exp new file mode 100644 index 000000000000..1a853c51f24b --- /dev/null +++ b/tests/functional/cli-characterisation/nix-env-qa-xml.out.exp @@ -0,0 +1,32 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/functional/cli-characterisation/nix-instantiate-no-drvpath.cmd b/tests/functional/cli-characterisation/nix-instantiate-no-drvpath.cmd new file mode 100644 index 000000000000..b8364682e704 --- /dev/null +++ b/tests/functional/cli-characterisation/nix-instantiate-no-drvpath.cmd @@ -0,0 +1 @@ +1 nix-instantiate ./cli-characterisation/no-drvpath.nix \ No newline at end of file diff --git a/tests/functional/cli-characterisation/nix-instantiate-no-drvpath.err.exp b/tests/functional/cli-characterisation/nix-instantiate-no-drvpath.err.exp new file mode 100644 index 000000000000..2afb987e1393 --- /dev/null +++ b/tests/functional/cli-characterisation/nix-instantiate-no-drvpath.err.exp @@ -0,0 +1 @@ +error: derivation does not contain a 'drvPath' attribute diff --git a/tests/functional/cli-characterisation/no-drvpath.nix b/tests/functional/cli-characterisation/no-drvpath.nix new file mode 100644 index 000000000000..3b9a511314a9 --- /dev/null +++ b/tests/functional/cli-characterisation/no-drvpath.nix @@ -0,0 +1,5 @@ +{ + type = "derivation"; + name = "no-drvpath-1.0"; + outPath = "/nix/store/fake"; +} diff --git a/tests/functional/cli-characterisation/no-name.nix b/tests/functional/cli-characterisation/no-name.nix new file mode 100644 index 000000000000..fb18a45bc68d --- /dev/null +++ b/tests/functional/cli-characterisation/no-name.nix @@ -0,0 +1,4 @@ +{ + type = "derivation"; + outPath = "/nix/store/fake"; +} diff --git a/tests/functional/cli-characterisation/no-outpath.nix b/tests/functional/cli-characterisation/no-outpath.nix new file mode 100644 index 000000000000..05fea85845de --- /dev/null +++ b/tests/functional/cli-characterisation/no-outpath.nix @@ -0,0 +1,4 @@ +{ + type = "derivation"; + name = "no-outpath-1.0"; +} diff --git a/tests/functional/cli-characterisation/no-system.nix b/tests/functional/cli-characterisation/no-system.nix new file mode 100644 index 000000000000..50a29a6e8f7b --- /dev/null +++ b/tests/functional/cli-characterisation/no-system.nix @@ -0,0 +1,5 @@ +{ + type = "derivation"; + name = "no-system-1.0"; + outPath = "/nix/store/fake"; +} diff --git a/tests/functional/cli-characterisation/normal.nix b/tests/functional/cli-characterisation/normal.nix new file mode 100644 index 000000000000..ae6daa66a98e --- /dev/null +++ b/tests/functional/cli-characterisation/normal.nix @@ -0,0 +1,10 @@ +with import ../config.nix; +mkDerivation { + name = "normal-1.0"; + buildCommand = "mkdir -p $out"; + meta = { + description = "A normal package"; + homepage = "https://example.com"; + license = "MIT"; + }; +} diff --git a/tests/functional/cli-characterisation/not-a-drv.nix b/tests/functional/cli-characterisation/not-a-drv.nix new file mode 100644 index 000000000000..7deb8b173227 --- /dev/null +++ b/tests/functional/cli-characterisation/not-a-drv.nix @@ -0,0 +1 @@ +"just a string" diff --git a/tests/functional/cli-characterisation/outputName-with-context.nix b/tests/functional/cli-characterisation/outputName-with-context.nix new file mode 100644 index 000000000000..c23c8f7d441f --- /dev/null +++ b/tests/functional/cli-characterisation/outputName-with-context.nix @@ -0,0 +1,13 @@ +with import ../config.nix; +let + normal = mkDerivation { + name = "normal-1.0"; + buildCommand = "mkdir -p $out"; + }; +in +{ + type = "derivation"; + name = "outputName-with-context-1.0"; + outPath = "/nix/store/fake"; + outputName = "${normal}"; +} diff --git a/tests/functional/cli-characterisation/outputsToInstall-with-context.nix b/tests/functional/cli-characterisation/outputsToInstall-with-context.nix new file mode 100644 index 000000000000..527cf0d415ff --- /dev/null +++ b/tests/functional/cli-characterisation/outputsToInstall-with-context.nix @@ -0,0 +1,14 @@ +with import ../config.nix; +let + normal = mkDerivation { + name = "normal-1.0"; + buildCommand = "mkdir -p $out"; + }; +in +mkDerivation { + name = "outputsToInstall-with-context-1.0"; + buildCommand = "mkdir -p $out"; + meta = { + outputsToInstall = [ "${normal}" ]; + }; +} diff --git a/tests/functional/cli-characterisation/sample-package-set.nix b/tests/functional/cli-characterisation/sample-package-set.nix new file mode 100644 index 000000000000..7f3fede84fac --- /dev/null +++ b/tests/functional/cli-characterisation/sample-package-set.nix @@ -0,0 +1,19 @@ +# A representative package set for enumeration tests (--description, --xml, +# --json, --json --meta, --json --out-path --drv-path, -P -a). +# +# This set is deliberately small and stable. Adding new individual test +# cases should NOT require changes here. +{ + normal = import ./normal.nix; + simple = import ./simple.nix; + deep-meta = import ./deep-meta.nix; + no-system = import ./no-system.nix; + infinite-meta = import ./infinite-meta.nix; + bad-drvpath = import ./bad-drvpath.nix; + bad-output-specified = import ./bad-output-specified.nix; + ghost-outpath = import ./ghost-outpath.nix; + ghost-output = import ./ghost-output.nix; + assert-fail = import ./assert-fail.nix; + not-a-drv = import ./not-a-drv.nix; + isolated.no-name = import ./no-name.nix; +} diff --git a/tests/functional/cli-characterisation/simple.nix b/tests/functional/cli-characterisation/simple.nix new file mode 100644 index 000000000000..aa5fb5406060 --- /dev/null +++ b/tests/functional/cli-characterisation/simple.nix @@ -0,0 +1,5 @@ +with import ../config.nix; +mkDerivation { + name = "simple-1.0"; + buildCommand = "mkdir -p $out"; +} diff --git a/tests/functional/cli-characterisation/system-with-context.nix b/tests/functional/cli-characterisation/system-with-context.nix new file mode 100644 index 000000000000..a88ef8ebbb95 --- /dev/null +++ b/tests/functional/cli-characterisation/system-with-context.nix @@ -0,0 +1,13 @@ +with import ../config.nix; +let + normal = mkDerivation { + name = "normal-1.0"; + buildCommand = "mkdir -p $out"; + }; +in +{ + type = "derivation"; + name = "system-with-context-1.0"; + system = "${normal}"; + outPath = "/nix/store/fake"; +} diff --git a/tests/functional/common/vars.sh b/tests/functional/common/vars.sh index d4d917dae8d2..a6b83ae4e3cf 100644 --- a/tests/functional/common/vars.sh +++ b/tests/functional/common/vars.sh @@ -64,6 +64,9 @@ unset XDG_CONFIG_HOME unset XDG_CONFIG_DIRS unset XDG_CACHE_HOME unset GIT_DIR +# Isolate tests from host git config (signing, url rewrites, etc.) +export GIT_CONFIG_SYSTEM=/dev/null +export GIT_CONFIG_GLOBAL=/dev/null export IMPURE_VAR1=foo export IMPURE_VAR2=bar diff --git a/tests/functional/dyn-drv/failing-outer.sh b/tests/functional/dyn-drv/failing-outer.sh index dcf3e830ed54..709f79619ea2 100644 --- a/tests/functional/dyn-drv/failing-outer.sh +++ b/tests/functional/dyn-drv/failing-outer.sh @@ -41,5 +41,8 @@ out=$(nix build --impure --no-link --expr ' builtins.outputOf failingProducer.outPath "out" ' 2>&1) || true +# Store layer needs bugfix +requireDaemonNewerThan "2.34pre" + # The error message must NOT be empty - it should mention the failed derivation echo "$out" | grepQuiet "failed to obtain derivation of" diff --git a/tests/functional/eval.sh b/tests/functional/eval.sh index f876f5ac4836..586c413c961c 100755 --- a/tests/functional/eval.sh +++ b/tests/functional/eval.sh @@ -39,6 +39,30 @@ nix-instantiate --eval -E 'assert 1 + 2 == 3; true' ln -sfn cycle.nix "$TEST_ROOT/cycle.nix" (! nix eval --file "$TEST_ROOT/cycle.nix") +# Test that printing deep data structures produces a controlled error. +# The expression creates a non-cyclic but infinitely deep structure: +# f returns immediately with a thunk, so Nix call depth stays at 1, +# but Printer::print recurses on the C++ stack. +expectStderr 1 nix eval --expr 'let f = n: { inner = f (n + 1); }; in f 0' --max-call-depth 100 \ + | grepQuiet "stack overflow; max-call-depth exceeded" + +# Same for builtins.toXML +expectStderr 1 nix eval --expr 'builtins.toXML (let f = n: { inner = f (n + 1); }; in f 0)' --max-call-depth 100 \ + | grepQuiet "stack overflow; max-call-depth exceeded" + +# Same for equality comparison (n is not observable, so structures are equal) +expectStderr 1 nix eval --expr 'let f = n: { inner = f (n + 1); }; in f 0 == f 1' --max-call-depth 100 \ + | grepQuiet "stack overflow; max-call-depth exceeded" + +# Same for assert with equality (uses assertEqValues) +expectStderr 1 nix eval --expr 'let f = n: { inner = f (n + 1); }; in assert f 0 == f 1; true' --max-call-depth 100 \ + | grepQuiet "stack overflow; max-call-depth exceeded" + +# Same for string coercion with __toString +# shellcheck disable=SC2016 +expectStderr 1 nix eval --expr 'let f = n: { __toString = _: f (n + 1); }; in "${f 0}"' --max-call-depth 100 \ + | grepQuiet "stack overflow; max-call-depth exceeded" + # --file and --pure-eval don't mix. expectStderr 1 nix eval --pure-eval --file "$TEST_ROOT/cycle.nix" | grepQuiet "not compatible" diff --git a/tests/functional/external-builders.sh b/tests/functional/external-builders.sh index 4c1d5636a0af..f01f0b5a7b3a 100644 --- a/tests/functional/external-builders.sh +++ b/tests/functional/external-builders.sh @@ -23,7 +23,7 @@ external_builder="$TEST_ROOT/external-builder.sh" cat > "$external_builder" </dev/null # Update 'path' to reflect latest master -path=$(nix eval --impure --raw --expr "(builtins.fetchGit file://$repo).outPath") +path=$(nix eval --impure --raw --expr "(builtins.fetchGit \"file://$repo\").outPath") # Check behavior when non-master branch is used git -C "$repo" checkout "$rev2" -b dev echo dev > "$repo"/hello # File URI uses dirty tree unless specified otherwise -path2=$(nix eval --impure --raw --expr "(builtins.fetchGit file://$repo).outPath") +path2=$(nix eval --impure --raw --expr "(builtins.fetchGit \"file://$repo\").outPath") [ "$(cat "$path2"/hello)" = dev ] # Using local path with branch other than 'master' should work when clean or dirty @@ -254,7 +254,7 @@ echo "/exported-wonky export-ignore=wonk" >> "$repo"/.gitattributes git -C "$repo" add not-exported-file exported-wonky .gitattributes git -C "$repo" commit -m 'Bla6' rev5=$(git -C "$repo" rev-parse HEAD) -path12=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$repo; rev = \"$rev5\"; }).outPath") +path12=$(nix eval --raw --expr "(builtins.fetchGit { url = \"file://$repo\"; rev = \"$rev5\"; }).outPath") [[ ! -e $path12/not-exported-file ]] [[ -e $path12/exported-wonky ]] diff --git a/tests/functional/fetchGitSubmodules.sh b/tests/functional/fetchGitSubmodules.sh index bf5fe5df3877..7839fef0d7e8 100755 --- a/tests/functional/fetchGitSubmodules.sh +++ b/tests/functional/fetchGitSubmodules.sh @@ -18,9 +18,11 @@ rm -rf "${rootRepo}" "${subRepo}" "$TEST_HOME"/.cache/nix # submodule is intentionally local and it's all trusted, so we # disable this restriction. Setting it per repo is not sufficient, as # the repo-local config does not apply to the commands run from -# outside the repos by Nix. -export XDG_CONFIG_HOME=$TEST_HOME/.config -git config --global protocol.file.allow always +# outside the repos by Nix. We use environment variables to avoid +# attempting to write to a read-only system git config. +export GIT_CONFIG_COUNT=1 +export GIT_CONFIG_KEY_0=protocol.file.allow +export GIT_CONFIG_VALUE_0=always addGitContent() { echo "lorem ipsum" > "$1"/content @@ -40,16 +42,16 @@ git -C "$rootRepo" commit -m "Add submodule" rev=$(git -C "$rootRepo" rev-parse HEAD) -r1=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; }).outPath") -r2=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = false; }).outPath") -r3=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = true; }).outPath") +r1=$(nix eval --raw --expr "(builtins.fetchGit { url = \"file://$rootRepo\"; rev = \"$rev\"; }).outPath") +r2=$(nix eval --raw --expr "(builtins.fetchGit { url = \"file://$rootRepo\"; rev = \"$rev\"; submodules = false; }).outPath") +r3=$(nix eval --raw --expr "(builtins.fetchGit { url = \"file://$rootRepo\"; rev = \"$rev\"; submodules = true; }).outPath") [[ $r1 == "$r2" ]] [[ $r2 != "$r3" ]] -r4=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$rootRepo; ref = \"master\"; rev = \"$rev\"; }).outPath") -r5=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$rootRepo; ref = \"master\"; rev = \"$rev\"; submodules = false; }).outPath") -r6=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$rootRepo; ref = \"master\"; rev = \"$rev\"; submodules = true; }).outPath") +r4=$(nix eval --raw --expr "(builtins.fetchGit { url = \"file://$rootRepo\"; ref = \"master\"; rev = \"$rev\"; }).outPath") +r5=$(nix eval --raw --expr "(builtins.fetchGit { url = \"file://$rootRepo\"; ref = \"master\"; rev = \"$rev\"; submodules = false; }).outPath") +r6=$(nix eval --raw --expr "(builtins.fetchGit { url = \"file://$rootRepo\"; ref = \"master\"; rev = \"$rev\"; submodules = true; }).outPath") r7=$(nix eval --raw --expr "(builtins.fetchGit { url = $rootRepo; ref = \"master\"; rev = \"$rev\"; submodules = true; }).outPath") r8=$(nix eval --raw --expr "(builtins.fetchGit { url = $rootRepo; rev = \"$rev\"; submodules = true; }).outPath") @@ -68,10 +70,10 @@ have_submodules=$(nix eval --expr "(builtins.fetchGit { url = $rootRepo; rev = \ have_submodules=$(nix eval --expr "(builtins.fetchGit { url = $rootRepo; rev = \"$rev\"; submodules = true; }).submodules") [[ $have_submodules == true ]] -pathWithoutSubmodules=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; }).outPath") -pathWithSubmodules=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = true; }).outPath") -pathWithSubmodulesAgain=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = true; }).outPath") -pathWithSubmodulesAgainWithRef=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$rootRepo; ref = \"master\"; rev = \"$rev\"; submodules = true; }).outPath") +pathWithoutSubmodules=$(nix eval --raw --expr "(builtins.fetchGit { url = \"file://$rootRepo\"; rev = \"$rev\"; }).outPath") +pathWithSubmodules=$(nix eval --raw --expr "(builtins.fetchGit { url = \"file://$rootRepo\"; rev = \"$rev\"; submodules = true; }).outPath") +pathWithSubmodulesAgain=$(nix eval --raw --expr "(builtins.fetchGit { url = \"file://$rootRepo\"; rev = \"$rev\"; submodules = true; }).outPath") +pathWithSubmodulesAgainWithRef=$(nix eval --raw --expr "(builtins.fetchGit { url = \"file://$rootRepo\"; ref = \"master\"; rev = \"$rev\"; submodules = true; }).outPath") # The resulting store path cannot be the same. [[ $pathWithoutSubmodules != "$pathWithSubmodules" ]] @@ -93,8 +95,8 @@ test "$(find "$pathWithSubmodules" -name .git)" = "" # Git repos without submodules can be fetched with submodules = true. subRev=$(git -C "$subRepo" rev-parse HEAD) -noSubmoduleRepoBaseline=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$subRepo; rev = \"$subRev\"; }).outPath") -noSubmoduleRepo=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$subRepo; rev = \"$subRev\"; submodules = true; }).outPath") +noSubmoduleRepoBaseline=$(nix eval --raw --expr "(builtins.fetchGit { url = \"file://$subRepo\"; rev = \"$subRev\"; }).outPath") +noSubmoduleRepo=$(nix eval --raw --expr "(builtins.fetchGit { url = \"file://$subRepo\"; rev = \"$subRev\"; submodules = true; }).outPath") [[ $noSubmoduleRepoBaseline == "$noSubmoduleRepo" ]] @@ -114,7 +116,7 @@ git -C "$rootRepo" commit -a -m "Add bad submodules" rev=$(git -C "$rootRepo" rev-parse HEAD) -r=$(nix eval --raw --expr "builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = true; }") +r=$(nix eval --raw --expr "builtins.fetchGit { url = \"file://$rootRepo\"; rev = \"$rev\"; submodules = true; }") [[ -f $r/file ]] [[ ! -e $r/missing ]] @@ -126,14 +128,14 @@ initGitRepo "$rootRepo" git -C "$rootRepo" submodule add ../gitSubmodulesSub sub git -C "$rootRepo" commit -m "Add submodule" rev2=$(git -C "$rootRepo" rev-parse HEAD) -pathWithRelative=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev2\"; submodules = true; }).outPath") +pathWithRelative=$(nix eval --raw --expr "(builtins.fetchGit { url = \"file://$rootRepo\"; rev = \"$rev2\"; submodules = true; }).outPath") diff -r -x .gitmodules "$pathWithSubmodules" "$pathWithRelative" # Test clones that have an upstream with relative submodule URLs. rm "$TEST_HOME"/.cache/nix/fetcher-cache* cloneRepo=$TEST_ROOT/a/b/gitSubmodulesClone # NB /a/b to make the relative path not work relative to $cloneRepo git clone "$rootRepo" "$cloneRepo" -pathIndirect=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$cloneRepo; rev = \"$rev2\"; submodules = true; }).outPath") +pathIndirect=$(nix eval --raw --expr "(builtins.fetchGit { url = \"file://$cloneRepo\"; rev = \"$rev2\"; submodules = true; }).outPath") [[ $pathIndirect = "$pathWithRelative" ]] # Test submodule export-ignore interaction @@ -161,7 +163,7 @@ git -C "$rootRepo" status # # TBD: not supported yet, because semantics are undecided and current implementation leaks rules from the root to submodules # # exportIgnore can be used with submodules -# pathWithExportIgnore=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = file://$rootRepo; submodules = true; exportIgnore = true; }).outPath") +# pathWithExportIgnore=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = \"file://$rootRepo\"; submodules = true; exportIgnore = true; }).outPath") # # find $pathWithExportIgnore # # git -C $rootRepo archive --format=tar HEAD | tar -t # # cp -a $rootRepo /tmp/rootRepo @@ -176,14 +178,14 @@ git -C "$rootRepo" status # exportIgnore can be explicitly disabled with submodules -pathWithoutExportIgnore=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = file://$rootRepo; submodules = true; exportIgnore = false; }).outPath") +pathWithoutExportIgnore=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = \"file://$rootRepo\"; submodules = true; exportIgnore = false; }).outPath") # find $pathWithoutExportIgnore [[ -e $pathWithoutExportIgnore/exclude-from-root ]] [[ -e $pathWithoutExportIgnore/sub/exclude-from-sub ]] # exportIgnore defaults to false when submodules = true -pathWithSubmodules=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = file://$rootRepo; submodules = true; }).outPath") +pathWithSubmodules=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = \"file://$rootRepo\"; submodules = true; }).outPath") [[ -e $pathWithoutExportIgnore/exclude-from-root ]] [[ -e $pathWithoutExportIgnore/sub/exclude-from-sub ]] diff --git a/tests/functional/fetchMercurial.sh b/tests/functional/fetchMercurial.sh index 6293fb76ac2f..c965936002b3 100755 --- a/tests/functional/fetchMercurial.sh +++ b/tests/functional/fetchMercurial.sh @@ -34,42 +34,42 @@ rev2=$(hg log --cwd "$repo" -r tip --template '{node}') # Fetch an unclean branch. echo unclean > "$repo"/hello -path=$(nix eval --impure --raw --expr "(builtins.fetchMercurial file://$repo).outPath") +path=$(nix eval --impure --raw --expr "(builtins.fetchMercurial \"file://$repo\").outPath") [[ $(cat "$path"/hello) = unclean ]] hg revert --cwd "$repo" --all # Fetch the default branch. -path=$(nix eval --impure --raw --expr "(builtins.fetchMercurial file://$repo).outPath") +path=$(nix eval --impure --raw --expr "(builtins.fetchMercurial \"file://$repo\").outPath") [[ $(cat "$path"/hello) = world ]] # In pure eval mode, fetchGit without a revision should fail. -[[ $(nix eval --impure --raw --expr "(builtins.readFile (fetchMercurial file://$repo + \"/hello\"))") = world ]] -(! nix eval --raw --expr "builtins.readFile (fetchMercurial file://$repo + \"/hello\")") +[[ $(nix eval --impure --raw --expr "(builtins.readFile (fetchMercurial \"file://$repo\" + \"/hello\"))") = world ]] +(! nix eval --raw --expr "builtins.readFile (fetchMercurial \"file://$repo\" + \"/hello\")") # Fetch using an explicit revision hash. -path2=$(nix eval --impure --raw --expr "(builtins.fetchMercurial { url = file://$repo; rev = \"$rev2\"; }).outPath") +path2=$(nix eval --impure --raw --expr "(builtins.fetchMercurial { url = \"file://$repo\"; rev = \"$rev2\"; }).outPath") [[ $path = "$path2" ]] # In pure eval mode, fetchGit with a revision should succeed. -[[ $(nix eval --raw --expr "builtins.readFile (fetchMercurial { url = file://$repo; rev = \"$rev2\"; } + \"/hello\")") = world ]] +[[ $(nix eval --raw --expr "builtins.readFile (fetchMercurial { url = \"file://$repo\"; rev = \"$rev2\"; } + \"/hello\")") = world ]] # Fetch again. This should be cached. mv "$repo" "${repo}"-tmp -path2=$(nix eval --impure --raw --expr "(builtins.fetchMercurial file://$repo).outPath") +path2=$(nix eval --impure --raw --expr "(builtins.fetchMercurial \"file://$repo\").outPath") [[ $path = "$path2" ]] -[[ $(nix eval --impure --raw --expr "(builtins.fetchMercurial file://$repo).branch") = default ]] -[[ $(nix eval --impure --expr "(builtins.fetchMercurial file://$repo).revCount") = 1 ]] -[[ $(nix eval --impure --raw --expr "(builtins.fetchMercurial file://$repo).rev") = "$rev2" ]] +[[ $(nix eval --impure --raw --expr "(builtins.fetchMercurial \"file://$repo\").branch") = default ]] +[[ $(nix eval --impure --expr "(builtins.fetchMercurial \"file://$repo\").revCount") = 1 ]] +[[ $(nix eval --impure --raw --expr "(builtins.fetchMercurial \"file://$repo\").rev") = "$rev2" ]] # But with TTL 0, it should fail. -(! nix eval --impure --refresh --expr "builtins.fetchMercurial file://$repo") +(! nix eval --impure --refresh --expr "builtins.fetchMercurial \"file://$repo\"") # Fetching with a explicit hash should succeed. -path2=$(nix eval --refresh --raw --expr "(builtins.fetchMercurial { url = file://$repo; rev = \"$rev2\"; }).outPath") +path2=$(nix eval --refresh --raw --expr "(builtins.fetchMercurial { url = \"file://$repo\"; rev = \"$rev2\"; }).outPath") [[ $path = "$path2" ]] -path2=$(nix eval --refresh --raw --expr "(builtins.fetchMercurial { url = file://$repo; rev = \"$rev1\"; }).outPath") +path2=$(nix eval --refresh --raw --expr "(builtins.fetchMercurial { url = \"file://$repo\"; rev = \"$rev1\"; }).outPath") [[ $(cat "$path2"/hello) = utrecht ]] mv "${repo}"-tmp "$repo" @@ -102,7 +102,7 @@ path3=$(nix eval --impure --raw --expr "(builtins.fetchMercurial { url = $repo; # Committing should not affect the store path. hg commit --cwd "$repo" -m 'Bla3' -path4=$(nix eval --impure --refresh --raw --expr "(builtins.fetchMercurial file://$repo).outPath") +path4=$(nix eval --impure --refresh --raw --expr "(builtins.fetchMercurial \"file://$repo\").outPath") [[ $path2 = "$path4" ]] echo paris > "$repo"/hello diff --git a/tests/functional/flakes/circular.sh b/tests/functional/flakes/circular.sh index 5304496ba574..0ee111d641a0 100755 --- a/tests/functional/flakes/circular.sh +++ b/tests/functional/flakes/circular.sh @@ -13,7 +13,7 @@ createGitRepo "$flakeB" cat > "$flakeA"/flake.nix < "$flakeB"/flake.nix < "$flake3Dir/flake.nix" < "$flake3Dir/flake.nix" < "$flakeDir/a" createGitRepo "$flakeDir" echo b > "$flakeDir/a" pushd "$flakeDir" -(! nix flake init --template "git+file://$templatesDir") |& grep "refusing to overwrite existing file '$flakeDir/a'" +(! nix flake init --template "git+file://$templatesDir") |& grep "refusing to overwrite existing file \"$flakeDir/a\"" popd git -C "$flakeDir" commit -a -m 'Changed' diff --git a/tests/functional/flakes/non-flake-inputs.sh b/tests/functional/flakes/non-flake-inputs.sh index 6b1c6a941069..fb7ebd7be23e 100644 --- a/tests/functional/flakes/non-flake-inputs.sh +++ b/tests/functional/flakes/non-flake-inputs.sh @@ -26,11 +26,11 @@ cat > "$flake3Dir/flake.nix" < $out/program < \$TEST_ROOT/fifo - sleep 10000 - EOF + open = mkDerivation { + name = "gc-runtime-open"; + buildCommand = "mkdir $out; echo open > $out/open"; + }; - chmod +x $out/program - ''; + program = mkDerivation { + name = "gc-runtime-program"; + builder = + # Test inline source file definitions. + builtins.toFile "builder.sh" '' + mkdir $out + + cat > $out/program << 'EOF' + #! ${shell} + echo x > "$2" + sleep 10000 < "$1" + EOF + + chmod +x $out/program + ''; + }; } diff --git a/tests/functional/gc-runtime.sh b/tests/functional/gc-runtime.sh index 34e99415d5c2..9b675cb97329 100755 --- a/tests/functional/gc-runtime.sh +++ b/tests/functional/gc-runtime.sh @@ -9,39 +9,43 @@ case $system in skipTest "Not running Linux"; esac -set -m # enable job control, needed for kill - TODO_NixOS -profiles="$NIX_STATE_DIR"/profiles -rm -rf "$profiles" - -nix-env -p "$profiles/test" -f ./gc-runtime.nix -i gc-runtime +set -m # enable job control, needed for kill -outPath=$(nix-env -p "$profiles/test" -q --no-name --out-path gc-runtime) -echo "$outPath" +programPath=$(nix-build --no-link ./gc-runtime.nix -A program) +environPath=$(nix-build --no-link ./gc-runtime.nix -A environ) +openPath=$(nix-build --no-link ./gc-runtime.nix -A open) fifo="$TEST_ROOT/fifo" mkfifo "$fifo" echo "backgrounding program..." -"$profiles"/test/program "$fifo" & +export environPath +"$programPath"/program "$openPath"/open "$fifo" & child=$! echo PID=$child cat "$fifo" -expectStderr 1 nix-store --delete "$outPath" | grepQuiet "Cannot delete path.*because it's referenced by the GC root '/proc/" - -nix-env -p "$profiles/test" -e gc-runtime -nix-env -p "$profiles/test" --delete-generations old +expectStderr 1 nix-store --delete "$openPath" | grepQuiet "Cannot delete path.*because it's referenced by the GC root '/proc/" nix-store --gc kill -- -$child -if ! test -e "$outPath"; then +if ! test -e "$programPath"; then echo "running program was garbage collected!" exit 1 fi +if ! test -e "$environPath"; then + echo "file in environment variable was garbage collected!" + exit 1 +fi + +if ! test -e "$openPath"; then + echo "opened file was garbage collected!" + exit 1 +fi + exit 0 diff --git a/tests/functional/git/packed-refs-no-cache.sh b/tests/functional/git/packed-refs-no-cache.sh index 54e0ab901cee..74ff9da67f52 100644 --- a/tests/functional/git/packed-refs-no-cache.sh +++ b/tests/functional/git/packed-refs-no-cache.sh @@ -34,10 +34,10 @@ git -C "$repo" add hello_world git -C "$repo" commit -m 'My first commit.' # We now do an eval -nix eval --impure --raw --expr "builtins.fetchGit { url = file://$repo; }" +nix eval --impure --raw --expr "builtins.fetchGit { url = \"file://$repo\"; }" # test that our eval even worked by checking for the presence of the file -[[ $(nix eval --impure --raw --expr "builtins.readFile ((builtins.fetchGit { url = file://$repo; }) + \"/hello_world\")") = 'hello world' ]] +[[ $(nix eval --impure --raw --expr "builtins.readFile ((builtins.fetchGit { url = \"file://$repo\"; }) + \"/hello_world\")") = 'hello world' ]] # Validate that refs/heads/master exists shopt -s nullglob @@ -67,7 +67,7 @@ git -C "$repo" add hello_again git -C "$repo" commit -m 'Second commit.' # re-eval — this should return the path to the cached version -store_path=$(nix eval --tarball-ttl 3600 --impure --raw --expr "(builtins.fetchGit { url = file://$repo; }).outPath") +store_path=$(nix eval --tarball-ttl 3600 --impure --raw --expr "(builtins.fetchGit { url = \"file://$repo\"; }).outPath") echo "Fetched store path: $store_path" # Validate that the new file is *not* there diff --git a/tests/functional/impure-derivations.sh b/tests/functional/impure-derivations.sh index 89392ce30712..2c36236044d0 100755 --- a/tests/functional/impure-derivations.sh +++ b/tests/functional/impure-derivations.sh @@ -69,5 +69,5 @@ path6=$(nix build -L --no-link --json --file ./impure-derivations.nix inputAddre [[ $(< "$TEST_ROOT"/counter) = 5 ]] # Test nix/fetchurl.nix. -path7=$(nix build -L --no-link --print-out-paths --expr "import { impure = true; url = file://$PWD/impure-derivations.sh; }") +path7=$(nix build -L --no-link --print-out-paths --expr "import { impure = true; url = \"file://$PWD/impure-derivations.sh\"; }") cmp "$path7" "$PWD"/impure-derivations.sh diff --git a/tests/functional/lang/eval-fail-abs-path-fatal.err.exp b/tests/functional/lang/eval-fail-abs-path-fatal.err.exp new file mode 100644 index 000000000000..698af03909a2 --- /dev/null +++ b/tests/functional/lang/eval-fail-abs-path-fatal.err.exp @@ -0,0 +1,5 @@ +error: absolute path literals are not portable. Consider replacing path literal '/tmp/foo' by a string, relative path, or parameter (lint-absolute-path-literals) + at /pwd/lang/eval-fail-abs-path-fatal.nix:1:1: + 1| /tmp/foo + | ^ + 2| diff --git a/tests/functional/lang/eval-fail-abs-path-fatal.flags b/tests/functional/lang/eval-fail-abs-path-fatal.flags new file mode 100644 index 000000000000..1615019e589c --- /dev/null +++ b/tests/functional/lang/eval-fail-abs-path-fatal.flags @@ -0,0 +1 @@ +--lint-absolute-path-literals fatal diff --git a/tests/functional/lang/eval-fail-abs-path-fatal.nix b/tests/functional/lang/eval-fail-abs-path-fatal.nix new file mode 100644 index 000000000000..7ddb87177c45 --- /dev/null +++ b/tests/functional/lang/eval-fail-abs-path-fatal.nix @@ -0,0 +1 @@ +/tmp/foo diff --git a/tests/functional/lang/eval-fail-blackhole.err.exp b/tests/functional/lang/eval-fail-blackhole.err.exp index d11eb338f9a6..6866c58dd414 100644 --- a/tests/functional/lang/eval-fail-blackhole.err.exp +++ b/tests/functional/lang/eval-fail-blackhole.err.exp @@ -7,8 +7,8 @@ error: 3| x = y; error: infinite recursion encountered - at /pwd/lang/eval-fail-blackhole.nix:2:3: + at /pwd/lang/eval-fail-blackhole.nix:2:10: 1| let { 2| body = x; - | ^ + | ^ 3| x = y; diff --git a/tests/functional/lang/eval-fail-fetchTree-relative-path.err.exp b/tests/functional/lang/eval-fail-fetchTree-relative-path.err.exp new file mode 100644 index 000000000000..863f143d19a5 --- /dev/null +++ b/tests/functional/lang/eval-fail-fetchTree-relative-path.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'fetchTree' builtin + at /pwd/lang/eval-fail-fetchTree-relative-path.nix:1:1: + 1| builtins.fetchTree "file:./relative-path.tar.gz" + | ^ + 2| + + … while fetching the input 'file:./relative-path.tar.gz' + + error: tarball 'file:./relative-path.tar.gz' must use an absolute path. The 'file' scheme does not support relative paths. diff --git a/tests/functional/lang/eval-fail-fetchTree-relative-path.nix b/tests/functional/lang/eval-fail-fetchTree-relative-path.nix new file mode 100644 index 000000000000..38826758134b --- /dev/null +++ b/tests/functional/lang/eval-fail-fetchTree-relative-path.nix @@ -0,0 +1 @@ +builtins.fetchTree "file:./relative-path.tar.gz" diff --git a/tests/functional/lang/eval-fail-home-path-fatal.err.exp b/tests/functional/lang/eval-fail-home-path-fatal.err.exp new file mode 100644 index 000000000000..c4f3ea3b8632 --- /dev/null +++ b/tests/functional/lang/eval-fail-home-path-fatal.err.exp @@ -0,0 +1,5 @@ +error: home path literals are not portable. Consider replacing path literal '~/foo' by a string, relative path, or parameter (lint-absolute-path-literals) + at /pwd/lang/eval-fail-home-path-fatal.nix:1:1: + 1| ~/foo + | ^ + 2| diff --git a/tests/functional/lang/eval-fail-home-path-fatal.flags b/tests/functional/lang/eval-fail-home-path-fatal.flags new file mode 100644 index 000000000000..a78e3b680857 --- /dev/null +++ b/tests/functional/lang/eval-fail-home-path-fatal.flags @@ -0,0 +1 @@ +--impure --lint-absolute-path-literals fatal diff --git a/tests/functional/lang/eval-fail-home-path-fatal.nix b/tests/functional/lang/eval-fail-home-path-fatal.nix new file mode 100644 index 000000000000..827774311d93 --- /dev/null +++ b/tests/functional/lang/eval-fail-home-path-fatal.nix @@ -0,0 +1 @@ +~/foo diff --git a/tests/functional/lang/eval-fail-memoised-error-trace-not-mutated.err.exp b/tests/functional/lang/eval-fail-memoised-error-trace-not-mutated.err.exp new file mode 100644 index 000000000000..9327371bf228 --- /dev/null +++ b/tests/functional/lang/eval-fail-memoised-error-trace-not-mutated.err.exp @@ -0,0 +1,29 @@ +error: + … while calling the 'seq' builtin + at /pwd/lang/eval-fail-memoised-error-trace-not-mutated.nix:10:1: + 9| # a fresh instance of the exceptions to avoid trace mutation. + 10| builtins.seq (builtins.tryEval b) (builtins.seq (builtins.tryEval c) d) + | ^ + 11| + + … while calling the 'seq' builtin + at /pwd/lang/eval-fail-memoised-error-trace-not-mutated.nix:10:36: + 9| # a fresh instance of the exceptions to avoid trace mutation. + 10| builtins.seq (builtins.tryEval b) (builtins.seq (builtins.tryEval c) d) + | ^ + 11| + + … forcing d + + … forcing c + + … forcing b + + … while calling the 'throw' builtin + at /pwd/lang/eval-fail-memoised-error-trace-not-mutated.nix:2:7: + 1| let + 2| a = throw "nope"; + | ^ + 3| b = builtins.addErrorContext "forcing b" a; + + error: nope diff --git a/tests/functional/lang/eval-fail-memoised-error-trace-not-mutated.nix b/tests/functional/lang/eval-fail-memoised-error-trace-not-mutated.nix new file mode 100644 index 000000000000..03db62843a3d --- /dev/null +++ b/tests/functional/lang/eval-fail-memoised-error-trace-not-mutated.nix @@ -0,0 +1,10 @@ +let + a = throw "nope"; + b = builtins.addErrorContext "forcing b" a; + c = builtins.addErrorContext "forcing c" a; + d = builtins.addErrorContext "forcing d" a; +in +# Since nix 2.34 errors are memoised. Trying to eval a failed thunk includes +# the trace from when it was first forced. When forcing a failed value it gets +# a fresh instance of the exceptions to avoid trace mutation. +builtins.seq (builtins.tryEval b) (builtins.seq (builtins.tryEval c) d) diff --git a/tests/functional/lang/eval-fail-scope-5.err.exp b/tests/functional/lang/eval-fail-scope-5.err.exp index 557054b53549..20027a7f1f57 100644 --- a/tests/functional/lang/eval-fail-scope-5.err.exp +++ b/tests/functional/lang/eval-fail-scope-5.err.exp @@ -21,8 +21,8 @@ error: 8| x ? y, error: infinite recursion encountered - at /pwd/lang/eval-fail-scope-5.nix:13:3: + at /pwd/lang/eval-fail-scope-5.nix:11:5: + 10| }: + 11| x + y; + | ^ 12| - 13| body = f { }; - | ^ - 14| diff --git a/tests/functional/lang/eval-fail-short-path-literal.err.exp b/tests/functional/lang/eval-fail-short-path-literal.err.exp new file mode 100644 index 000000000000..447c29db0f7f --- /dev/null +++ b/tests/functional/lang/eval-fail-short-path-literal.err.exp @@ -0,0 +1,5 @@ +error: relative path literal 'test/subdir' should be prefixed with '.' for clarity: './test/subdir' (lint-short-path-literals) + at /pwd/lang/eval-fail-short-path-literal.nix:1:1: + 1| test/subdir + | ^ + 2| diff --git a/tests/functional/lang/eval-fail-short-path-literal.flags b/tests/functional/lang/eval-fail-short-path-literal.flags new file mode 100644 index 000000000000..674d96dbeb7c --- /dev/null +++ b/tests/functional/lang/eval-fail-short-path-literal.flags @@ -0,0 +1 @@ +--lint-short-path-literals fatal diff --git a/tests/functional/lang/eval-fail-short-path-literal.nix b/tests/functional/lang/eval-fail-short-path-literal.nix new file mode 100644 index 000000000000..481e220c4d7a --- /dev/null +++ b/tests/functional/lang/eval-fail-short-path-literal.nix @@ -0,0 +1 @@ +test/subdir diff --git a/tests/functional/lang/eval-fail-url-literal.err.exp b/tests/functional/lang/eval-fail-url-literal.err.exp new file mode 100644 index 000000000000..1e7fd0db7ec2 --- /dev/null +++ b/tests/functional/lang/eval-fail-url-literal.err.exp @@ -0,0 +1,5 @@ +error: URL literals are disallowed. Consider using a string literal "http://example.com" instead (lint-url-literals) + at /pwd/lang/eval-fail-url-literal.nix:1:1: + 1| http://example.com + | ^ + 2| diff --git a/tests/functional/lang/eval-fail-url-literal.flags b/tests/functional/lang/eval-fail-url-literal.flags new file mode 100644 index 000000000000..518207573ad3 --- /dev/null +++ b/tests/functional/lang/eval-fail-url-literal.flags @@ -0,0 +1 @@ +--lint-url-literals fatal diff --git a/tests/functional/lang/eval-fail-url-literal.nix b/tests/functional/lang/eval-fail-url-literal.nix new file mode 100644 index 000000000000..7bc94d03292f --- /dev/null +++ b/tests/functional/lang/eval-fail-url-literal.nix @@ -0,0 +1 @@ +http://example.com diff --git a/tests/functional/lang/eval-okay-abs-path-default.exp b/tests/functional/lang/eval-okay-abs-path-default.exp new file mode 100644 index 000000000000..7ddb87177c45 --- /dev/null +++ b/tests/functional/lang/eval-okay-abs-path-default.exp @@ -0,0 +1 @@ +/tmp/foo diff --git a/tests/functional/lang/eval-okay-abs-path-default.nix b/tests/functional/lang/eval-okay-abs-path-default.nix new file mode 100644 index 000000000000..3727c0af4e16 --- /dev/null +++ b/tests/functional/lang/eval-okay-abs-path-default.nix @@ -0,0 +1,2 @@ +# Test: By default, absolute path literals are allowed +/tmp/foo diff --git a/tests/functional/lang/eval-okay-abs-path-warn.err.exp b/tests/functional/lang/eval-okay-abs-path-warn.err.exp new file mode 100644 index 000000000000..1b93c9798fd9 --- /dev/null +++ b/tests/functional/lang/eval-okay-abs-path-warn.err.exp @@ -0,0 +1,5 @@ +warning: absolute path literals are not portable. Consider replacing path literal '/tmp/foo' by a string, relative path, or parameter (lint-absolute-path-literals) + at /pwd/lang/eval-okay-abs-path-warn.nix:1:1: + 1| /tmp/foo + | ^ + 2| diff --git a/tests/functional/lang/eval-okay-abs-path-warn.exp b/tests/functional/lang/eval-okay-abs-path-warn.exp new file mode 100644 index 000000000000..7ddb87177c45 --- /dev/null +++ b/tests/functional/lang/eval-okay-abs-path-warn.exp @@ -0,0 +1 @@ +/tmp/foo diff --git a/tests/functional/lang/eval-okay-abs-path-warn.flags b/tests/functional/lang/eval-okay-abs-path-warn.flags new file mode 100644 index 000000000000..93b76fc26a1b --- /dev/null +++ b/tests/functional/lang/eval-okay-abs-path-warn.flags @@ -0,0 +1 @@ +--lint-absolute-path-literals warn diff --git a/tests/functional/lang/eval-okay-abs-path-warn.nix b/tests/functional/lang/eval-okay-abs-path-warn.nix new file mode 100644 index 000000000000..7ddb87177c45 --- /dev/null +++ b/tests/functional/lang/eval-okay-abs-path-warn.nix @@ -0,0 +1 @@ +/tmp/foo diff --git a/tests/functional/lang/eval-okay-dotdotslash-abs-fatal.exp b/tests/functional/lang/eval-okay-dotdotslash-abs-fatal.exp new file mode 100644 index 000000000000..27ba77ddaf61 --- /dev/null +++ b/tests/functional/lang/eval-okay-dotdotslash-abs-fatal.exp @@ -0,0 +1 @@ +true diff --git a/tests/functional/lang/eval-okay-dotdotslash-abs-fatal.flags b/tests/functional/lang/eval-okay-dotdotslash-abs-fatal.flags new file mode 100644 index 000000000000..1615019e589c --- /dev/null +++ b/tests/functional/lang/eval-okay-dotdotslash-abs-fatal.flags @@ -0,0 +1 @@ +--lint-absolute-path-literals fatal diff --git a/tests/functional/lang/eval-okay-dotdotslash-abs-fatal.nix b/tests/functional/lang/eval-okay-dotdotslash-abs-fatal.nix new file mode 100644 index 000000000000..5c01768f9e93 --- /dev/null +++ b/tests/functional/lang/eval-okay-dotdotslash-abs-fatal.nix @@ -0,0 +1,2 @@ +# Test: ../ relative paths work even when absolute paths are fatal +builtins.isPath ../lang/lang.nix diff --git a/tests/functional/lang/eval-okay-dotdotslash-path-fatal.exp b/tests/functional/lang/eval-okay-dotdotslash-path-fatal.exp new file mode 100644 index 000000000000..6cdb45ac7afb --- /dev/null +++ b/tests/functional/lang/eval-okay-dotdotslash-path-fatal.exp @@ -0,0 +1 @@ +/pwd/test/subdir diff --git a/tests/functional/lang/eval-okay-dotdotslash-path-fatal.flags b/tests/functional/lang/eval-okay-dotdotslash-path-fatal.flags new file mode 100644 index 000000000000..674d96dbeb7c --- /dev/null +++ b/tests/functional/lang/eval-okay-dotdotslash-path-fatal.flags @@ -0,0 +1 @@ +--lint-short-path-literals fatal diff --git a/tests/functional/lang/eval-okay-dotdotslash-path-fatal.nix b/tests/functional/lang/eval-okay-dotdotslash-path-fatal.nix new file mode 100644 index 000000000000..d29a8981c3f9 --- /dev/null +++ b/tests/functional/lang/eval-okay-dotdotslash-path-fatal.nix @@ -0,0 +1,2 @@ +# Test: ../ paths are not affected by lint-short-path-literals=fatal +../test/subdir diff --git a/tests/functional/lang/eval-okay-dotslash-abs-fatal.exp b/tests/functional/lang/eval-okay-dotslash-abs-fatal.exp new file mode 100644 index 000000000000..27ba77ddaf61 --- /dev/null +++ b/tests/functional/lang/eval-okay-dotslash-abs-fatal.exp @@ -0,0 +1 @@ +true diff --git a/tests/functional/lang/eval-okay-dotslash-abs-fatal.flags b/tests/functional/lang/eval-okay-dotslash-abs-fatal.flags new file mode 100644 index 000000000000..1615019e589c --- /dev/null +++ b/tests/functional/lang/eval-okay-dotslash-abs-fatal.flags @@ -0,0 +1 @@ +--lint-absolute-path-literals fatal diff --git a/tests/functional/lang/eval-okay-dotslash-abs-fatal.nix b/tests/functional/lang/eval-okay-dotslash-abs-fatal.nix new file mode 100644 index 000000000000..dda55fefd078 --- /dev/null +++ b/tests/functional/lang/eval-okay-dotslash-abs-fatal.nix @@ -0,0 +1,2 @@ +# Test: ./ relative paths work even when absolute paths are fatal +builtins.isPath ./lang.nix diff --git a/tests/functional/lang/eval-okay-dotslash-path-fatal.exp b/tests/functional/lang/eval-okay-dotslash-path-fatal.exp new file mode 100644 index 000000000000..c2926da802c3 --- /dev/null +++ b/tests/functional/lang/eval-okay-dotslash-path-fatal.exp @@ -0,0 +1 @@ +/pwd/lang/test/subdir diff --git a/tests/functional/lang/eval-okay-dotslash-path-fatal.flags b/tests/functional/lang/eval-okay-dotslash-path-fatal.flags new file mode 100644 index 000000000000..674d96dbeb7c --- /dev/null +++ b/tests/functional/lang/eval-okay-dotslash-path-fatal.flags @@ -0,0 +1 @@ +--lint-short-path-literals fatal diff --git a/tests/functional/lang/eval-okay-dotslash-path-fatal.nix b/tests/functional/lang/eval-okay-dotslash-path-fatal.nix new file mode 100644 index 000000000000..500b32375b9a --- /dev/null +++ b/tests/functional/lang/eval-okay-dotslash-path-fatal.nix @@ -0,0 +1,2 @@ +# Test: ./ paths are not affected by lint-short-path-literals=fatal +./test/subdir diff --git a/tests/functional/lang/eval-okay-home-path-warn.err.exp b/tests/functional/lang/eval-okay-home-path-warn.err.exp new file mode 100644 index 000000000000..b7a3a88cdf61 --- /dev/null +++ b/tests/functional/lang/eval-okay-home-path-warn.err.exp @@ -0,0 +1,5 @@ +warning: home path literals are not portable. Consider replacing path literal '~/foo' by a string, relative path, or parameter (lint-absolute-path-literals) + at /pwd/lang/eval-okay-home-path-warn.nix:1:17: + 1| builtins.isPath ~/foo + | ^ + 2| diff --git a/tests/functional/lang/eval-okay-home-path-warn.exp b/tests/functional/lang/eval-okay-home-path-warn.exp new file mode 100644 index 000000000000..27ba77ddaf61 --- /dev/null +++ b/tests/functional/lang/eval-okay-home-path-warn.exp @@ -0,0 +1 @@ +true diff --git a/tests/functional/lang/eval-okay-home-path-warn.flags b/tests/functional/lang/eval-okay-home-path-warn.flags new file mode 100644 index 000000000000..7be81df22710 --- /dev/null +++ b/tests/functional/lang/eval-okay-home-path-warn.flags @@ -0,0 +1 @@ +--impure --lint-absolute-path-literals warn diff --git a/tests/functional/lang/eval-okay-home-path-warn.nix b/tests/functional/lang/eval-okay-home-path-warn.nix new file mode 100644 index 000000000000..42f317720ca5 --- /dev/null +++ b/tests/functional/lang/eval-okay-home-path-warn.nix @@ -0,0 +1 @@ +builtins.isPath ~/foo diff --git a/tests/functional/lang/eval-okay-path.exp b/tests/functional/lang/eval-okay-path.exp index 635e2243a2ab..2698f2f18ce9 100644 --- a/tests/functional/lang/eval-okay-path.exp +++ b/tests/functional/lang/eval-okay-path.exp @@ -1 +1 @@ -[ "/nix/store/ya937r4ydw0l6kayq8jkyqaips9c75jm-output" "/nix/store/m7y372g6jb0g4hh1dzmj847rd356fhnz-output" ] +[ "/nix/store/ya937r4ydw0l6kayq8jkyqaips9c75jm-output" "/nix/store/m7y372g6jb0g4hh1dzmj847rd356fhnz-output" "/nix/store/a517xfygy9w2q5i3c2dbm50sw4p70b4c-output" ] diff --git a/tests/functional/lang/eval-okay-path.nix b/tests/functional/lang/eval-okay-path.nix index b8b48aae1a61..64b5486b616b 100644 --- a/tests/functional/lang/eval-okay-path.nix +++ b/tests/functional/lang/eval-okay-path.nix @@ -1,4 +1,5 @@ [ + # NAR hash of directory with filter (builtins.path { path = ./.; filter = path: _: baseNameOf path == "data"; @@ -6,10 +7,17 @@ sha256 = "1yhm3gwvg5a41yylymgblsclk95fs6jy72w0wv925mmidlhcq4sw"; name = "output"; }) + # Flat hash of file (builtins.path { path = ./data; recursive = false; sha256 = "0k4lwj58f2w5yh92ilrwy9917pycipbrdrr13vbb3yd02j09vfxm"; name = "output"; }) + # NAR hash of directory (recursive = true is the default) + (builtins.path { + path = ./dir1; + sha256 = "02vlkcjkl1rvy081n6d40qi73biv2w4b9x9biklay4ncgk77zr1f"; + name = "output"; + }) ] diff --git a/tests/functional/lang/eval-okay-short-path-literal-warn.err.exp b/tests/functional/lang/eval-okay-short-path-literal-warn.err.exp new file mode 100644 index 000000000000..f6b986ac32e3 --- /dev/null +++ b/tests/functional/lang/eval-okay-short-path-literal-warn.err.exp @@ -0,0 +1,5 @@ +warning: relative path literal 'test/subdir' should be prefixed with '.' for clarity: './test/subdir' (lint-short-path-literals) + at /pwd/lang/eval-okay-short-path-literal-warn.nix:1:1: + 1| test/subdir + | ^ + 2| diff --git a/tests/functional/lang/eval-okay-short-path-literal-warn.exp b/tests/functional/lang/eval-okay-short-path-literal-warn.exp new file mode 100644 index 000000000000..c2926da802c3 --- /dev/null +++ b/tests/functional/lang/eval-okay-short-path-literal-warn.exp @@ -0,0 +1 @@ +/pwd/lang/test/subdir diff --git a/tests/functional/lang/eval-okay-short-path-literal-warn.flags b/tests/functional/lang/eval-okay-short-path-literal-warn.flags new file mode 100644 index 000000000000..e514f53eed7e --- /dev/null +++ b/tests/functional/lang/eval-okay-short-path-literal-warn.flags @@ -0,0 +1 @@ +--lint-short-path-literals warn diff --git a/tests/functional/lang/eval-okay-short-path-literal-warn.nix b/tests/functional/lang/eval-okay-short-path-literal-warn.nix new file mode 100644 index 000000000000..481e220c4d7a --- /dev/null +++ b/tests/functional/lang/eval-okay-short-path-literal-warn.nix @@ -0,0 +1 @@ +test/subdir diff --git a/tests/functional/lang/eval-okay-short-path-variation.err.exp b/tests/functional/lang/eval-okay-short-path-variation.err.exp new file mode 100644 index 000000000000..64f58bd62ff9 --- /dev/null +++ b/tests/functional/lang/eval-okay-short-path-variation.err.exp @@ -0,0 +1,12 @@ +warning: relative path literal 'foo/bar' should be prefixed with '.' for clarity: './foo/bar' (lint-short-path-literals) + at /pwd/lang/eval-okay-short-path-variation.nix:4:3: + 3| [ + 4| foo/bar + | ^ + 5| a/b/c/d +warning: relative path literal 'a/b/c/d' should be prefixed with '.' for clarity: './a/b/c/d' (lint-short-path-literals) + at /pwd/lang/eval-okay-short-path-variation.nix:5:3: + 4| foo/bar + 5| a/b/c/d + | ^ + 6| ] diff --git a/tests/functional/lang/eval-okay-short-path-variation.exp b/tests/functional/lang/eval-okay-short-path-variation.exp new file mode 100644 index 000000000000..45524cce3f6c --- /dev/null +++ b/tests/functional/lang/eval-okay-short-path-variation.exp @@ -0,0 +1 @@ +[ /pwd/lang/foo/bar /pwd/lang/a/b/c/d ] diff --git a/tests/functional/lang/eval-okay-short-path-variation.flags b/tests/functional/lang/eval-okay-short-path-variation.flags new file mode 100644 index 000000000000..e514f53eed7e --- /dev/null +++ b/tests/functional/lang/eval-okay-short-path-variation.flags @@ -0,0 +1 @@ +--lint-short-path-literals warn diff --git a/tests/functional/lang/eval-okay-short-path-variation.nix b/tests/functional/lang/eval-okay-short-path-variation.nix new file mode 100644 index 000000000000..173c10d9e502 --- /dev/null +++ b/tests/functional/lang/eval-okay-short-path-variation.nix @@ -0,0 +1,6 @@ +# Test: Different short path literals should produce warnings +# This tests variation in path patterns +[ + foo/bar + a/b/c/d +] diff --git a/tests/functional/lang/eval-okay-tryeval-failed-thunk-reeval.err.exp b/tests/functional/lang/eval-okay-tryeval-failed-thunk-reeval.err.exp new file mode 100644 index 000000000000..ba0da519dcbb --- /dev/null +++ b/tests/functional/lang/eval-okay-tryeval-failed-thunk-reeval.err.exp @@ -0,0 +1 @@ +trace: throwing diff --git a/tests/functional/lang/eval-okay-tryeval-failed-thunk-reeval.exp b/tests/functional/lang/eval-okay-tryeval-failed-thunk-reeval.exp new file mode 100644 index 000000000000..be54b4b4e39e --- /dev/null +++ b/tests/functional/lang/eval-okay-tryeval-failed-thunk-reeval.exp @@ -0,0 +1 @@ +"done" diff --git a/tests/functional/lang/eval-okay-tryeval-failed-thunk-reeval.nix b/tests/functional/lang/eval-okay-tryeval-failed-thunk-reeval.nix new file mode 100644 index 000000000000..212986199077 --- /dev/null +++ b/tests/functional/lang/eval-okay-tryeval-failed-thunk-reeval.nix @@ -0,0 +1,7 @@ +# Since Nix 2.34, errors are memoized +let + # This attribute value will only be evaluated once. + foo = builtins.trace "throwing" throw "nope"; +in +# Trigger and catch the error twice. +builtins.seq (builtins.tryEval foo).success builtins.seq (builtins.tryEval foo).success "done" diff --git a/tests/functional/lang/eval-okay-url-literal-default.exp b/tests/functional/lang/eval-okay-url-literal-default.exp new file mode 100644 index 000000000000..0573d0594288 --- /dev/null +++ b/tests/functional/lang/eval-okay-url-literal-default.exp @@ -0,0 +1 @@ +"http://example.com" diff --git a/tests/functional/lang/eval-okay-url-literal-default.nix b/tests/functional/lang/eval-okay-url-literal-default.nix new file mode 100644 index 000000000000..ad07bc804c33 --- /dev/null +++ b/tests/functional/lang/eval-okay-url-literal-default.nix @@ -0,0 +1,2 @@ +# Test: By default (no flags), unquoted URL literals are accepted +http://example.com diff --git a/tests/functional/lang/eval-okay-url-literal-quoted-fatal.exp b/tests/functional/lang/eval-okay-url-literal-quoted-fatal.exp new file mode 100644 index 000000000000..0573d0594288 --- /dev/null +++ b/tests/functional/lang/eval-okay-url-literal-quoted-fatal.exp @@ -0,0 +1 @@ +"http://example.com" diff --git a/tests/functional/lang/eval-okay-url-literal-quoted-fatal.flags b/tests/functional/lang/eval-okay-url-literal-quoted-fatal.flags new file mode 100644 index 000000000000..518207573ad3 --- /dev/null +++ b/tests/functional/lang/eval-okay-url-literal-quoted-fatal.flags @@ -0,0 +1 @@ +--lint-url-literals fatal diff --git a/tests/functional/lang/eval-okay-url-literal-quoted-fatal.nix b/tests/functional/lang/eval-okay-url-literal-quoted-fatal.nix new file mode 100644 index 000000000000..9b4f20755ec7 --- /dev/null +++ b/tests/functional/lang/eval-okay-url-literal-quoted-fatal.nix @@ -0,0 +1,2 @@ +# Test: Quoted URLs are always accepted even with fatal setting +"http://example.com" diff --git a/tests/functional/lang/eval-okay-url-literal-warn.err.exp b/tests/functional/lang/eval-okay-url-literal-warn.err.exp new file mode 100644 index 000000000000..f3636f403ee6 --- /dev/null +++ b/tests/functional/lang/eval-okay-url-literal-warn.err.exp @@ -0,0 +1,5 @@ +warning: URL literals are discouraged. Consider using a string literal "http://example.com" instead (lint-url-literals) + at /pwd/lang/eval-okay-url-literal-warn.nix:1:1: + 1| http://example.com + | ^ + 2| diff --git a/tests/functional/lang/eval-okay-url-literal-warn.exp b/tests/functional/lang/eval-okay-url-literal-warn.exp new file mode 100644 index 000000000000..0573d0594288 --- /dev/null +++ b/tests/functional/lang/eval-okay-url-literal-warn.exp @@ -0,0 +1 @@ +"http://example.com" diff --git a/tests/functional/lang/eval-okay-url-literal-warn.flags b/tests/functional/lang/eval-okay-url-literal-warn.flags new file mode 100644 index 000000000000..7ee85110e8ef --- /dev/null +++ b/tests/functional/lang/eval-okay-url-literal-warn.flags @@ -0,0 +1 @@ +--lint-url-literals warn diff --git a/tests/functional/lang/eval-okay-url-literal-warn.nix b/tests/functional/lang/eval-okay-url-literal-warn.nix new file mode 100644 index 000000000000..7bc94d03292f --- /dev/null +++ b/tests/functional/lang/eval-okay-url-literal-warn.nix @@ -0,0 +1 @@ +http://example.com diff --git a/tests/functional/lang/parse-fail-unexpected-and.err.exp b/tests/functional/lang/parse-fail-unexpected-and.err.exp new file mode 100644 index 000000000000..8c3b11241d03 --- /dev/null +++ b/tests/functional/lang/parse-fail-unexpected-and.err.exp @@ -0,0 +1,5 @@ +error: syntax error, unexpected '&&' + at «stdin»:1:1: + 1| && true + | ^ + 2| diff --git a/tests/functional/lang/parse-fail-unexpected-and.nix b/tests/functional/lang/parse-fail-unexpected-and.nix new file mode 100644 index 000000000000..92b7ef1c7dd0 --- /dev/null +++ b/tests/functional/lang/parse-fail-unexpected-and.nix @@ -0,0 +1 @@ +&& true diff --git a/tests/functional/lang/parse-fail-unexpected-assert.err.exp b/tests/functional/lang/parse-fail-unexpected-assert.err.exp new file mode 100644 index 000000000000..7f00dcd58638 --- /dev/null +++ b/tests/functional/lang/parse-fail-unexpected-assert.err.exp @@ -0,0 +1,5 @@ +error: syntax error, unexpected 'assert', expecting identifier or 'or' or '${' or '"' + at «stdin»:1:3: + 1| a.assert + | ^ + 2| diff --git a/tests/functional/lang/parse-fail-unexpected-assert.nix b/tests/functional/lang/parse-fail-unexpected-assert.nix new file mode 100644 index 000000000000..989a981111bc --- /dev/null +++ b/tests/functional/lang/parse-fail-unexpected-assert.nix @@ -0,0 +1 @@ +a.assert diff --git a/tests/functional/lang/parse-fail-unexpected-concat.err.exp b/tests/functional/lang/parse-fail-unexpected-concat.err.exp new file mode 100644 index 000000000000..e1845bde8ded --- /dev/null +++ b/tests/functional/lang/parse-fail-unexpected-concat.err.exp @@ -0,0 +1,5 @@ +error: syntax error, unexpected '++' + at «stdin»:1:1: + 1| ++ [] + | ^ + 2| diff --git a/tests/functional/lang/parse-fail-unexpected-concat.nix b/tests/functional/lang/parse-fail-unexpected-concat.nix new file mode 100644 index 000000000000..0e84d3295618 --- /dev/null +++ b/tests/functional/lang/parse-fail-unexpected-concat.nix @@ -0,0 +1 @@ +++ [] diff --git a/tests/functional/lang/parse-fail-unexpected-ellipsis.err.exp b/tests/functional/lang/parse-fail-unexpected-ellipsis.err.exp new file mode 100644 index 000000000000..8a2019298f26 --- /dev/null +++ b/tests/functional/lang/parse-fail-unexpected-ellipsis.err.exp @@ -0,0 +1,5 @@ +error: syntax error, unexpected '...', expecting '.' or '=' + at «stdin»:1:5: + 1| { a ... } + | ^ + 2| diff --git a/tests/functional/lang/parse-fail-unexpected-ellipsis.nix b/tests/functional/lang/parse-fail-unexpected-ellipsis.nix new file mode 100644 index 000000000000..ba04161cf3cf --- /dev/null +++ b/tests/functional/lang/parse-fail-unexpected-ellipsis.nix @@ -0,0 +1 @@ +{ a ... } diff --git a/tests/functional/lang/parse-fail-unexpected-else.err.exp b/tests/functional/lang/parse-fail-unexpected-else.err.exp new file mode 100644 index 000000000000..5a8f0b2a5e5d --- /dev/null +++ b/tests/functional/lang/parse-fail-unexpected-else.err.exp @@ -0,0 +1,5 @@ +error: syntax error, unexpected 'else', expecting 'then' + at «stdin»:1:6: + 1| if 1 else + | ^ + 2| diff --git a/tests/functional/lang/parse-fail-unexpected-else.nix b/tests/functional/lang/parse-fail-unexpected-else.nix new file mode 100644 index 000000000000..b9a805eb7367 --- /dev/null +++ b/tests/functional/lang/parse-fail-unexpected-else.nix @@ -0,0 +1 @@ +if 1 else diff --git a/tests/functional/lang/parse-fail-unexpected-eq.err.exp b/tests/functional/lang/parse-fail-unexpected-eq.err.exp new file mode 100644 index 000000000000..6fb9db002c66 --- /dev/null +++ b/tests/functional/lang/parse-fail-unexpected-eq.err.exp @@ -0,0 +1,5 @@ +error: syntax error, unexpected '==' + at «stdin»:1:1: + 1| == 1 + | ^ + 2| diff --git a/tests/functional/lang/parse-fail-unexpected-eq.nix b/tests/functional/lang/parse-fail-unexpected-eq.nix new file mode 100644 index 000000000000..2badb0f30db8 --- /dev/null +++ b/tests/functional/lang/parse-fail-unexpected-eq.nix @@ -0,0 +1 @@ +== 1 diff --git a/tests/functional/lang/parse-fail-unexpected-float.err.exp b/tests/functional/lang/parse-fail-unexpected-float.err.exp new file mode 100644 index 000000000000..6db7be9ab687 --- /dev/null +++ b/tests/functional/lang/parse-fail-unexpected-float.err.exp @@ -0,0 +1,5 @@ +error: syntax error, unexpected floating-point literal, expecting 'inherit' + at «stdin»:1:3: + 1| { 1.5 = x; } + | ^ + 2| diff --git a/tests/functional/lang/parse-fail-unexpected-float.nix b/tests/functional/lang/parse-fail-unexpected-float.nix new file mode 100644 index 000000000000..8b0189501448 --- /dev/null +++ b/tests/functional/lang/parse-fail-unexpected-float.nix @@ -0,0 +1 @@ +{ 1.5 = x; } diff --git a/tests/functional/lang/parse-fail-unexpected-geq.err.exp b/tests/functional/lang/parse-fail-unexpected-geq.err.exp new file mode 100644 index 000000000000..666f1e336d82 --- /dev/null +++ b/tests/functional/lang/parse-fail-unexpected-geq.err.exp @@ -0,0 +1,5 @@ +error: syntax error, unexpected '>=' + at «stdin»:1:1: + 1| >= 1 + | ^ + 2| diff --git a/tests/functional/lang/parse-fail-unexpected-geq.nix b/tests/functional/lang/parse-fail-unexpected-geq.nix new file mode 100644 index 000000000000..4eee0cc8826a --- /dev/null +++ b/tests/functional/lang/parse-fail-unexpected-geq.nix @@ -0,0 +1 @@ +>= 1 diff --git a/tests/functional/lang/parse-fail-unexpected-hpath.err.exp b/tests/functional/lang/parse-fail-unexpected-hpath.err.exp new file mode 100644 index 000000000000..15193927be06 --- /dev/null +++ b/tests/functional/lang/parse-fail-unexpected-hpath.err.exp @@ -0,0 +1,5 @@ +error: syntax error, unexpected '~/…' path, expecting identifier or 'or' or '${' or '"' + at «stdin»:1:3: + 1| a.~/foo + | ^ + 2| diff --git a/tests/functional/lang/parse-fail-unexpected-hpath.nix b/tests/functional/lang/parse-fail-unexpected-hpath.nix new file mode 100644 index 000000000000..b0422ef35d9b --- /dev/null +++ b/tests/functional/lang/parse-fail-unexpected-hpath.nix @@ -0,0 +1 @@ +a.~/foo diff --git a/tests/functional/lang/parse-fail-unexpected-if.err.exp b/tests/functional/lang/parse-fail-unexpected-if.err.exp new file mode 100644 index 000000000000..9941adbd2861 --- /dev/null +++ b/tests/functional/lang/parse-fail-unexpected-if.err.exp @@ -0,0 +1,5 @@ +error: syntax error, unexpected 'if', expecting identifier or 'or' or '${' or '"' + at «stdin»:1:3: + 1| a.if + | ^ + 2| diff --git a/tests/functional/lang/parse-fail-unexpected-if.nix b/tests/functional/lang/parse-fail-unexpected-if.nix new file mode 100644 index 000000000000..7dd8b96b8592 --- /dev/null +++ b/tests/functional/lang/parse-fail-unexpected-if.nix @@ -0,0 +1 @@ +a.if diff --git a/tests/functional/lang/parse-fail-unexpected-impl.err.exp b/tests/functional/lang/parse-fail-unexpected-impl.err.exp new file mode 100644 index 000000000000..36421578de1b --- /dev/null +++ b/tests/functional/lang/parse-fail-unexpected-impl.err.exp @@ -0,0 +1,5 @@ +error: syntax error, unexpected '->' + at «stdin»:1:1: + 1| -> x + | ^ + 2| diff --git a/tests/functional/lang/parse-fail-unexpected-impl.nix b/tests/functional/lang/parse-fail-unexpected-impl.nix new file mode 100644 index 000000000000..05217c763574 --- /dev/null +++ b/tests/functional/lang/parse-fail-unexpected-impl.nix @@ -0,0 +1 @@ +-> x diff --git a/tests/functional/lang/parse-fail-unexpected-in.err.exp b/tests/functional/lang/parse-fail-unexpected-in.err.exp new file mode 100644 index 000000000000..7557552d2fcc --- /dev/null +++ b/tests/functional/lang/parse-fail-unexpected-in.err.exp @@ -0,0 +1,5 @@ +error: syntax error, unexpected 'in', expecting identifier or 'or' or '${' or '"' + at «stdin»:1:3: + 1| a.in + | ^ + 2| diff --git a/tests/functional/lang/parse-fail-unexpected-in.nix b/tests/functional/lang/parse-fail-unexpected-in.nix new file mode 100644 index 000000000000..20e5c8ea3462 --- /dev/null +++ b/tests/functional/lang/parse-fail-unexpected-in.nix @@ -0,0 +1 @@ +a.in diff --git a/tests/functional/lang/parse-fail-unexpected-ind-string.err.exp b/tests/functional/lang/parse-fail-unexpected-ind-string.err.exp new file mode 100644 index 000000000000..2e348ae0d9c4 --- /dev/null +++ b/tests/functional/lang/parse-fail-unexpected-ind-string.err.exp @@ -0,0 +1,5 @@ +error: syntax error, unexpected start of an indented string, expecting identifier or 'or' or '${' or '"' + at «stdin»:1:3: + 1| a.'' + | ^ + 2| diff --git a/tests/functional/lang/parse-fail-unexpected-ind-string.nix b/tests/functional/lang/parse-fail-unexpected-ind-string.nix new file mode 100644 index 000000000000..667303fb2e73 --- /dev/null +++ b/tests/functional/lang/parse-fail-unexpected-ind-string.nix @@ -0,0 +1 @@ +a.'' diff --git a/tests/functional/lang/parse-fail-unexpected-inherit.err.exp b/tests/functional/lang/parse-fail-unexpected-inherit.err.exp new file mode 100644 index 000000000000..56389139a8a8 --- /dev/null +++ b/tests/functional/lang/parse-fail-unexpected-inherit.err.exp @@ -0,0 +1,5 @@ +error: syntax error, unexpected 'inherit', expecting identifier or 'or' or '${' or '"' + at «stdin»:1:3: + 1| a.inherit + | ^ + 2| diff --git a/tests/functional/lang/parse-fail-unexpected-inherit.nix b/tests/functional/lang/parse-fail-unexpected-inherit.nix new file mode 100644 index 000000000000..324e683c302e --- /dev/null +++ b/tests/functional/lang/parse-fail-unexpected-inherit.nix @@ -0,0 +1 @@ +a.inherit diff --git a/tests/functional/lang/parse-fail-unexpected-int.err.exp b/tests/functional/lang/parse-fail-unexpected-int.err.exp new file mode 100644 index 000000000000..d20d9cd8bb3d --- /dev/null +++ b/tests/functional/lang/parse-fail-unexpected-int.err.exp @@ -0,0 +1,5 @@ +error: syntax error, unexpected integer, expecting 'inherit' + at «stdin»:1:3: + 1| { 1 = x; } + | ^ + 2| diff --git a/tests/functional/lang/parse-fail-unexpected-int.nix b/tests/functional/lang/parse-fail-unexpected-int.nix new file mode 100644 index 000000000000..bf3622703ba0 --- /dev/null +++ b/tests/functional/lang/parse-fail-unexpected-int.nix @@ -0,0 +1 @@ +{ 1 = x; } diff --git a/tests/functional/lang/parse-fail-unexpected-leq.err.exp b/tests/functional/lang/parse-fail-unexpected-leq.err.exp new file mode 100644 index 000000000000..a521e195385a --- /dev/null +++ b/tests/functional/lang/parse-fail-unexpected-leq.err.exp @@ -0,0 +1,5 @@ +error: syntax error, unexpected '<=' + at «stdin»:1:1: + 1| <= 1 + | ^ + 2| diff --git a/tests/functional/lang/parse-fail-unexpected-leq.nix b/tests/functional/lang/parse-fail-unexpected-leq.nix new file mode 100644 index 000000000000..e642af429669 --- /dev/null +++ b/tests/functional/lang/parse-fail-unexpected-leq.nix @@ -0,0 +1 @@ +<= 1 diff --git a/tests/functional/lang/parse-fail-unexpected-let.err.exp b/tests/functional/lang/parse-fail-unexpected-let.err.exp new file mode 100644 index 000000000000..4468547cb19c --- /dev/null +++ b/tests/functional/lang/parse-fail-unexpected-let.err.exp @@ -0,0 +1,5 @@ +error: syntax error, unexpected 'let', expecting identifier or 'or' or '${' or '"' + at «stdin»:1:3: + 1| a.let + | ^ + 2| diff --git a/tests/functional/lang/parse-fail-unexpected-let.nix b/tests/functional/lang/parse-fail-unexpected-let.nix new file mode 100644 index 000000000000..632c7b3a0668 --- /dev/null +++ b/tests/functional/lang/parse-fail-unexpected-let.nix @@ -0,0 +1 @@ +a.let diff --git a/tests/functional/lang/parse-fail-unexpected-neq.err.exp b/tests/functional/lang/parse-fail-unexpected-neq.err.exp new file mode 100644 index 000000000000..7a75ac9a778d --- /dev/null +++ b/tests/functional/lang/parse-fail-unexpected-neq.err.exp @@ -0,0 +1,5 @@ +error: syntax error, unexpected '!=' + at «stdin»:1:6: + 1| 1 != != 2 + | ^ + 2| diff --git a/tests/functional/lang/parse-fail-unexpected-neq.nix b/tests/functional/lang/parse-fail-unexpected-neq.nix new file mode 100644 index 000000000000..ca366caa3a7b --- /dev/null +++ b/tests/functional/lang/parse-fail-unexpected-neq.nix @@ -0,0 +1 @@ +1 != != 2 diff --git a/tests/functional/lang/parse-fail-unexpected-or.err.exp b/tests/functional/lang/parse-fail-unexpected-or.err.exp new file mode 100644 index 000000000000..ed1a1a675f59 --- /dev/null +++ b/tests/functional/lang/parse-fail-unexpected-or.err.exp @@ -0,0 +1,5 @@ +error: syntax error, unexpected '||' + at «stdin»:1:1: + 1| || true + | ^ + 2| diff --git a/tests/functional/lang/parse-fail-unexpected-or.nix b/tests/functional/lang/parse-fail-unexpected-or.nix new file mode 100644 index 000000000000..2922d991965f --- /dev/null +++ b/tests/functional/lang/parse-fail-unexpected-or.nix @@ -0,0 +1 @@ +|| true diff --git a/tests/functional/lang/parse-fail-unexpected-path.err.exp b/tests/functional/lang/parse-fail-unexpected-path.err.exp new file mode 100644 index 000000000000..aff95ab81dcb --- /dev/null +++ b/tests/functional/lang/parse-fail-unexpected-path.err.exp @@ -0,0 +1,5 @@ +error: syntax error, unexpected path, expecting identifier or 'or' or '${' or '"' + at «stdin»:1:7: + 1| { } ? ./foo + | ^ + 2| diff --git a/tests/functional/lang/parse-fail-unexpected-path.nix b/tests/functional/lang/parse-fail-unexpected-path.nix new file mode 100644 index 000000000000..8f261419fb35 --- /dev/null +++ b/tests/functional/lang/parse-fail-unexpected-path.nix @@ -0,0 +1 @@ +{ } ? ./foo diff --git a/tests/functional/lang/parse-fail-unexpected-rec.err.exp b/tests/functional/lang/parse-fail-unexpected-rec.err.exp new file mode 100644 index 000000000000..f1db1b4ec33d --- /dev/null +++ b/tests/functional/lang/parse-fail-unexpected-rec.err.exp @@ -0,0 +1,5 @@ +error: syntax error, unexpected 'rec', expecting identifier or 'or' or '${' or '"' + at «stdin»:1:3: + 1| a.rec + | ^ + 2| diff --git a/tests/functional/lang/parse-fail-unexpected-rec.nix b/tests/functional/lang/parse-fail-unexpected-rec.nix new file mode 100644 index 000000000000..01f4facc42d5 --- /dev/null +++ b/tests/functional/lang/parse-fail-unexpected-rec.nix @@ -0,0 +1 @@ +a.rec diff --git a/tests/functional/lang/parse-fail-unexpected-spath.err.exp b/tests/functional/lang/parse-fail-unexpected-spath.err.exp new file mode 100644 index 000000000000..30b8ac83c2c9 --- /dev/null +++ b/tests/functional/lang/parse-fail-unexpected-spath.err.exp @@ -0,0 +1,5 @@ +error: syntax error, unexpected '<…>' path, expecting identifier or 'or' or '${' or '"' + at «stdin»:1:3: + 1| a. + | ^ + 2| diff --git a/tests/functional/lang/parse-fail-unexpected-spath.nix b/tests/functional/lang/parse-fail-unexpected-spath.nix new file mode 100644 index 000000000000..42c8a0cac222 --- /dev/null +++ b/tests/functional/lang/parse-fail-unexpected-spath.nix @@ -0,0 +1 @@ +a. diff --git a/tests/functional/lang/parse-fail-unexpected-then.err.exp b/tests/functional/lang/parse-fail-unexpected-then.err.exp new file mode 100644 index 000000000000..8fbfdd9462d4 --- /dev/null +++ b/tests/functional/lang/parse-fail-unexpected-then.err.exp @@ -0,0 +1,5 @@ +error: syntax error, unexpected 'then' + at «stdin»:1:4: + 1| if then + | ^ + 2| diff --git a/tests/functional/lang/parse-fail-unexpected-then.nix b/tests/functional/lang/parse-fail-unexpected-then.nix new file mode 100644 index 000000000000..f767077b93f2 --- /dev/null +++ b/tests/functional/lang/parse-fail-unexpected-then.nix @@ -0,0 +1 @@ +if then diff --git a/tests/functional/lang/parse-fail-unexpected-update.err.exp b/tests/functional/lang/parse-fail-unexpected-update.err.exp new file mode 100644 index 000000000000..e2a31b98bab0 --- /dev/null +++ b/tests/functional/lang/parse-fail-unexpected-update.err.exp @@ -0,0 +1,5 @@ +error: syntax error, unexpected '//' + at «stdin»:1:1: + 1| // 1 + | ^ + 2| diff --git a/tests/functional/lang/parse-fail-unexpected-update.nix b/tests/functional/lang/parse-fail-unexpected-update.nix new file mode 100644 index 000000000000..9d63dce53f51 --- /dev/null +++ b/tests/functional/lang/parse-fail-unexpected-update.nix @@ -0,0 +1 @@ +// 1 diff --git a/tests/functional/lang/parse-fail-unexpected-uri.err.exp b/tests/functional/lang/parse-fail-unexpected-uri.err.exp new file mode 100644 index 000000000000..148085d56759 --- /dev/null +++ b/tests/functional/lang/parse-fail-unexpected-uri.err.exp @@ -0,0 +1,5 @@ +error: syntax error, unexpected URI + at «stdin»:1:11: + 1| { inherit http://x; } + | ^ + 2| diff --git a/tests/functional/lang/parse-fail-unexpected-uri.nix b/tests/functional/lang/parse-fail-unexpected-uri.nix new file mode 100644 index 000000000000..c57dec0e0f7f --- /dev/null +++ b/tests/functional/lang/parse-fail-unexpected-uri.nix @@ -0,0 +1 @@ +{ inherit http://x; } diff --git a/tests/functional/lang/parse-fail-unexpected-with.err.exp b/tests/functional/lang/parse-fail-unexpected-with.err.exp new file mode 100644 index 000000000000..a79581d919fc --- /dev/null +++ b/tests/functional/lang/parse-fail-unexpected-with.err.exp @@ -0,0 +1,5 @@ +error: syntax error, unexpected 'with', expecting identifier or 'or' or '${' or '"' + at «stdin»:1:3: + 1| a.with + | ^ + 2| diff --git a/tests/functional/lang/parse-fail-unexpected-with.nix b/tests/functional/lang/parse-fail-unexpected-with.nix new file mode 100644 index 000000000000..1c1ec0c61952 --- /dev/null +++ b/tests/functional/lang/parse-fail-unexpected-with.nix @@ -0,0 +1 @@ +a.with diff --git a/tests/functional/lang/parse-okay-regression-20041027.nix b/tests/functional/lang/parse-okay-regression-20041027.nix index ae2e256eeaaa..e1bd41a2114c 100644 --- a/tests/functional/lang/parse-okay-regression-20041027.nix +++ b/tests/functional/lang/parse-okay-regression-20041027.nix @@ -3,7 +3,7 @@ stdenv.mkDerivation { name = "libXi-6.0.1"; src = fetchurl { - url = http://freedesktop.org/~xlibs/release/libXi-6.0.1.tar.bz2; + url = "http://freedesktop.org/~xlibs/release/libXi-6.0.1.tar.bz2"; md5 = "7e935a42428d63a387b3c048be0f2756"; }; /* buildInputs = [pkgconfig]; diff --git a/tests/functional/lang/parse-okay-subversion.nix b/tests/functional/lang/parse-okay-subversion.nix index 356272815d26..360b695b2910 100644 --- a/tests/functional/lang/parse-okay-subversion.nix +++ b/tests/functional/lang/parse-okay-subversion.nix @@ -21,7 +21,7 @@ stdenv.mkDerivation { builder = /foo/bar; src = fetchurl { - url = http://subversion.tigris.org/tarballs/subversion-1.1.1.tar.bz2; + url = "http://subversion.tigris.org/tarballs/subversion-1.1.1.tar.bz2"; md5 = "a180c3fe91680389c210c99def54d9e0"; }; diff --git a/tests/functional/linux-sandbox.sh b/tests/functional/linux-sandbox.sh index 484ad1d2b688..429f1f0dc9a7 100755 --- a/tests/functional/linux-sandbox.sh +++ b/tests/functional/linux-sandbox.sh @@ -101,4 +101,4 @@ nix-sandbox-build symlink-derivation.nix -A test_sandbox_paths \ # shellcheck disable=SC2016 expectStderr 1 nix-sandbox-build --option extra-sandbox-paths '/does-not-exist' \ -E 'with import '"${config_nix}"'; mkDerivation { name = "trivial"; buildCommand = "echo > $out"; }' | - grepQuiet "path '/does-not-exist' is configured as part of the \`sandbox-paths\` option, but is inaccessible" + grepQuiet "path \"/does-not-exist\" is configured as part of the \`sandbox-paths\` option, but is inaccessible" diff --git a/tests/functional/local-overlay-store/common.sh b/tests/functional/local-overlay-store/common.sh index 39ffa6e5a4f0..2c21eaa998db 100644 --- a/tests/functional/local-overlay-store/common.sh +++ b/tests/functional/local-overlay-store/common.sh @@ -70,6 +70,7 @@ mountOverlayfs () { "$storeBRoot/nix/store" \ || skipTest "overlayfs is not supported" + # shellcheck disable=SC2329 cleanupOverlay () { # shellcheck disable=2317 umount -n "$storeBRoot/nix/store" diff --git a/tests/functional/meson.build b/tests/functional/meson.build index d917d91c3f34..bc4d2643e265 100644 --- a/tests/functional/meson.build +++ b/tests/functional/meson.build @@ -64,6 +64,7 @@ suites = [ 'gc-auto.sh', 'user-envs.sh', 'user-envs-migration.sh', + 'cli-characterisation.sh', 'binary-cache.sh', 'multiple-outputs.sh', 'nix-build.sh', @@ -114,6 +115,7 @@ suites = [ 'pure-eval.sh', 'eval.sh', 'short-path-literals.sh', + 'absolute-path-literals.sh', 'no-url-literals.sh', 'repl.sh', 'binary-cache-build-remote.sh', @@ -160,6 +162,7 @@ suites = [ 'nix-profile.sh', 'suggestions.sh', 'store-info.sh', + 'store-print-env.sh', 'fetchClosure.sh', 'completions.sh', 'impure-derivations.sh', diff --git a/tests/functional/nars.sh b/tests/functional/nars.sh index 2925177c5c9f..68bd191a2543 100755 --- a/tests/functional/nars.sh +++ b/tests/functional/nars.sh @@ -114,7 +114,7 @@ if (( unicodeTestCode == 1 )); then # If the command failed (MacOS or ZFS + normalization), checks that it failed # with the expected "already exists" error, and that this is the same # behavior as `touch` - echo "$unicodeTestOut" | grepQuiet "creating directory '.*/out/â': File exists" + echo "$unicodeTestOut" | grepQuiet "creating directory \".*/out/â\": File exists" (( touchFilesCount == 1 )) elif (( unicodeTestCode == 0 )); then diff --git a/tests/functional/nix-profile.sh b/tests/functional/nix-profile.sh index df9a8e0f4b69..6ee78c5dc5eb 100755 --- a/tests/functional/nix-profile.sh +++ b/tests/functional/nix-profile.sh @@ -234,11 +234,11 @@ diff -u <( ) <(cat << EOF error: An existing package already provides the following file: - $(nix build --no-link --print-out-paths "${flake1Dir}""#default.out")/bin/hello + "$(nix build --no-link --print-out-paths "${flake1Dir}""#default.out")/bin/hello" This is the conflicting file from the new package: - $(nix build --no-link --print-out-paths "${flake2Dir}""#default.out")/bin/hello + "$(nix build --no-link --print-out-paths "${flake2Dir}""#default.out")/bin/hello" To remove the existing package: diff --git a/tests/functional/nix-shell.sh b/tests/functional/nix-shell.sh index cdeea32687a1..263679347680 100755 --- a/tests/functional/nix-shell.sh +++ b/tests/functional/nix-shell.sh @@ -39,8 +39,8 @@ testTmpDir=$(pwd)/nix-shell mkdir -p "$testTmpDir" # shellcheck disable=SC2016 output=$(TMPDIR="$testTmpDir" nix-shell --pure "$shellDotNix" -A shellDrv --run 'echo $NIX_BUILD_TOP') -[[ "$output" =~ ${testTmpDir}.* ]] || { - echo "expected $output =~ ${testTmpDir}.*" >&2 +[[ "$output" == "${testTmpDir}"/* ]] || { + echo "expected $output == ${testTmpDir}/*" >&2 exit 1 } @@ -280,9 +280,6 @@ assert (!(args ? inNixShell)); EOF nix-shell "$TEST_ROOT"/shell-ellipsis.nix --run "true" -# FIXME unclear why this (newly made) test is failing in this case. -if ! isTestOnNixOS; then - # `nix develop` should also work with fixed-output derivations - # shellcheck disable=SC2016 - nix develop -f "$shellDotNix" fixed -c bash -c '[[ -n $stdenv ]]' -fi +# `nix develop` should also work with fixed-output derivations +# shellcheck disable=SC2016 +nix develop -f "$shellDotNix" fixed -c bash -c '[[ $FOO == "was a fixed-output derivation" ]]' diff --git a/tests/functional/no-url-literals.sh b/tests/functional/no-url-literals.sh index fbc6e1cec24a..7b86d79b5a2c 100644 --- a/tests/functional/no-url-literals.sh +++ b/tests/functional/no-url-literals.sh @@ -2,27 +2,18 @@ source common.sh -clearStoreIfPossible +# Tests covered by lang tests: +# - Default: unquoted URLs accepted → eval-okay-url-literal-default +# - Fatal: unquoted URLs rejected → eval-fail-url-literal +# - Warn: produces warning → eval-okay-url-literal-warn +# - Quoted URLs accepted with fatal → eval-okay-url-literal-quoted-fatal -# Test 1: By default, unquoted URLs are accepted -nix eval --expr 'http://example.com' 2>&1 | grepQuietInverse "error: URL literals are disabled" +# Test: URLs with parameters (which must be quoted) are accepted +nix eval --lint-url-literals fatal --expr '"http://example.com?foo=bar"' 2>&1 | grepQuietInverse "error:" -# Test 2: With the experimental feature enabled, unquoted URLs are rejected -expect 1 nix eval --extra-experimental-features 'no-url-literals' --expr 'http://example.com' 2>&1 | grepQuiet "error: URL literals are disabled" +# Test: The setting can be enabled via NIX_CONFIG +expect 1 env NIX_CONFIG='lint-url-literals = fatal' nix eval --expr 'http://example.com' 2>&1 | grepQuiet "error: URL literal" -# Test 3: Quoted URLs are always accepted -nix eval --extra-experimental-features 'no-url-literals' --expr '"http://example.com"' 2>&1 | grepQuietInverse "error: URL literals are disabled" - -# Test 4: URLs with parameters (which must be quoted) are accepted -nix eval --extra-experimental-features 'no-url-literals' --expr '"http://example.com?foo=bar"' 2>&1 | grepQuietInverse "error: URL literals are disabled" - -# Test 5: The feature can be enabled via NIX_CONFIG -expect 1 env NIX_CONFIG='extra-experimental-features = no-url-literals' nix eval --expr 'http://example.com' 2>&1 | grepQuiet "error: URL literals are disabled" - -# Test 6: The feature can be enabled via CLI even if not set in config -expect 1 env NIX_CONFIG='' nix eval --extra-experimental-features 'no-url-literals' --expr 'http://example.com' 2>&1 | grepQuiet "error: URL literals are disabled" - -# Test 7: Evaluation still works for quoted URLs -nix eval --raw --extra-experimental-features no-url-literals --expr '"http://example.com"' | grepQuiet "^http://example.com$" - -echo "no-url-literals test passed!" +# Test: Using old experimental feature name produces helpful warning +nix eval --extra-experimental-features no-url-literals --expr '"test"' 2>&1 \ + | grepQuiet "experimental feature 'no-url-literals' has been stabilized and renamed; use 'lint-url-literals = fatal' setting instead" diff --git a/tests/functional/plugins/plugintest.cc b/tests/functional/plugins/plugintest.cc index e8f80a4aa965..f562bab56af0 100644 --- a/tests/functional/plugins/plugintest.cc +++ b/tests/functional/plugins/plugintest.cc @@ -23,5 +23,5 @@ static void prim_anotherNull(EvalState & state, const PosIdx pos, Value ** args, static RegisterPrimOp rp({ .name = "anotherNull", .arity = 0, - .fun = prim_anotherNull, + .impl = prim_anotherNull, }); diff --git a/tests/functional/pure-eval.sh b/tests/functional/pure-eval.sh index b769b2150f11..4af4bc0d244e 100755 --- a/tests/functional/pure-eval.sh +++ b/tests/functional/pure-eval.sh @@ -22,9 +22,9 @@ echo "$missingImpureErrorMsg" | grepQuiet -- --impure || \ (! nix-instantiate --pure-eval ./simple.nix) -[[ $(nix eval --impure --expr "(import (builtins.fetchurl { url = file://$(pwd)/pure-eval.nix; })).x") == 123 ]] -(! nix eval --expr "(import (builtins.fetchurl { url = file://$(pwd)/pure-eval.nix; })).x") -nix eval --expr "(import (builtins.fetchurl { url = file://$(pwd)/pure-eval.nix; sha256 = \"$(nix hash file pure-eval.nix --type sha256)\"; })).x" +[[ $(nix eval --impure --expr "(import (builtins.fetchurl { url = \"file://$(pwd)/pure-eval.nix\"; })).x") == 123 ]] +(! nix eval --expr "(import (builtins.fetchurl { url = \"file://$(pwd)/pure-eval.nix\"; })).x") +nix eval --expr "(import (builtins.fetchurl { url = \"file://$(pwd)/pure-eval.nix\"; sha256 = \"$(nix hash file pure-eval.nix --type sha256)\"; })).x" rm -rf "$TEST_ROOT"/eval-out nix eval --store dummy:// --write-to "$TEST_ROOT"/eval-out --expr '{ x = "foo" + "bar"; y = { z = "bla"; }; }' diff --git a/tests/functional/repl.sh b/tests/functional/repl.sh index 0e84a3d14388..7c28a7529009 100755 --- a/tests/functional/repl.sh +++ b/tests/functional/repl.sh @@ -68,7 +68,7 @@ testRepl () { echo "$replOutput" | grepInverse "error: Cannot run 'nix-shell'" expectStderr 1 nix repl "${testDir}/simple.nix" \ - | grepQuiet -s "error: path '$testDir/simple.nix' is not a flake" + | grepQuiet -s "error: path \"$testDir/simple.nix\" is not a flake" } # Simple test, try building a drv @@ -139,6 +139,64 @@ testReplResponseNoRegex ' "$" + "{hi}" ' '"\${hi}"' +# Test inherit statement support (issue #15053) +testReplResponseNoRegex ' +a = { b = 1; c = 2; } +inherit (a) b +b +' '1' + +# inherit multiple attributes +testReplResponseNoRegex ' +a = { x = 10; y = 20; } +inherit (a) x y +x + y +' '30' + +# inherit from current scope +testReplResponseNoRegex ' +foo = 42 +inherit foo +foo +' '42' + +# inherit with semicolon (also works) +testReplResponseNoRegex ' +a = { z = 99; } +inherit (a) z; +z +' '99' + +# multiple bindings on one line +testReplResponseNoRegex ' +a = 1; b = 2; +a + b +' '3' + +# nested attribute path +testReplResponseNoRegex ' +a.b.c = 1; +a.b +' '{ c = 1; }' + +# mixed bindings: inherit and assignment +testReplResponseNoRegex ' +x = { p = 10; } +inherit (x) p; q = 20; +p + q +' '30' + +# inherit error shows position (without spurious semicolon from retry) +testReplResponse ' +a = { x = 1; } +inherit (a) y +y +' "error: attribute 'y' missing +.*at .string.:1:13: +.*inherit (a) y +.* \\^ +.*Did you mean x" + testReplResponse ' drvPath ' '".*-simple.drv"' \ @@ -311,6 +369,12 @@ testReplResponseNoRegex ' } ' +testReplResponseNoRegex ' +:ll +' \ +'error: nothing has been loaded yet +' + # Don't prompt for more input when getting unexpected EOF in imported files. testReplResponse " import $testDir/lang/parse-fail-eof-pos.nix diff --git a/tests/functional/restricted.sh b/tests/functional/restricted.sh index 2f65f15fe5d0..c86abebee3d0 100755 --- a/tests/functional/restricted.sh +++ b/tests/functional/restricted.sh @@ -26,18 +26,18 @@ nix-instantiate --restrict-eval --eval -E 'builtins.readFile ./simple.nix' -I sr expectStderr 1 nix-instantiate --restrict-eval --eval -E 'let __nixPath = [ { prefix = "foo"; path = ./.; } ]; in builtins.readFile ' | grepQuiet "forbidden in restricted mode" nix-instantiate --restrict-eval --eval -E 'let __nixPath = [ { prefix = "foo"; path = ./.; } ]; in builtins.readFile ' -I src=. -p=$(nix eval --raw --expr "builtins.fetchurl file://${_NIX_TEST_SOURCE_DIR}/restricted.sh" --impure --restrict-eval --allowed-uris "file://${_NIX_TEST_SOURCE_DIR}") +p=$(nix eval --raw --expr "builtins.fetchurl \"file://${_NIX_TEST_SOURCE_DIR}/restricted.sh\"" --impure --restrict-eval --allowed-uris "file://${_NIX_TEST_SOURCE_DIR}") cmp "$p" "${_NIX_TEST_SOURCE_DIR}/restricted.sh" -(! nix eval --raw --expr "builtins.fetchurl file://${_NIX_TEST_SOURCE_DIR}/restricted.sh" --impure --restrict-eval) +(! nix eval --raw --expr "builtins.fetchurl \"file://${_NIX_TEST_SOURCE_DIR}/restricted.sh\"" --impure --restrict-eval) -(! nix eval --raw --expr "builtins.fetchurl file://${_NIX_TEST_SOURCE_DIR}/restricted.sh" --impure --restrict-eval --allowed-uris "file://${_NIX_TEST_SOURCE_DIR}/restricted.sh/") +(! nix eval --raw --expr "builtins.fetchurl \"file://${_NIX_TEST_SOURCE_DIR}/restricted.sh\"" --impure --restrict-eval --allowed-uris "file://${_NIX_TEST_SOURCE_DIR}/restricted.sh/") -nix eval --raw --expr "builtins.fetchurl file://${_NIX_TEST_SOURCE_DIR}/restricted.sh" --impure --restrict-eval --allowed-uris "file://${_NIX_TEST_SOURCE_DIR}/restricted.sh" +nix eval --raw --expr "builtins.fetchurl \"file://${_NIX_TEST_SOURCE_DIR}/restricted.sh\"" --impure --restrict-eval --allowed-uris "file://${_NIX_TEST_SOURCE_DIR}/restricted.sh" -(! nix eval --raw --expr "builtins.fetchurl https://github.com/NixOS/patchelf/archive/master.tar.gz" --impure --restrict-eval) -(! nix eval --raw --expr "builtins.fetchTarball https://github.com/NixOS/patchelf/archive/master.tar.gz" --impure --restrict-eval) -(! nix eval --raw --expr "fetchGit git://github.com/NixOS/patchelf.git" --impure --restrict-eval) +(! nix eval --raw --expr "builtins.fetchurl \"https://github.com/NixOS/patchelf/archive/master.tar.gz\"" --impure --restrict-eval) +(! nix eval --raw --expr "builtins.fetchTarball \"https://github.com/NixOS/patchelf/archive/master.tar.gz\"" --impure --restrict-eval) +(! nix eval --raw --expr "fetchGit \"git://github.com/NixOS/patchelf.git\"" --impure --restrict-eval) ln -sfn "${_NIX_TEST_SOURCE_DIR}/restricted.nix" "$TEST_ROOT/restricted.nix" [[ $(nix-instantiate --eval "$TEST_ROOT"/restricted.nix) == 3 ]] diff --git a/tests/functional/shell.nix b/tests/functional/shell.nix index 267b0c8f0161..071b99b3f24a 100644 --- a/tests/functional/shell.nix +++ b/tests/functional/shell.nix @@ -106,14 +106,16 @@ let foo = runCommand "foo" { } '' mkdir -p $out/bin - echo 'echo ${fooContents}' > $out/bin/foo + echo '#!${shell}' > $out/bin/foo + echo 'echo ${fooContents}' >> $out/bin/foo chmod a+rx $out/bin/foo ln -s ${shell} $out/bin/bash ''; bar = runCommand "bar" { } '' mkdir -p $out/bin - echo 'echo bar' > $out/bin/bar + echo '#!${shell}' > $out/bin/bar + echo 'echo bar' >> $out/bin/bar chmod a+rx $out/bin/bar ''; @@ -126,7 +128,8 @@ let # ruby "interpreter" that outputs "$@" ruby = runCommand "ruby" { } '' mkdir -p $out/bin - echo 'printf %s "$*"' > $out/bin/ruby + echo '#!${shell}' > $out/bin/ruby + echo 'printf %s "$*"' >> $out/bin/ruby chmod a+rx $out/bin/ruby ''; diff --git a/tests/functional/short-path-literals.sh b/tests/functional/short-path-literals.sh index f74044ddad74..5ee864f0cb08 100644 --- a/tests/functional/short-path-literals.sh +++ b/tests/functional/short-path-literals.sh @@ -2,54 +2,53 @@ source common.sh -clearStoreIfPossible +# Tests covered by lang tests: +# - Warning for short paths → eval-okay-short-path-literal-warn +# - Fatal for short paths → eval-fail-short-path-literal +# - Variation paths (foo/bar, a/b/c/d) → eval-okay-short-path-variation +# - ./ paths with fatal → eval-okay-dotslash-path-fatal +# - ../ paths with fatal → eval-okay-dotdotslash-path-fatal -# Test 1: Without the setting (default), no warnings should be produced +# Tests for the deprecated --warn-short-path-literals boolean setting + +# Test: Without the setting (default), no warnings should be produced nix eval --expr 'test/subdir' 2>"$TEST_ROOT"/stderr grepQuietInverse < "$TEST_ROOT/stderr" -E "relative path|path literal" || fail "Should not produce warnings by default" -# Test 2: With the setting enabled, warnings should be produced for short path literals -nix eval --warn-short-path-literals --expr 'test/subdir' 2>"$TEST_ROOT"/stderr -grepQuiet "relative path literal 'test/subdir' should be prefixed with '.' for clarity: './test/subdir'" "$TEST_ROOT/stderr" - -# Test 3: Different short path literals should all produce warnings -nix eval --warn-short-path-literals --expr 'foo/bar' 2>"$TEST_ROOT"/stderr -grepQuiet "relative path literal 'foo/bar' should be prefixed with '.' for clarity: './foo/bar'" "$TEST_ROOT/stderr" - -nix eval --warn-short-path-literals --expr 'a/b/c/d' 2>"$TEST_ROOT"/stderr -grepQuiet "relative path literal 'a/b/c/d' should be prefixed with '.' for clarity: './a/b/c/d'" "$TEST_ROOT/stderr" - -# Test 4: Paths starting with ./ should NOT produce warnings +# Test: Paths starting with ./ should NOT produce warnings nix eval --warn-short-path-literals --expr './test/subdir' 2>"$TEST_ROOT"/stderr grepQuietInverse "relative path literal" "$TEST_ROOT/stderr" -# Test 5: Paths starting with ../ should NOT produce warnings +# Test: Paths starting with ../ should NOT produce warnings nix eval --warn-short-path-literals --expr '../test/subdir' 2>"$TEST_ROOT"/stderr grepQuietInverse "relative path literal" "$TEST_ROOT/stderr" -# Test 6: Absolute paths should NOT produce warnings +# Test: Absolute paths should NOT produce warnings nix eval --warn-short-path-literals --expr '/absolute/path' 2>"$TEST_ROOT"/stderr grepQuietInverse "relative path literal" "$TEST_ROOT/stderr" -# Test 7: Test that the warning is at the correct position -nix eval --warn-short-path-literals --expr 'foo/bar' 2>"$TEST_ROOT"/stderr -grepQuiet "at «string»:1:1:" "$TEST_ROOT/stderr" - -# Test 8: Test that evaluation still works correctly despite the warning -result=$(nix eval --warn-short-path-literals --expr 'test/subdir' 2>/dev/null) -expected="$PWD/test/subdir" -[[ "$result" == "$expected" ]] || fail "Evaluation result should be correct despite warning" - -# Test 9: Test with nix-instantiate as well +# Test: Test with nix-instantiate as well nix-instantiate --warn-short-path-literals --eval -E 'foo/bar' 2>"$TEST_ROOT"/stderr grepQuiet "relative path literal 'foo/bar' should be prefixed" "$TEST_ROOT/stderr" -# Test 10: Test that the setting can be set via configuration +# Test: Test that the deprecated setting can be set via configuration NIX_CONFIG='warn-short-path-literals = true' nix eval --expr 'test/file' 2>"$TEST_ROOT"/stderr grepQuiet "relative path literal 'test/file' should be prefixed" "$TEST_ROOT/stderr" -# Test 11: Test that command line flag overrides config +# Test: Test that command line flag overrides config NIX_CONFIG='warn-short-path-literals = true' nix eval --no-warn-short-path-literals --expr 'test/file' 2>"$TEST_ROOT"/stderr grepQuietInverse "relative path literal" "$TEST_ROOT/stderr" -echo "short-path-literals test passed!" +# Tests for NIX_CONFIG and setting precedence + +# Test: New setting via NIX_CONFIG +NIX_CONFIG='lint-short-path-literals = warn' nix eval --expr 'test/file' 2>"$TEST_ROOT"/stderr +grepQuiet "relative path literal 'test/file' should be prefixed" "$TEST_ROOT/stderr" + +# Test: New setting overrides deprecated setting +NIX_CONFIG='warn-short-path-literals = true' nix eval --lint-short-path-literals ignore --expr 'test/file' 2>"$TEST_ROOT"/stderr +grepQuietInverse "relative path literal" "$TEST_ROOT/stderr" + +# Test: Explicit new setting takes precedence (error over deprecated warn) +NIX_CONFIG='warn-short-path-literals = true' expectStderr 1 nix eval --lint-short-path-literals fatal --expr 'test/subdir' \ + | grepQuiet "error:" diff --git a/tests/functional/simple.sh b/tests/functional/simple.sh index e54ad860ca97..366c75a3af0e 100755 --- a/tests/functional/simple.sh +++ b/tests/functional/simple.sh @@ -35,3 +35,24 @@ if test "$outPath" != "/foo/xxiwa5zlaajv6xdjynf9yym9g319d6mn-big-derivation-attr echo "big-derivation-attr.nix hash appears broken, got $outPath. Memory corruption in large drv attr?" exit 1 fi + +# Test that nix-instantiate on a deeply nested recurseForDerivations structure +# produces a controlled stack overflow error rather than a segfault. +expectStderr 1 nix-instantiate --expr 'let x = { recurseForDerivations = true; more = x; }; in x' \ + | grepQuiet "stack overflow; max-call-depth exceeded" + +# Test that nix-env -qa --meta on deeply nested meta attributes produces a +# controlled stack overflow error rather than a segfault. +echo 'let f = n: { type = "derivation"; name = "test"; system = "x86_64-linux"; meta.nested = f (n + 1); }; in { pkg = f 0; }' > "$TEST_ROOT/deep-meta.nix" +expectStderr 1 nix-env -qa -f "$TEST_ROOT/deep-meta.nix" --json --meta \ + | grepQuiet "stack overflow; max-call-depth exceeded" + +# Test that nix-instantiate --eval on a pre-forced deep structure (built with +# foldl' to avoid thunks) produces a controlled stack overflow error rather than +# a segfault when printAmbiguous traverses the structure. +# Note: Without the fix, this test may pass if the system stack is large enough. +# The fix ensures we get a controlled error at max-call-depth (default 10000) +# rather than relying on the system stack limit. +# shellcheck disable=SC2016 +expectStderr 1 nix-instantiate --eval --expr 'builtins.foldl'\'' (acc: _: { inner = acc; }) null (builtins.genList (x: x) 20000)' \ + | grepQuiet "stack overflow; max-call-depth exceeded" diff --git a/tests/functional/store-print-env.sh b/tests/functional/store-print-env.sh new file mode 100755 index 000000000000..894dc2e3ac9d --- /dev/null +++ b/tests/functional/store-print-env.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash + +source common.sh + +clearStoreIfPossible + +# Regression test for nix-store --print-env argument escaping +# This tests that arguments in _args are properly escaped as a single string +# rather than double-escaped which could lead to command injection + +cat > "$TEST_ROOT/test-args.nix" <<'EOF' +derivation { + name = "test-print-env-args"; + system = builtins.currentSystem; + builder = "/bin/sh"; + args = [ "-c" "echo hello world" ]; +} +EOF + +drvPath=$(nix-instantiate "$TEST_ROOT/test-args.nix") +output=$(nix-store --print-env "$drvPath" | grep "^export _args") + +# The output should be: export _args; _args='-c echo hello world' +# NOT: export _args; _args=''-c' 'echo hello world'' + +# Test that it can be safely evaluated +eval "$output" +expected="-c echo hello world" +# shellcheck disable=SC2154 # _args is set by the eval above +if [ "$_args" != "$expected" ]; then + echo "ERROR: _args not properly escaped!" + echo "Expected: $expected" + echo "Got: $_args" + echo "Raw output: $output" + exit 1 +fi diff --git a/tests/functional/structured-attrs.sh b/tests/functional/structured-attrs.sh index 01bdc10d1162..321cb6107992 100755 --- a/tests/functional/structured-attrs.sh +++ b/tests/functional/structured-attrs.sh @@ -50,3 +50,16 @@ expectStderr 0 nix-instantiate --expr "$hackyExpr" --eval --strict | grepQuiet " # Check it works with the expected structured attrs hacky=$(nix-instantiate --expr "$hackyExpr") nix derivation show "$hacky" | jq --exit-status '.derivations."'"$(basename "$hacky")"'".structuredAttrs | . == {"a": 1}' + +# Test warning for non-object exportReferencesGraph in structured attrs +# shellcheck disable=SC2016 +expectStderr 0 nix-build --no-out-link --expr ' + with import ./config.nix; + mkDerivation { + name = "export-graph-non-object"; + __structuredAttrs = true; + exportReferencesGraph = [ "foo" "bar" ]; + builder = "/bin/sh"; + args = ["-c" "echo foo > ${builtins.placeholder "out"}"]; + } +' | grepQuiet "warning:.*exportReferencesGraph.*not a JSON object" diff --git a/tests/functional/tarball.sh b/tests/functional/tarball.sh index 451ee879a5b3..38c183d9bab6 100755 --- a/tests/functional/tarball.sh +++ b/tests/functional/tarball.sh @@ -28,21 +28,21 @@ test_tarball() { nix-build -o "$TEST_ROOT"/result '' -I foo=file://"$tarball" - nix-build -o "$TEST_ROOT"/result -E "import (fetchTarball file://$tarball)" + nix-build -o "$TEST_ROOT"/result -E "import (fetchTarball \"file://$tarball\")" # Do not re-fetch paths already present - nix-build -o "$TEST_ROOT"/result -E "import (fetchTarball { url = file:///does-not-exist/must-remain-unused/$tarball; sha256 = \"$hash\"; })" + nix-build -o "$TEST_ROOT"/result -E "import (fetchTarball { url = \"file:///does-not-exist/must-remain-unused/$tarball\"; sha256 = \"$hash\"; })" - nix-build -o "$TEST_ROOT"/result -E "import (fetchTree file://$tarball)" - nix-build -o "$TEST_ROOT"/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; })" - nix-build -o "$TEST_ROOT"/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"$hash\"; })" + nix-build -o "$TEST_ROOT"/result -E "import (fetchTree \"file://$tarball\")" + nix-build -o "$TEST_ROOT"/result -E "import (fetchTree { type = \"tarball\"; url = \"file://$tarball\"; })" + nix-build -o "$TEST_ROOT"/result -E "import (fetchTree { type = \"tarball\"; url = \"file://$tarball\"; narHash = \"$hash\"; })" - [[ $(nix eval --impure --expr "(fetchTree file://$tarball).lastModified") = 1000000000 ]] + [[ $(nix eval --impure --expr "(fetchTree \"file://$tarball\").lastModified") = 1000000000 ]] # fetchTree with a narHash is implicitly final, so it doesn't return attributes like lastModified. - [[ $(nix eval --impure --expr "(fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"$hash\"; }) ? lastModified") = false ]] + [[ $(nix eval --impure --expr "(fetchTree { type = \"tarball\"; url = \"file://$tarball\"; narHash = \"$hash\"; }) ? lastModified") = false ]] - nix-instantiate --strict --eval -E "!((import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"$hash\"; })) ? submodules)" >&2 - nix-instantiate --strict --eval -E "!((import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"$hash\"; })) ? submodules)" 2>&1 | grep 'true' + nix-instantiate --strict --eval -E "!((import (fetchTree { type = \"tarball\"; url = \"file://$tarball\"; narHash = \"$hash\"; })) ? submodules)" >&2 + nix-instantiate --strict --eval -E "!((import (fetchTree { type = \"tarball\"; url = \"file://$tarball\"; narHash = \"$hash\"; })) ? submodules)" 2>&1 | grep 'true' nix-instantiate --eval -E '1 + 2' -I fnord=file:///no-such-tarball.tar"$ext" nix-instantiate --eval -E 'with ; 1 + 2' -I fnord=file:///no-such-tarball"$ext" @@ -52,7 +52,7 @@ test_tarball() { # Ensure that the `name` attribute isn’t accepted as that would mess # with the content-addressing - (! nix-instantiate --eval -E "fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"$hash\"; name = \"foo\"; }") + (! nix-instantiate --eval -E "fetchTree { type = \"tarball\"; url = \"file://$tarball\"; narHash = \"$hash\"; name = \"foo\"; }") store_path=$(nix store prefetch-file --json "file://$tarball" | jq -r .storePath) if ! cmp -s "$store_path" "$tarball"; then @@ -115,3 +115,7 @@ path="$(nix flake prefetch --refresh --json "tarball+file://$TEST_ROOT/tar.tar" [[ $(cat "$path/a/b/xyzzy") = xyzzy ]] [[ $(cat "$path/a/b/foo") = foo ]] [[ $(cat "$path/bla") = abc ]] + +# Test that unpacking an empty file does not segfault (see https://github.com/NixOS/nix/issues/15116). +touch "$TEST_ROOT/empty" +expectStderr 1 nix store prefetch-file --unpack "file://$TEST_ROOT/empty" | grepQuiet "archive.*is empty" diff --git a/tests/functional/test-infra.sh b/tests/functional/test-infra.sh index 2da26b08ccd8..b702dfe5d29f 100755 --- a/tests/functional/test-infra.sh +++ b/tests/functional/test-infra.sh @@ -48,7 +48,7 @@ expectStderr 1 noisyFalse | grepQuiet NAY # `set -o pipefile` is enabled -# shellcheck disable=SC2317# shellcheck disable=SC2317 +# shellcheck disable=SC2329 pipefailure () { # shellcheck disable=SC2216 true | false | true @@ -56,7 +56,7 @@ pipefailure () { expect 1 pipefailure unset pipefailure -# shellcheck disable=SC2317 +# shellcheck disable=2329 pipefailure () { # shellcheck disable=SC2216 false | true | true @@ -84,7 +84,7 @@ expect 1 useUnbound # ! alone unfortunately negates `set -e`, but it works in functions: # shellcheck disable=SC2251 ! true -# shellcheck disable=SC2317 +# shellcheck disable=SC2329 funBang () { ! true } diff --git a/tests/functional/timeout.sh b/tests/functional/timeout.sh index ae47fdc9684e..1bd23118f1aa 100755 --- a/tests/functional/timeout.sh +++ b/tests/functional/timeout.sh @@ -7,39 +7,14 @@ source common.sh # XXX: This shouldn’t be, but #4813 cause this test to fail needLocalStore "see #4813" -messages=$(nix-build -Q timeout.nix -A infiniteLoop --timeout 2 2>&1) && status=0 || status=$? - -if [ "$status" -ne 101 ]; then - echo "error: 'nix-store' exited with '$status'; should have exited 101" - - # FIXME: https://github.com/NixOS/nix/issues/4813 - skipTest "Do not block CI until fixed" - - exit 1 -fi - -if echo "$messages" | grepQuietInvert "timed out"; then - echo "error: build may have failed for reasons other than timeout; output:" - echo "$messages" >&2 - exit 1 -fi - -if nix-build -Q timeout.nix -A infiniteLoop --max-build-log-size 100; then - echo "build should have failed" - exit 1 -fi - -if nix-build timeout.nix -A silent --max-silent-time 2; then - echo "build should have failed" - exit 1 -fi - -if nix-build timeout.nix -A closeLog; then - echo "build should have failed" - exit 1 -fi - -if nix build -f timeout.nix silent --max-silent-time 2; then - echo "build should have failed" - exit 1 -fi +# FIXME: https://github.com/NixOS/nix/issues/4813 +expectStderr 101 nix-build -Q timeout.nix -A infiniteLoop --timeout 2 | grepQuiet "timed out" \ + || skipTest "Do not block CI until fixed" + +expectStderr 1 nix-build -Q timeout.nix -A infiniteLoop --max-build-log-size 100 | grepQuiet "killed after writing more than 100 bytes of log output" + +expectStderr 101 nix-build timeout.nix -A silent --max-silent-time 2 | grepQuiet "timed out after 2 seconds" + +expectStderr 100 nix-build timeout.nix -A closeLog | grepQuiet "builder failed due to signal" + +expectStderr 1 nix build -f timeout.nix silent --max-silent-time 2 | grepQuiet "timed out after 2 seconds" diff --git a/tests/installer/default.nix b/tests/installer/default.nix index dc831cc97b1b..c0ce41233e65 100644 --- a/tests/installer/default.nix +++ b/tests/installer/default.nix @@ -44,7 +44,7 @@ let mockChannel = pkgs: - pkgs.runCommandNoCC "mock-channel" { } '' + pkgs.runCommand "mock-channel" { } '' mkdir nixexprs mkdir -p $out/channel echo -n 'someContent' > nixexprs/someFile @@ -154,7 +154,7 @@ let image = image.image; postBoot = image.postBoot or ""; installScript = installScripts.${testName}.script; - binaryTarball = binaryTarballs.${system}; + binaryTarball = binaryTarballs.${image.system}; } '' shopt -s nullglob diff --git a/tests/nixos/content-encoding.nix b/tests/nixos/content-encoding.nix index 1e188cb060b7..aa8d76fe7b87 100644 --- a/tests/nixos/content-encoding.nix +++ b/tests/nixos/content-encoding.nix @@ -82,7 +82,7 @@ in # Serve .narinfo files with gzip encoding location ~ \.narinfo$ { - add_header Content-Encoding gzip; + add_header Content-Encoding x-gzip; default_type "text/x-nix-narinfo"; } @@ -173,7 +173,7 @@ in # Check Content-Encoding headers on the download endpoint narinfo_headers = machine.succeed(f"curl -I http://localhost/cache/{narinfoHash}.narinfo 2>&1") - assert "content-encoding: gzip" in narinfo_headers.lower(), f"Expected 'content-encoding: gzip' for .narinfo file, but headers were: {narinfo_headers}" + assert "content-encoding: x-gzip" in narinfo_headers.lower(), f"Expected 'content-encoding: x-gzip' for .narinfo file, but headers were: {narinfo_headers}" ls_headers = machine.succeed(f"curl -I http://localhost/cache/{narinfoHash}.ls 2>&1") assert "content-encoding: gzip" in ls_headers.lower(), f"Expected 'content-encoding: gzip' for .ls file, but headers were: {ls_headers}" diff --git a/tests/nixos/default.nix b/tests/nixos/default.nix index edfa4124f3f5..f06ad1c30937 100644 --- a/tests/nixos/default.nix +++ b/tests/nixos/default.nix @@ -75,21 +75,6 @@ let ]; }; - otherNixes.nix_2_3.setNixPackage = - { lib, pkgs, ... }: - { - imports = [ checkOverrideNixVersion ]; - nix.package = lib.mkForce ( - pkgs.nixVersions.nix_2_3.overrideAttrs (o: { - meta = o.meta // { - # This version shouldn't be used by end-users, but we run tests against - # it to ensure we don't break protocol compatibility. - knownVulnerabilities = [ ]; - }; - }) - ); - }; - otherNixes.nix_2_13.setNixPackage = { lib, pkgs, ... }: { @@ -106,6 +91,14 @@ let ); }; + otherNixes.nix_2_18.setNixPackage = + { lib, pkgs, ... }: + { + imports = [ checkOverrideNixVersion ]; + nix.package = lib.mkForce ( + nixpkgs-23-11.legacyPackages.${pkgs.stdenv.hostPlatform.system}.nixVersions.nix_2_18 + ); + }; in { @@ -197,6 +190,8 @@ in functional_symlinked-home = runNixOSTest ./functional/symlinked-home.nix; + functional_unprivileged-daemon = runNixOSTest ./functional/unprivileged-daemon.nix; + user-sandboxing = runNixOSTest ./user-sandboxing; s3-binary-cache-store = runNixOSTest ./s3-binary-cache-store.nix; @@ -210,4 +205,21 @@ in fetchersSubstitute = runNixOSTest ./fetchers-substitute.nix; chrootStore = runNixOSTest ./chroot-store.nix; + + upgrade-nix = runNixOSTest { + imports = [ ./upgrade-nix.nix ]; + upgrade-nix.oldNix = nixComponents.nix-cli; + }; + + upgrade-nix_fromStable = runNixOSTest { + imports = [ ./upgrade-nix.nix ]; + name = lib.mkForce "upgrade-nix-from-stable"; + upgrade-nix.oldNix = pkgs.nixVersions.stable; + }; + + upgrade-nix_fromLatest = runNixOSTest { + imports = [ ./upgrade-nix.nix ]; + name = lib.mkForce "upgrade-nix-from-latest"; + upgrade-nix.oldNix = pkgs.nixVersions.latest; + }; } diff --git a/tests/nixos/fetch-git/testsupport/setup.nix b/tests/nixos/fetch-git/testsupport/setup.nix index 3c9f4bddea12..7e5423e9d6e1 100644 --- a/tests/nixos/fetch-git/testsupport/setup.nix +++ b/tests/nixos/fetch-git/testsupport/setup.nix @@ -82,7 +82,7 @@ in _NIX_FORCE_HTTP = "1"; }; }; - setupScript = ''''; + setupScript = ""; testScript = '' start_all(); diff --git a/tests/nixos/fetchers-substitute.nix b/tests/nixos/fetchers-substitute.nix index a26748dca658..1ca4e4825155 100644 --- a/tests/nixos/fetchers-substitute.nix +++ b/tests/nixos/fetchers-substitute.nix @@ -1,9 +1,26 @@ +{ nixComponents, ... }: { name = "fetchers-substitute"; nodes.substituter = { pkgs, ... }: { + # nix-serve is broken while cross-compiling in nixpkgs 25.11. It's been + # fixed since, but while we're pinning 25.11 we use this workaround. + nixpkgs.overlays = [ + (final: prev: { + nix-serve = + final.lib.warnIf (final.lib.versions.majorMinor final.lib.version != "25.11") + "remove the hack in fetchers-substitute.nix when updating nixpkgs from 25.11" + ( + prev.nix-serve.override { + nix = prev.nix // { + libs.nix-perl-bindings = nixComponents.nix-perl-bindings; + }; + } + ); + }) + ]; virtualisation.writableStore = true; nix.settings.extra-experimental-features = [ diff --git a/tests/nixos/fsync.nix b/tests/nixos/fsync.nix index 50105f1ccd98..4cdb17ada0c7 100644 --- a/tests/nixos/fsync.nix +++ b/tests/nixos/fsync.nix @@ -39,7 +39,7 @@ in for fs in ("ext4", "btrfs", "xfs"): machine.succeed("mkfs.{} {} /dev/vdb".format(fs, "-F" if fs == "ext4" else "-f")) machine.succeed("mkdir -p /mnt") - machine.succeed("mount /dev/vdb /mnt") + machine.succeed("mount -t {} /dev/vdb /mnt".format(fs)) machine.succeed("sync") machine.succeed("nix copy --offline ${pkg1} --to /mnt") machine.crash() @@ -47,7 +47,7 @@ in machine.start() machine.wait_for_unit("multi-user.target") machine.succeed("mkdir -p /mnt") - machine.succeed("mount /dev/vdb /mnt") + machine.succeed("mount -t {} /dev/vdb /mnt".format(fs)) machine.succeed("nix path-info --offline --store /mnt ${pkg1}") machine.succeed("nix store verify --all --store /mnt --no-trust") diff --git a/tests/nixos/functional/common.nix b/tests/nixos/functional/common.nix index 72b7b61d12c7..74ef7ea27623 100644 --- a/tests/nixos/functional/common.nix +++ b/tests/nixos/functional/common.nix @@ -1,14 +1,5 @@ { lib, nixComponents, ... }: -let - # FIXME (roberth) reference issue - inputDerivation = - pkg: - (pkg.overrideAttrs (o: { - disallowedReferences = [ ]; - })).inputDerivation; - -in { # We rarely change the script in a way that benefits from type checking, so # we skip it to save time. @@ -20,7 +11,7 @@ in virtualisation.writableStore = true; system.extraDependencies = [ - (inputDerivation config.nix.package) + config.nix.package.inputDerivation ]; nix.settings.substituters = lib.mkForce [ ]; diff --git a/tests/nixos/functional/unprivileged-daemon.nix b/tests/nixos/functional/unprivileged-daemon.nix new file mode 100644 index 000000000000..ac99c2881d46 --- /dev/null +++ b/tests/nixos/functional/unprivileged-daemon.nix @@ -0,0 +1,43 @@ +{ lib, nixComponents, ... }: + +{ + name = "functional-tests-on-nixos_unprivileged-daemon"; + + imports = [ ./common.nix ]; + + nodes.machine = + { config, pkgs, ... }: + { + users.groups.nix-daemon = { }; + users.users.nix-daemon = { + isSystemUser = true; + group = "nix-daemon"; + }; + users.users.alice = { + isNormalUser = true; + }; + + nix = { + # We have to use nix-everything for nswrapper, nix-cli doesn't have it. + package = lib.mkForce nixComponents.nix-everything; + daemonUser = "nix-daemon"; + daemonGroup = "nix-daemon"; + settings.experimental-features = [ + "local-overlay-store" + "auto-allocate-uids" + ]; + }; + + # The store setting `ignore-gc-delete-failure` isn't set by default, + # but is needed since the daemon won't own the entire store. + systemd.services.nix-daemon.environment.NIX_REMOTE = + lib.mkForce "local?ignore-gc-delete-failure=true&use-roots-daemon=true"; + }; + + testScript = '' + machine.wait_for_unit("multi-user.target") + machine.succeed(""" + su --login --command "run-test-suite" alice >&2 + """) + ''; +} diff --git a/tests/nixos/remote-builds-ssh-ng.nix b/tests/nixos/remote-builds-ssh-ng.nix index c298ab92d46d..d23183f351c7 100644 --- a/tests/nixos/remote-builds-ssh-ng.nix +++ b/tests/nixos/remote-builds-ssh-ng.nix @@ -20,7 +20,7 @@ let builder = "''${utils}/bin/sh"; args = [ "-c" "${ lib.concatStringsSep "; " [ - ''if [[ -n $NIX_LOG_FD ]]'' + "if [[ -n $NIX_LOG_FD ]]" ''then echo '@nix {\"action\":\"setPhase\",\"phase\":\"buildPhase\"}' >&''$NIX_LOG_FD'' "fi" "echo Hello" diff --git a/tests/nixos/s3-binary-cache-store.nix b/tests/nixos/s3-binary-cache-store.nix index 8085d7b526f3..54ca03228597 100644 --- a/tests/nixos/s3-binary-cache-store.nix +++ b/tests/nixos/s3-binary-cache-store.nix @@ -126,7 +126,7 @@ in def verify_no_compression(machine, bucket, object_path): """Verify S3 object has no compression headers""" stat = machine.succeed(f"mc stat minio/{bucket}/{object_path}") - if "Content-Encoding" in stat and ("gzip" in stat or "xz" in stat): + if "Content-Encoding" in stat and ("gzip" in stat or "br" in stat): print(f"mc stat output for {object_path}:") print(stat) raise Exception(f"Object {object_path} should not have compression Content-Encoding") @@ -535,20 +535,20 @@ in @setup_s3() def test_compression_mixed(bucket): - """Test mixed compression (narinfo=xz, ls=gzip)""" - print("\n=== Testing Compression: mixed (narinfo=xz, ls=gzip) ===") + """Test mixed compression (narinfo=br, ls=gzip)""" + print("\n=== Testing Compression: mixed (narinfo=br, ls=gzip) ===") store_url = make_s3_url( bucket, - **{'narinfo-compression': 'xz', 'write-nar-listing': 'true', 'ls-compression': 'gzip'} + **{'narinfo-compression': 'br', 'write-nar-listing': 'true', 'ls-compression': 'gzip'} ) server.succeed(f"{ENV_WITH_CREDS} nix copy --to '{store_url}' {PKGS['C']}") pkg_hash = get_package_hash(PKGS['C']) - # Verify .narinfo has xz compression - verify_content_encoding(server, bucket, f"{pkg_hash}.narinfo", "xz") + # Verify .narinfo has br compression + verify_content_encoding(server, bucket, f"{pkg_hash}.narinfo", "br") # Verify .ls has gzip compression verify_content_encoding(server, bucket, f"{pkg_hash}.ls", "gzip") @@ -671,7 +671,7 @@ in ).strip() chunk_size = 5 * 1024 * 1024 - expected_parts = 3 # 10 MB raw becomes ~10.5 MB compressed (NAR + xz overhead) + expected_parts = 3 # 10 MB raw becomes ~10.5 MB compressed (NAR + br overhead) store_url = make_s3_url( bucket, @@ -753,7 +753,7 @@ in "multipart-upload": "true", "multipart-threshold": str(5 * 1024 * 1024), "multipart-chunk-size": str(5 * 1024 * 1024), - "log-compression": "xz", + "log-compression": "br", } ) @@ -953,6 +953,100 @@ in ) verify_packages_in_store(client, PKGS['A']) + @setup_s3( + populate_bucket=[PKGS['A']], + profiles={ + "valid": {"access_key": ACCESS_KEY, "secret_key": SECRET_KEY}, + "invalid": {"access_key": "INVALIDKEY", "secret_key": "INVALIDSECRET"}, + } + ) + def test_profile_credentials(bucket): + """Test that profile-based credentials work without environment variables""" + print("\n=== Testing Profile-Based Credentials ===") + + store_url = make_s3_url(bucket, profile="valid") + + # Verify store info works with profile credentials (no env vars) + client.succeed(f"HOME=/root nix store info --store '{store_url}' >&2") + + # Verify we can copy from the store using profile + verify_packages_in_store(client, PKGS['A'], should_exist=False) + client.succeed(f"HOME=/root nix copy --no-check-sigs --from '{store_url}' {PKGS['A']}") + verify_packages_in_store(client, PKGS['A']) + + # Clean up the package we just copied so we can test invalid profile + client.succeed(f"nix store delete --ignore-liveness {PKGS['A']}") + verify_packages_in_store(client, PKGS['A'], should_exist=False) + + # Verify invalid profile fails when trying to copy + invalid_url = make_s3_url(bucket, profile="invalid") + client.fail(f"HOME=/root nix copy --no-check-sigs --from '{invalid_url}' {PKGS['A']} 2>&1") + + @setup_s3( + populate_bucket=[PKGS['A']], + profiles={ + "wrong": {"access_key": "WRONGKEY", "secret_key": "WRONGSECRET"}, + } + ) + def test_env_vars_precedence(bucket): + """Test that environment variables take precedence over profile credentials""" + print("\n=== Testing Environment Variables Precedence ===") + + # Use profile with wrong credentials, but provide correct creds via env vars + store_url = make_s3_url(bucket, profile="wrong") + + # Ensure package is not in client store + verify_packages_in_store(client, PKGS['A'], should_exist=False) + + # This should succeed because env vars (correct) override profile (wrong) + output = client.succeed( + f"HOME=/root {ENV_WITH_CREDS} nix copy --no-check-sigs --debug --from '{store_url}' {PKGS['A']} 2>&1" + ) + + # Verify the credential chain shows Environment provider was added + if "Added AWS Environment Credential Provider" not in output: + print("Debug output:") + print(output) + raise Exception("Expected Environment provider to be added to chain") + + # Clean up the package so we can test again without env vars + client.succeed(f"nix store delete --ignore-liveness {PKGS['A']}") + verify_packages_in_store(client, PKGS['A'], should_exist=False) + + # Without env vars, same URL should fail (proving profile creds are actually wrong) + client.fail(f"HOME=/root nix copy --no-check-sigs --from '{store_url}' {PKGS['A']} 2>&1") + + @setup_s3( + populate_bucket=[PKGS['A']], + profiles={ + "testprofile": {"access_key": ACCESS_KEY, "secret_key": SECRET_KEY}, + } + ) + def test_credential_provider_chain(bucket): + """Test that debug logging shows which providers are added to the chain""" + print("\n=== Testing Credential Provider Chain Logging ===") + + store_url = make_s3_url(bucket, profile="testprofile") + + output = client.succeed( + f"HOME=/root nix store info --debug --store '{store_url}' 2>&1" + ) + + # For a named profile, we expect to see these providers in the chain + expected_providers = ["Environment", "Profile", "IMDS"] + for provider in expected_providers: + msg = f"Added AWS {provider} Credential Provider to chain for profile 'testprofile'" + if msg not in output: + print("Debug output:") + print(output) + raise Exception(f"Expected to find: {msg}") + + # SSO should be skipped (no SSO config for this profile) + if "Skipped AWS SSO Credential Provider for profile 'testprofile'" not in output: + print("Debug output:") + print(output) + raise Exception("Expected SSO provider to be skipped") + # ============================================================================ # Main Test Execution # ============================================================================ @@ -967,7 +1061,7 @@ in server.wait_for_unit("minio") server.wait_for_unit("network-addresses-eth1.service") server.wait_for_open_port(9000) - server.succeed(f"mc config host add minio http://localhost:9000 {ACCESS_KEY} {SECRET_KEY} --api s3v4") + server.succeed(f"mc alias set minio http://localhost:9000 {ACCESS_KEY} {SECRET_KEY} --api s3v4") # Run tests (each gets isolated bucket via decorator) test_credential_caching() diff --git a/tests/nixos/upgrade-nix.nix b/tests/nixos/upgrade-nix.nix new file mode 100644 index 000000000000..947fa4533121 --- /dev/null +++ b/tests/nixos/upgrade-nix.nix @@ -0,0 +1,104 @@ +# This installs an older Nix into a nix-env profile, and then runs `nix upgrade-nix` +# pointing at a local fallback-paths file containing the locally built nix. +# +# This is based on nixpkgs' nixosTests.nix-upgrade test. +# See https://github.com/NixOS/nixpkgs/blob/e3469a82fbd496d9c8e6192bbaf7cf056c6449ff/nixos/tests/nix/upgrade.nix. + +{ + config, + lib, + nixComponents, + system, + ... +}: +let + pkgs = config.nodes.machine.nixpkgs.pkgs; + + newNix = nixComponents.nix-cli; + oldNix = config.upgrade-nix.oldNix; + + fallback-paths = pkgs.writeTextDir "fallback-paths.nix" '' + { + ${system} = "${newNix}"; + } + ''; +in +{ + options.upgrade-nix.oldNix = lib.mkOption { + type = lib.types.package; + default = newNix; + description = "The Nix package to install before upgrading."; + }; + + config = { + name = "upgrade-nix"; + + nodes.machine = + { lib, ... }: + { + virtualisation.writableStore = true; + nix.settings.substituters = lib.mkForce [ ]; + nix.settings.hashed-mirrors = null; + nix.settings.connect-timeout = 1; + nix.extraOptions = "experimental-features = nix-command"; + environment.localBinInPath = true; + system.extraDependencies = [ + fallback-paths + newNix + oldNix + ]; + users.users.alice = { + isNormalUser = true; + packages = [ newNix ]; + }; + }; + + testScript = /* py */ '' + machine.start() + machine.wait_for_unit("multi-user.target") + + with subtest("nix-current"): + # Create a profile to pretend we are on non-NixOS + + print(machine.succeed("nix --version")) + machine.succeed("nix-env -i ${oldNix} -p /root/.local") + result = machine.succeed("/root/.local/bin/nix --version") + print(f"installed: {result}") + + with subtest("nix-upgrade"): + machine.succeed( + "nix upgrade-nix" + " --nix-store-paths-url file://${fallback-paths}/fallback-paths.nix" + " --profile /root/.local" + ) + result = machine.succeed("/root/.local/bin/nix --version") + print(f"after upgrade: {result}") + assert "${newNix.version}" in result, \ + f"expected ${newNix.version} in: {result}" + + with subtest("nix-build-with-mismatched-daemon"): + # The daemon is still running oldNix; verify the new client works. + machine.succeed( + "runuser -u alice -- nix build" + " --expr 'derivation { name = \"test\"; system = \"${system}\";" + " builder = \"/bin/sh\"; args = [\"-c\" \"echo test > $out\"]; }'" + " --print-out-paths" + ) + + with subtest("nix-upgrade-auto-detect"): + # Without passing in --profile, getProfileDir auto-detects the profile + # by finding nix-env in PATH and resolving the symlink chain back + # to the store. + machine.succeed("nix-env -i ${oldNix} -p /root/.local") + machine.succeed( + "PATH=/root/.local/bin:$PATH" + " ${newNix}/bin/nix upgrade-nix" + " --nix-store-paths-url file://${fallback-paths}/fallback-paths.nix" + ) + result = machine.succeed("/root/.local/bin/nix --version") + print(f"after auto-detect upgrade: {result}") + assert "${newNix.version}" in result, \ + f"expected ${newNix.version} in: {result}" + ''; + }; +} diff --git a/tests/repl-completion.nix b/tests/repl-completion.nix index 9ae37796bf57..29a48738f8b3 100644 --- a/tests/repl-completion.nix +++ b/tests/repl-completion.nix @@ -29,6 +29,25 @@ runCommand "repl-completion" exit 1 } } + # Regression https://github.com/NixOS/nix/issues/15133 + # Tab-completing an expression that throws a non-EvalError (e.g. + # JSONParseError from fromJSON) should not crash the REPL. + send "err1 = builtins.fromJSON \"nixnix\"\n" + expect "nix-repl>" + send "err1.\t" + sleep 0.5 + # Send Ctrl-C to cancel the current line and get a fresh prompt, + # since tab with no completions leaves the cursor on the same line. + send "\x03" + expect { + "nix-repl>" { + puts "Got another prompt after fromJSON error." + } + eof { + puts "REPL crashed after fromJSON tab-complete." + exit 1 + } + } exit 0 ''; passAsFile = [ "expectScript" ];