From 4ce241c88276a0892cc0907e805f2142b1f2b478 Mon Sep 17 00:00:00 2001 From: Wolfgang Walther Date: Sat, 30 Nov 2024 18:16:56 +0100 Subject: [PATCH] replaceVarsWith: init Takes the extended features of nix substituteAll to a replaceVars variant to get rid of those cases that use substituteAll to build a full package with meta information etc. --- nixos/modules/installer/tools/tools.nix | 2 +- pkgs/build-support/replace-vars/default.nix | 86 ------------ .../replace-vars/replace-vars-with.nix | 131 ++++++++++++++++++ .../replace-vars/replace-vars.nix | 36 +++++ .../deterministic-uname.sh | 2 +- .../de/deterministic-uname/package.nix | 54 ++++---- pkgs/by-name/ls/lsb-release/package.nix | 40 +++--- pkgs/by-name/ni/nixos-build-vms/package.nix | 12 +- .../ni/nixos-container/nixos-container.pl | 2 +- pkgs/by-name/ni/nixos-container/package.nix | 50 +++---- pkgs/by-name/ni/nixos-enter/package.nix | 14 +- pkgs/by-name/ni/nixos-install/package.nix | 19 +-- .../bsd/freebsd/pkgs/xargs-j/package.nix | 9 +- .../bsd/freebsd/pkgs/xargs-j/xargs-j.sh | 2 +- pkgs/top-level/all-packages.nix | 4 +- 15 files changed, 281 insertions(+), 182 deletions(-) delete mode 100644 pkgs/build-support/replace-vars/default.nix create mode 100644 pkgs/build-support/replace-vars/replace-vars-with.nix create mode 100644 pkgs/build-support/replace-vars/replace-vars.nix diff --git a/nixos/modules/installer/tools/tools.nix b/nixos/modules/installer/tools/tools.nix index 2eb70e5211b68..0e641fed8c938 100644 --- a/nixos/modules/installer/tools/tools.nix +++ b/nixos/modules/installer/tools/tools.nix @@ -43,7 +43,7 @@ let manPage = ./manpages/nixos-version.8; }; - nixos-install = pkgs.nixos-install.override { nix = config.nix.package; }; + nixos-install = pkgs.nixos-install.override { }; nixos-rebuild = pkgs.nixos-rebuild.override { nix = config.nix.package; }; defaultConfigTemplate = '' diff --git a/pkgs/build-support/replace-vars/default.nix b/pkgs/build-support/replace-vars/default.nix deleted file mode 100644 index 0088279ad9390..0000000000000 --- a/pkgs/build-support/replace-vars/default.nix +++ /dev/null @@ -1,86 +0,0 @@ -{ lib, stdenvNoCC }: - -/** - `replaceVars` is a wrapper around the [bash function `substitute`](https://nixos.org/manual/nixpkgs/stable/#fun-substitute) - in the stdenv. It allows for terse replacement of names in the specified path, while checking - for common mistakes such as naming a replacement that does nothing or forgetting a variable which - needs to be replaced. - - As with the [`--subst-var-by`](https://nixos.org/manual/nixpkgs/stable/#fun-substitute-subst-var-by) - flag, names are encoded as `@name@` in the provided file at the provided path. - - Any unmatched variable names in the file at the provided path will cause a build failure. - - By default, any remaining text that matches `@[A-Za-z_][0-9A-Za-z_'-]@` in the output after replacement - has occurred will cause a build failure. Variables can be excluded from this check by passing "null" for them. - - # Inputs - - `path` ([Store Path](https://nixos.org/manual/nix/latest/store/store-path.html#store-path) String) - : The file in which to replace variables. - - `attrs` (AttrsOf String) - : Each entry in this set corresponds to a `--subst-var-by` entry in [`substitute`](https://nixos.org/manual/nixpkgs/stable/#fun-substitute) or - null to keep it unchanged. - - # Example - - ```nix - { replaceVars }: - - replaceVars ./greeting.txt { world = "hello"; } - ``` - - See `../../test/replace-vars/default.nix` for tests of this function. -*/ -path: attrs: - -let - # We use `--replace-fail` instead of `--subst-var-by` so that if the thing isn't there, we fail. - subst-var-by = - name: value: - lib.optionals (value != null) [ - "--replace-fail" - (lib.escapeShellArg "@${name}@") - (lib.escapeShellArg value) - ]; - - replacements = lib.concatLists (lib.mapAttrsToList subst-var-by attrs); - - left-overs = map ({ name, ... }: name) ( - builtins.filter ({ value, ... }: value == null) (lib.attrsToList attrs) - ); -in - -stdenvNoCC.mkDerivation { - name = baseNameOf (toString path); - src = path; - doCheck = true; - dontUnpack = true; - preferLocalBuild = true; - allowSubstitutes = false; - - buildPhase = '' - runHook preBuild - substitute "$src" "$out" ${lib.concatStringsSep " " replacements} - runHook postBuild - ''; - - # Look for Nix identifiers surrounded by `@` that aren't substituted. - checkPhase = - let - lookahead = - if builtins.length left-overs == 0 then "" else "(?!${builtins.concatStringsSep "|" left-overs}@)"; - regex = lib.escapeShellArg "@${lookahead}[a-zA-Z_][0-9A-Za-z_'-]*@"; - in - '' - runHook preCheck - if grep -Pqe ${regex} "$out"; then - echo The following look like unsubstituted Nix identifiers that remain in "$out": - grep -Poe ${regex} "$out" - echo Use the more precise '`substitute`' function if this check is in error. - exit 1 - fi - runHook postCheck - ''; -} diff --git a/pkgs/build-support/replace-vars/replace-vars-with.nix b/pkgs/build-support/replace-vars/replace-vars-with.nix new file mode 100644 index 0000000000000..90ccf508f7ba0 --- /dev/null +++ b/pkgs/build-support/replace-vars/replace-vars-with.nix @@ -0,0 +1,131 @@ +{ lib, stdenvNoCC }: + +/** + `replaceVarsWith` is a wrapper around the [bash function `substitute`](https://nixos.org/manual/nixpkgs/stable/#fun-substitute) + in the stdenv. It allows for terse replacement of names in the specified path, while checking + for common mistakes such as naming a replacement that does nothing or forgetting a variable which + needs to be replaced. + + As with the [`--subst-var-by`](https://nixos.org/manual/nixpkgs/stable/#fun-substitute-subst-var-by) + flag, names are encoded as `@name@` in the provided file at the provided path. + + Any unmatched variable names in the file at the provided path will cause a build failure. + + By default, any remaining text that matches `@[A-Za-z_][0-9A-Za-z_'-]@` in the output after replacement + has occurred will cause a build failure. Variables can be excluded from this check by passing "null" for them. + + # Inputs + + `src` ([Store Path](https://nixos.org/manual/nix/latest/store/store-path.html#store-path) String) + : The file in which to replace variables. + + `replacements` (AttrsOf String) + : Each entry in this set corresponds to a `--subst-var-by` entry in [`substitute`](https://nixos.org/manual/nixpkgs/stable/#fun-substitute) or + null to keep it unchanged. + + `dir` (String) + : Sub directory in $out to store the result in. Commonly set to "bin". + + `isExecutable` (Boolean) + : Whether to mark the output file as executable. + + Most arguments supported by mkDerivation are also supported, with some exceptions for which + an error will be thrown. + + # Example + + ```nix + { replaceVarsWith }: + + replaceVarsWith { + src = ./my-setup-hook.sh; + replacements = { world = "hello"; }; + dir = "bin"; + isExecutable = true; + } + ``` + + See `../../test/replace-vars/default.nix` for tests of this function. Also see `replaceVars` for a short + version with src and replacements only. +*/ +{ + src, + replacements, + dir ? null, + isExecutable ? false, + ... +}@attrs: + +let + # We use `--replace-fail` instead of `--subst-var-by` so that if the thing isn't there, we fail. + subst-var-by = + name: value: + lib.optionals (value != null) [ + "--replace-fail" + (lib.escapeShellArg "@${name}@") + (lib.escapeShellArg value) + ]; + + substitutions = lib.concatLists (lib.mapAttrsToList subst-var-by replacements); + + left-overs = map ({ name, ... }: name) ( + builtins.filter ({ value, ... }: value == null) (lib.attrsToList replacements) + ); + + optionalAttrs = + if (builtins.intersectAttrs attrs forcedAttrs == { }) then + builtins.removeAttrs attrs [ "replacements" ] + else + throw "Passing any of ${builtins.concatStringsSep ", " (builtins.attrNames forcedAttrs)} to replaceVarsWith is not supported."; + + forcedAttrs = { + doCheck = true; + dontUnpack = true; + preferLocalBuild = true; + allowSubstitutes = false; + + buildPhase = '' + runHook preBuild + + target=$out + if test -n "$dir"; then + target=$out/$dir/$name + mkdir -p $out/$dir + fi + + substitute "$src" "$target" ${lib.concatStringsSep " " substitutions} + + if test -n "$isExecutable"; then + chmod +x $target + fi + + runHook postBuild + ''; + + # Look for Nix identifiers surrounded by `@` that aren't substituted. + checkPhase = + let + lookahead = + if builtins.length left-overs == 0 then "" else "(?!${builtins.concatStringsSep "|" left-overs}@)"; + regex = lib.escapeShellArg "@${lookahead}[a-zA-Z_][0-9A-Za-z_'-]*@"; + in + '' + runHook preCheck + if grep -Pqe ${regex} "$out"; then + echo The following look like unsubstituted Nix identifiers that remain in "$out": + grep -Poe ${regex} "$out" + echo Use the more precise '`substitute`' function if this check is in error. + exit 1 + fi + runHook postCheck + ''; + }; +in + +stdenvNoCC.mkDerivation ( + { + name = baseNameOf (toString src); + } + // optionalAttrs + // forcedAttrs +) diff --git a/pkgs/build-support/replace-vars/replace-vars.nix b/pkgs/build-support/replace-vars/replace-vars.nix new file mode 100644 index 0000000000000..4c9241187b82f --- /dev/null +++ b/pkgs/build-support/replace-vars/replace-vars.nix @@ -0,0 +1,36 @@ +{ replaceVarsWith }: + +/** + `replaceVars` is a wrapper around the [bash function `substitute`](https://nixos.org/manual/nixpkgs/stable/#fun-substitute) + in the stdenv. It allows for terse replacement of names in the specified path, while checking + for common mistakes such as naming a replacement that does nothing or forgetting a variable which + needs to be replaced. + + As with the [`--subst-var-by`](https://nixos.org/manual/nixpkgs/stable/#fun-substitute-subst-var-by) + flag, names are encoded as `@name@` in the provided file at the provided path. + + Any unmatched variable names in the file at the provided path will cause a build failure. + + By default, any remaining text that matches `@[A-Za-z_][0-9A-Za-z_'-]@` in the output after replacement + has occurred will cause a build failure. Variables can be excluded from this check by passing "null" for them. + + # Inputs + + `src` ([Store Path](https://nixos.org/manual/nix/latest/store/store-path.html#store-path) String) + : The file in which to replace variables. + + `replacements` (AttrsOf String) + : Each entry in this set corresponds to a `--subst-var-by` entry in [`substitute`](https://nixos.org/manual/nixpkgs/stable/#fun-substitute) or + null to keep it unchanged. + + # Example + + ```nix + { replaceVars }: + + replaceVars ./greeting.txt { world = "hello"; } + ``` + + See `../../test/replace-vars/default.nix` for tests of this function. +*/ +src: replacements: replaceVarsWith { inherit src replacements; } diff --git a/pkgs/by-name/de/deterministic-uname/deterministic-uname.sh b/pkgs/by-name/de/deterministic-uname/deterministic-uname.sh index cb6f419b03311..d4d5b333cac97 100644 --- a/pkgs/by-name/de/deterministic-uname/deterministic-uname.sh +++ b/pkgs/by-name/de/deterministic-uname/deterministic-uname.sh @@ -1,4 +1,4 @@ -#! @shell@ +#! @runtimeShell@ set -o errexit set -o nounset diff --git a/pkgs/by-name/de/deterministic-uname/package.nix b/pkgs/by-name/de/deterministic-uname/package.nix index 1a4e0a0f089b7..4a5c92f4a2c7a 100644 --- a/pkgs/by-name/de/deterministic-uname/package.nix +++ b/pkgs/by-name/de/deterministic-uname/package.nix @@ -1,14 +1,14 @@ -# expr and script based on our lsb_release { stdenv , lib -, substituteAll +, replaceVarsWith , coreutils , getopt +, runtimeShell , modDirVersion ? "" , forPlatform ? stdenv.buildPlatform }: -substituteAll { +replaceVarsWith { name = "uname"; src = ./deterministic-uname.sh; @@ -16,29 +16,31 @@ substituteAll { dir = "bin"; isExecutable = true; - inherit coreutils getopt; - - uSystem = if forPlatform.uname.system != null then forPlatform.uname.system else "unknown"; - inherit (forPlatform.uname) processor; - - # uname -o - # maybe add to lib/systems/default.nix uname attrset - # https://github.com/coreutils/coreutils/blob/7fc84d1c0f6b35231b0b4577b70aaa26bf548a7c/src/uname.c#L373-L374 - # https://stackoverflow.com/questions/61711186/where-does-host-operating-system-in-uname-c-comes-from - # https://github.com/coreutils/gnulib/blob/master/m4/host-os.m4 - operatingSystem = - if forPlatform.isLinux - then "GNU/Linux" - else if forPlatform.isDarwin - then "Darwin" # darwin isn't in host-os.m4 so where does this come from? - else if forPlatform.isFreeBSD - then "FreeBSD" - else "unknown"; - - # in os-specific/linux module packages - # --replace '$(shell uname -r)' "${kernel.modDirVersion}" \ - # is a common thing to do. - modDirVersion = if modDirVersion != "" then modDirVersion else "unknown"; + replacements = { + inherit coreutils getopt runtimeShell; + + uSystem = if forPlatform.uname.system != null then forPlatform.uname.system else "unknown"; + inherit (forPlatform.uname) processor; + + # uname -o + # maybe add to lib/systems/default.nix uname attrset + # https://github.com/coreutils/coreutils/blob/7fc84d1c0f6b35231b0b4577b70aaa26bf548a7c/src/uname.c#L373-L374 + # https://stackoverflow.com/questions/61711186/where-does-host-operating-system-in-uname-c-comes-from + # https://github.com/coreutils/gnulib/blob/master/m4/host-os.m4 + operatingSystem = + if forPlatform.isLinux + then "GNU/Linux" + else if forPlatform.isDarwin + then "Darwin" # darwin isn't in host-os.m4 so where does this come from? + else if forPlatform.isFreeBSD + then "FreeBSD" + else "unknown"; + + # in os-specific/linux module packages + # --replace '$(shell uname -r)' "${kernel.modDirVersion}" \ + # is a common thing to do. + modDirVersion = if modDirVersion != "" then modDirVersion else "unknown"; + }; meta = with lib; { description = "Print certain system information (hardcoded with lib/system values)"; diff --git a/pkgs/by-name/ls/lsb-release/package.nix b/pkgs/by-name/ls/lsb-release/package.nix index 6973d50e99aa9..9a4ce5ee4935e 100644 --- a/pkgs/by-name/ls/lsb-release/package.nix +++ b/pkgs/by-name/ls/lsb-release/package.nix @@ -1,26 +1,28 @@ { - replaceVars, - runCommand, + replaceVarsWith, lib, runtimeShell, coreutils, getopt, }: -runCommand "lsb_release" - { - meta = with lib; { - description = "Prints certain LSB (Linux Standard Base) and Distribution information"; - mainProgram = "lsb_release"; - license = [ licenses.mit ]; - maintainers = with maintainers; [ primeos ]; - platforms = platforms.linux; - }; - } - '' - install -Dm 555 ${ - replaceVars ./lsb_release.sh { - inherit runtimeShell coreutils getopt; - } - } $out/bin/lsb_release - '' +replaceVarsWith { + name = "lsb_release"; + + src = ./lsb_release.sh; + + dir = "bin"; + isExecutable = true; + + replacements = { + inherit coreutils getopt runtimeShell; + }; + + meta = with lib; { + description = "Prints certain LSB (Linux Standard Base) and Distribution information"; + mainProgram = "lsb_release"; + license = [ licenses.mit ]; + maintainers = with maintainers; [ primeos ]; + platforms = platforms.linux; + }; +} diff --git a/pkgs/by-name/ni/nixos-build-vms/package.nix b/pkgs/by-name/ni/nixos-build-vms/package.nix index fc8b4f403e94c..a3798232d0fc2 100644 --- a/pkgs/by-name/ni/nixos-build-vms/package.nix +++ b/pkgs/by-name/ni/nixos-build-vms/package.nix @@ -1,13 +1,17 @@ { - substituteAll, + replaceVarsWith, runtimeShell, installShellFiles, }: -substituteAll { +replaceVarsWith { name = "nixos-build-vms"; + src = ./nixos-build-vms.sh; - inherit runtimeShell; - buildVms = ./build-vms.nix; + + replacements = { + inherit runtimeShell; + buildVms = ./build-vms.nix; + }; dir = "bin"; isExecutable = true; diff --git a/pkgs/by-name/ni/nixos-container/nixos-container.pl b/pkgs/by-name/ni/nixos-container/nixos-container.pl index 5e504eca749ae..d6375f5db66f3 100755 --- a/pkgs/by-name/ni/nixos-container/nixos-container.pl +++ b/pkgs/by-name/ni/nixos-container/nixos-container.pl @@ -9,7 +9,7 @@ use Cwd 'abs_path'; use Time::HiRes; -my $nsenter = "@utillinux@/bin/nsenter"; +my $nsenter = "@util-linux@/bin/nsenter"; my $su = "@su@"; my $configurationDirectory = "@configurationDirectory@"; diff --git a/pkgs/by-name/ni/nixos-container/package.nix b/pkgs/by-name/ni/nixos-container/package.nix index e8b2f1dc88e59..eddcb03f7902d 100644 --- a/pkgs/by-name/ni/nixos-container/package.nix +++ b/pkgs/by-name/ni/nixos-container/package.nix @@ -1,4 +1,4 @@ -{ substituteAll +{ replaceVarsWith , perl , shadow , util-linux @@ -7,33 +7,35 @@ , nixosTests }: -substituteAll { - name = "nixos-container"; - dir = "bin"; - isExecutable = true; - src = ./nixos-container.pl; +replaceVarsWith { + name = "nixos-container"; + dir = "bin"; + isExecutable = true; + src = ./nixos-container.pl; + + replacements = { perl = perl.withPackages (p: [ p.FileSlurp ]); su = "${shadow.su}/bin/su"; - utillinux = util-linux; - inherit configurationDirectory stateDirectory; + inherit configurationDirectory stateDirectory util-linux; + }; - passthru = { - tests = { - inherit (nixosTests) - containers-imperative - containers-ip - containers-tmpfs - containers-ephemeral - containers-unified-hierarchy - ; - }; + passthru = { + tests = { + inherit (nixosTests) + containers-imperative + containers-ip + containers-tmpfs + containers-ephemeral + containers-unified-hierarchy + ; }; + }; - postInstall = '' - t=$out/share/bash-completion/completions - mkdir -p $t - cp ${./nixos-container-completion.sh} $t/nixos-container - ''; - meta.mainProgram = "nixos-container"; + postInstall = '' + t=$out/share/bash-completion/completions + mkdir -p $t + cp ${./nixos-container-completion.sh} $t/nixos-container + ''; + meta.mainProgram = "nixos-container"; } diff --git a/pkgs/by-name/ni/nixos-enter/package.nix b/pkgs/by-name/ni/nixos-enter/package.nix index 38fe2de663fd3..4839b3a340533 100644 --- a/pkgs/by-name/ni/nixos-enter/package.nix +++ b/pkgs/by-name/ni/nixos-enter/package.nix @@ -1,19 +1,21 @@ { lib, - substituteAll, + replaceVarsWith, runtimeShell, installShellFiles, util-linuxMinimal, }: -substituteAll { +replaceVarsWith { name = "nixos-enter"; src = ./nixos-enter.sh; - inherit runtimeShell; + replacements = { + inherit runtimeShell; - path = lib.makeBinPath [ - util-linuxMinimal - ]; + path = lib.makeBinPath [ + util-linuxMinimal + ]; + }; dir = "bin"; isExecutable = true; diff --git a/pkgs/by-name/ni/nixos-install/package.nix b/pkgs/by-name/ni/nixos-install/package.nix index fbe75619674ed..40c9c9790f6ae 100644 --- a/pkgs/by-name/ni/nixos-install/package.nix +++ b/pkgs/by-name/ni/nixos-install/package.nix @@ -1,24 +1,25 @@ { lib, - substituteAll, + replaceVarsWith, runtimeShell, installShellFiles, - nix, jq, nixos-enter, util-linuxMinimal, }: -substituteAll { +replaceVarsWith { name = "nixos-install"; src = ./nixos-install.sh; - inherit runtimeShell nix; + replacements = { + inherit runtimeShell; - path = lib.makeBinPath [ - jq - nixos-enter - util-linuxMinimal - ]; + path = lib.makeBinPath [ + jq + nixos-enter + util-linuxMinimal + ]; + }; dir = "bin"; isExecutable = true; diff --git a/pkgs/os-specific/bsd/freebsd/pkgs/xargs-j/package.nix b/pkgs/os-specific/bsd/freebsd/pkgs/xargs-j/package.nix index 3a6b0ff004287..4e7915c733e19 100644 --- a/pkgs/os-specific/bsd/freebsd/pkgs/xargs-j/package.nix +++ b/pkgs/os-specific/bsd/freebsd/pkgs/xargs-j/package.nix @@ -1,9 +1,12 @@ -{ substituteAll, runtimeShell }: +{ replaceVarsWith, runtimeShell }: -substituteAll { +replaceVarsWith { name = "xargs-j"; - shell = runtimeShell; src = ./xargs-j.sh; dir = "bin"; isExecutable = true; + + replacements = { + inherit runtimeShell; + }; } diff --git a/pkgs/os-specific/bsd/freebsd/pkgs/xargs-j/xargs-j.sh b/pkgs/os-specific/bsd/freebsd/pkgs/xargs-j/xargs-j.sh index 3dd27c2cd2cd7..be3d5f852cec8 100644 --- a/pkgs/os-specific/bsd/freebsd/pkgs/xargs-j/xargs-j.sh +++ b/pkgs/os-specific/bsd/freebsd/pkgs/xargs-j/xargs-j.sh @@ -1,4 +1,4 @@ -#! @shell@ +#! @runtimeShell@ declare -a args=() diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 30ebb0e1755db..f82acae4a0b57 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -770,7 +770,9 @@ with pkgs; cutoffPackages = [ newDependency ]; }; - replaceVars = callPackage ../build-support/replace-vars { }; + replaceVarsWith = callPackage ../build-support/replace-vars/replace-vars-with.nix { }; + + replaceVars = callPackage ../build-support/replace-vars/replace-vars.nix { }; replaceDirectDependencies = callPackage ../build-support/replace-direct-dependencies.nix { };