Skip to content

Commit 21f17f0

Browse files
committed
chore(nix): Add nix scripts
1 parent 90a6c76 commit 21f17f0

8 files changed

Lines changed: 14042 additions & 0 deletions

File tree

Cargo.nix

Lines changed: 13604 additions & 0 deletions
Large diffs are not rendered by default.

crate-hashes.json

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
{
2+
"git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.92.0#k8s-version@0.1.2": "0li9smdrz7danqz17lfkl0j9zl2i84csgc7d01lxs5qi8jcs9fzw",
3+
"git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.92.0#stackable-operator-derive@0.3.1": "0li9smdrz7danqz17lfkl0j9zl2i84csgc7d01lxs5qi8jcs9fzw",
4+
"git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.92.0#stackable-operator@0.92.0": "0li9smdrz7danqz17lfkl0j9zl2i84csgc7d01lxs5qi8jcs9fzw",
5+
"git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.92.0#stackable-shared@0.0.1": "0li9smdrz7danqz17lfkl0j9zl2i84csgc7d01lxs5qi8jcs9fzw",
6+
"git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.92.0#stackable-telemetry@0.6.0": "0li9smdrz7danqz17lfkl0j9zl2i84csgc7d01lxs5qi8jcs9fzw",
7+
"git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.92.0#stackable-versioned-macros@0.7.1": "0li9smdrz7danqz17lfkl0j9zl2i84csgc7d01lxs5qi8jcs9fzw",
8+
"git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.92.0#stackable-versioned@0.7.1": "0li9smdrz7danqz17lfkl0j9zl2i84csgc7d01lxs5qi8jcs9fzw",
9+
"git+https://github.com/stackabletech/product-config.git?tag=0.7.0#product-config@0.7.0": "0gjsm80g6r75pm3824dcyiz4ysq1ka4c1if6k1mjm9cnd5ym0gny"
10+
}

default.nix

Lines changed: 119 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,119 @@
1+
{ sources ? import ./nix/sources.nix # managed by https://github.com/nmattia/niv
2+
, nixpkgs ? sources.nixpkgs
3+
, pkgs ? import nixpkgs {}
4+
, cargo ? import ./Cargo.nix {
5+
inherit nixpkgs pkgs; release = false;
6+
defaultCrateOverrides = pkgs.defaultCrateOverrides // {
7+
prost-build = attrs: {
8+
buildInputs = [ pkgs.protobuf ];
9+
};
10+
tonic-reflection = attrs: {
11+
buildInputs = [ pkgs.rustfmt ];
12+
};
13+
csi-grpc = attrs: {
14+
nativeBuildInputs = [ pkgs.protobuf ];
15+
};
16+
stackable-secret-operator = attrs: {
17+
buildInputs = [ pkgs.protobuf pkgs.rustfmt ];
18+
};
19+
stackable-opa-user-info-fetcher = attrs: {
20+
# TODO: why is this not pulled in via libgssapi-sys?
21+
buildInputs = [ pkgs.krb5 ];
22+
};
23+
krb5-sys = attrs: {
24+
nativeBuildInputs = [ pkgs.pkg-config ];
25+
buildInputs = [ pkgs.krb5 ];
26+
LIBCLANG_PATH = "${pkgs.libclang.lib}/lib";
27+
# Clang's resource directory is located at ${pkgs.clang.cc.lib}/lib/clang/<version>.
28+
# Starting with Clang 16, only the major version is used for the resource directory,
29+
# whereas the full version was used in prior Clang versions (see
30+
# https://github.com/llvm/llvm-project/commit/e1b88c8a09be25b86b13f98755a9bd744b4dbf14).
31+
# The clang wrapper ${pkgs.clang} provides a symlink to the resource directory, which
32+
# we use instead.
33+
BINDGEN_EXTRA_CLANG_ARGS = "-I${pkgs.glibc.dev}/include -I${pkgs.clang}/resource-root/include";
34+
};
35+
libgssapi-sys = attrs: {
36+
buildInputs = [ pkgs.krb5 ];
37+
LIBCLANG_PATH = "${pkgs.libclang.lib}/lib";
38+
BINDGEN_EXTRA_CLANG_ARGS = "-I${pkgs.glibc.dev}/include -I${pkgs.clang}/resource-root/include";
39+
};
40+
};
41+
}
42+
, meta ? pkgs.lib.importJSON ./nix/meta.json
43+
, dockerName ? "oci.stackable.tech/sandbox/${meta.operator.name}"
44+
, dockerTag ? null
45+
}:
46+
rec {
47+
inherit cargo sources pkgs meta;
48+
build = cargo.allWorkspaceMembers;
49+
entrypoint = build+"/bin/stackable-${meta.operator.name}";
50+
crds = pkgs.runCommand "${meta.operator.name}-crds.yaml" {}
51+
''
52+
${entrypoint} crd > $out
53+
'';
54+
55+
dockerImage = pkgs.dockerTools.streamLayeredImage {
56+
name = dockerName;
57+
tag = dockerTag;
58+
contents = [
59+
# Common debugging tools
60+
pkgs.bashInteractive pkgs.coreutils pkgs.util-linuxMinimal
61+
# Kerberos 5 must be installed globally to load plugins correctly
62+
pkgs.krb5
63+
# Make the whole cargo workspace available on $PATH
64+
build
65+
];
66+
config = {
67+
Env =
68+
let
69+
fileRefVars = {
70+
PRODUCT_CONFIG = deploy/config-spec/properties.yaml;
71+
};
72+
in pkgs.lib.concatLists (pkgs.lib.mapAttrsToList (env: path: pkgs.lib.optional (pkgs.lib.pathExists path) "${env}=${path}") fileRefVars);
73+
Entrypoint = [ entrypoint ];
74+
Cmd = [ "run" ];
75+
};
76+
};
77+
docker = pkgs.linkFarm "opensearch-operator-docker" [
78+
{
79+
name = "load-image";
80+
path = dockerImage;
81+
}
82+
{
83+
name = "ref";
84+
path = pkgs.writeText "${dockerImage.name}-image-tag" "${dockerImage.imageName}:${dockerImage.imageTag}";
85+
}
86+
{
87+
name = "image-repo";
88+
path = pkgs.writeText "${dockerImage.name}-repo" dockerImage.imageName;
89+
}
90+
{
91+
name = "image-tag";
92+
path = pkgs.writeText "${dockerImage.name}-tag" dockerImage.imageTag;
93+
}
94+
{
95+
name = "crds.yaml";
96+
path = crds;
97+
}
98+
];
99+
100+
# need to use vendored crate2nix because of https://github.com/kolloch/crate2nix/issues/264
101+
crate2nix = import sources.crate2nix {};
102+
tilt = pkgs.tilt;
103+
104+
regenerateNixLockfiles = pkgs.writeScriptBin "regenerate-nix-lockfiles"
105+
''
106+
#!/usr/bin/env bash
107+
set -euo pipefail
108+
echo Running crate2nix
109+
${crate2nix}/bin/crate2nix generate
110+
111+
# crate2nix adds a trailing newline (see
112+
# https://github.com/nix-community/crate2nix/commit/5dd04e6de2fbdbeb067ab701de8ec29bc228c389).
113+
# The pre-commit hook trailing-whitespace wants to remove it again
114+
# (see https://github.com/pre-commit/pre-commit-hooks?tab=readme-ov-file#trailing-whitespace).
115+
# So, remove the trailing newline already here to avoid that an
116+
# unnecessary change is shown in Git.
117+
sed -i '$d' Cargo.nix
118+
'';
119+
}

nix/README.md

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
<!--
2+
This file is automatically generated from the templates in stackabletech/operator-templating
3+
DO NOT MANUALLY EDIT THIS FILE
4+
-->
5+
6+
# Updating nix dependencies
7+
8+
## Run the following for an operator
9+
10+
> [!NOTE]
11+
> We track the `master` branch of crate2nix as that is relatively up to date, but the releases are infrequent.
12+
13+
```shell
14+
niv update crate2nix
15+
niv update nixpkgs
16+
niv update beku.py -b X.Y.Z # Using the release tag
17+
```
18+
19+
### Test
20+
21+
- Run make `regenerate-nix` to ensure crate2nix works
22+
- Run a smoke test to ensure beku.py works.
23+
- Run `make run-dev` to ensure nixpkgs are fine.
24+
25+
## Update operator-templating
26+
27+
Do the same as above, but from `template/`

nix/meta.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
{"operator": {"name": "zookeeper-operator", "pretty_string": "Apache ZooKeeper", "product_string": "zookeeper", "url": "stackabletech/zookeeper-operator.git"}}

nix/sources.json

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
{
2+
"beku.py": {
3+
"branch": "0.0.10",
4+
"description": "Test suite expander for Stackable Kuttl tests.",
5+
"homepage": null,
6+
"owner": "stackabletech",
7+
"repo": "beku.py",
8+
"rev": "fc75202a38529a4ac6776dd8a5dfee278d927f58",
9+
"sha256": "152yary0p11h87yabv74jnwkghsal7lx16az0qlzrzdrs6n5v8id",
10+
"type": "tarball",
11+
"url": "https://github.com/stackabletech/beku.py/archive/fc75202a38529a4ac6776dd8a5dfee278d927f58.tar.gz",
12+
"url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
13+
},
14+
"crate2nix": {
15+
"branch": "master",
16+
"description": "nix build file generator for rust crates",
17+
"homepage": "",
18+
"owner": "kolloch",
19+
"repo": "crate2nix",
20+
"rev": "be31feae9a82c225c0fd1bdf978565dc452a483a",
21+
"sha256": "14d0ymlrwk7dynv35qcw4xn0dylfpwjmf6f8znflbk2l6fk23l12",
22+
"type": "tarball",
23+
"url": "https://github.com/kolloch/crate2nix/archive/be31feae9a82c225c0fd1bdf978565dc452a483a.tar.gz",
24+
"url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
25+
},
26+
"nixpkgs": {
27+
"branch": "nixpkgs-unstable",
28+
"description": "Nix Packages collection",
29+
"homepage": "",
30+
"owner": "NixOS",
31+
"repo": "nixpkgs",
32+
"rev": "b0b4b5f8f621bfe213b8b21694bab52ecfcbf30b",
33+
"sha256": "1y8kwbb5b0r1m88nk871ai56qi2drygvibjgc2swp48jfyp5ya99",
34+
"type": "tarball",
35+
"url": "https://github.com/NixOS/nixpkgs/archive/b0b4b5f8f621bfe213b8b21694bab52ecfcbf30b.tar.gz",
36+
"url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
37+
}
38+
}

nix/sources.nix

Lines changed: 198 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,198 @@
1+
# This file has been generated by Niv.
2+
3+
let
4+
5+
#
6+
# The fetchers. fetch_<type> fetches specs of type <type>.
7+
#
8+
9+
fetch_file = pkgs: name: spec:
10+
let
11+
name' = sanitizeName name + "-src";
12+
in
13+
if spec.builtin or true then
14+
builtins_fetchurl { inherit (spec) url sha256; name = name'; }
15+
else
16+
pkgs.fetchurl { inherit (spec) url sha256; name = name'; };
17+
18+
fetch_tarball = pkgs: name: spec:
19+
let
20+
name' = sanitizeName name + "-src";
21+
in
22+
if spec.builtin or true then
23+
builtins_fetchTarball { name = name'; inherit (spec) url sha256; }
24+
else
25+
pkgs.fetchzip { name = name'; inherit (spec) url sha256; };
26+
27+
fetch_git = name: spec:
28+
let
29+
ref =
30+
spec.ref or (
31+
if spec ? branch then "refs/heads/${spec.branch}" else
32+
if spec ? tag then "refs/tags/${spec.tag}" else
33+
abort "In git source '${name}': Please specify `ref`, `tag` or `branch`!"
34+
);
35+
submodules = spec.submodules or false;
36+
submoduleArg =
37+
let
38+
nixSupportsSubmodules = builtins.compareVersions builtins.nixVersion "2.4" >= 0;
39+
emptyArgWithWarning =
40+
if submodules
41+
then
42+
builtins.trace
43+
(
44+
"The niv input \"${name}\" uses submodules "
45+
+ "but your nix's (${builtins.nixVersion}) builtins.fetchGit "
46+
+ "does not support them"
47+
)
48+
{ }
49+
else { };
50+
in
51+
if nixSupportsSubmodules
52+
then { inherit submodules; }
53+
else emptyArgWithWarning;
54+
in
55+
builtins.fetchGit
56+
({ url = spec.repo; inherit (spec) rev; inherit ref; } // submoduleArg);
57+
58+
fetch_local = spec: spec.path;
59+
60+
fetch_builtin-tarball = name: throw
61+
''[${name}] The niv type "builtin-tarball" is deprecated. You should instead use `builtin = true`.
62+
$ niv modify ${name} -a type=tarball -a builtin=true'';
63+
64+
fetch_builtin-url = name: throw
65+
''[${name}] The niv type "builtin-url" will soon be deprecated. You should instead use `builtin = true`.
66+
$ niv modify ${name} -a type=file -a builtin=true'';
67+
68+
#
69+
# Various helpers
70+
#
71+
72+
# https://github.com/NixOS/nixpkgs/pull/83241/files#diff-c6f540a4f3bfa4b0e8b6bafd4cd54e8bR695
73+
sanitizeName = name:
74+
(
75+
concatMapStrings (s: if builtins.isList s then "-" else s)
76+
(
77+
builtins.split "[^[:alnum:]+._?=-]+"
78+
((x: builtins.elemAt (builtins.match "\\.*(.*)" x) 0) name)
79+
)
80+
);
81+
82+
# The set of packages used when specs are fetched using non-builtins.
83+
mkPkgs = sources: system:
84+
let
85+
sourcesNixpkgs =
86+
import (builtins_fetchTarball { inherit (sources.nixpkgs) url sha256; }) { inherit system; };
87+
hasNixpkgsPath = builtins.any (x: x.prefix == "nixpkgs") builtins.nixPath;
88+
hasThisAsNixpkgsPath = <nixpkgs> == ./.;
89+
in
90+
if builtins.hasAttr "nixpkgs" sources
91+
then sourcesNixpkgs
92+
else if hasNixpkgsPath && ! hasThisAsNixpkgsPath then
93+
import <nixpkgs> { }
94+
else
95+
abort
96+
''
97+
Please specify either <nixpkgs> (through -I or NIX_PATH=nixpkgs=...) or
98+
add a package called "nixpkgs" to your sources.json.
99+
'';
100+
101+
# The actual fetching function.
102+
fetch = pkgs: name: spec:
103+
104+
if ! builtins.hasAttr "type" spec then
105+
abort "ERROR: niv spec ${name} does not have a 'type' attribute"
106+
else if spec.type == "file" then fetch_file pkgs name spec
107+
else if spec.type == "tarball" then fetch_tarball pkgs name spec
108+
else if spec.type == "git" then fetch_git name spec
109+
else if spec.type == "local" then fetch_local spec
110+
else if spec.type == "builtin-tarball" then fetch_builtin-tarball name
111+
else if spec.type == "builtin-url" then fetch_builtin-url name
112+
else
113+
abort "ERROR: niv spec ${name} has unknown type ${builtins.toJSON spec.type}";
114+
115+
# If the environment variable NIV_OVERRIDE_${name} is set, then use
116+
# the path directly as opposed to the fetched source.
117+
replace = name: drv:
118+
let
119+
saneName = stringAsChars (c: if (builtins.match "[a-zA-Z0-9]" c) == null then "_" else c) name;
120+
ersatz = builtins.getEnv "NIV_OVERRIDE_${saneName}";
121+
in
122+
if ersatz == "" then drv else
123+
# this turns the string into an actual Nix path (for both absolute and
124+
# relative paths)
125+
if builtins.substring 0 1 ersatz == "/" then /. + ersatz else /. + builtins.getEnv "PWD" + "/${ersatz}";
126+
127+
# Ports of functions for older nix versions
128+
129+
# a Nix version of mapAttrs if the built-in doesn't exist
130+
mapAttrs = builtins.mapAttrs or (
131+
f: set: with builtins;
132+
listToAttrs (map (attr: { name = attr; value = f attr set.${attr}; }) (attrNames set))
133+
);
134+
135+
# https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/lists.nix#L295
136+
range = first: last: if first > last then [ ] else builtins.genList (n: first + n) (last - first + 1);
137+
138+
# https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/strings.nix#L257
139+
stringToCharacters = s: map (p: builtins.substring p 1 s) (range 0 (builtins.stringLength s - 1));
140+
141+
# https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/strings.nix#L269
142+
stringAsChars = f: s: concatStrings (map f (stringToCharacters s));
143+
concatMapStrings = f: list: concatStrings (map f list);
144+
concatStrings = builtins.concatStringsSep "";
145+
146+
# https://github.com/NixOS/nixpkgs/blob/8a9f58a375c401b96da862d969f66429def1d118/lib/attrsets.nix#L331
147+
optionalAttrs = cond: as: if cond then as else { };
148+
149+
# fetchTarball version that is compatible between all the versions of Nix
150+
builtins_fetchTarball = { url, name ? null, sha256 }@attrs:
151+
let
152+
inherit (builtins) lessThan nixVersion fetchTarball;
153+
in
154+
if lessThan nixVersion "1.12" then
155+
fetchTarball ({ inherit url; } // (optionalAttrs (name != null) { inherit name; }))
156+
else
157+
fetchTarball attrs;
158+
159+
# fetchurl version that is compatible between all the versions of Nix
160+
builtins_fetchurl = { url, name ? null, sha256 }@attrs:
161+
let
162+
inherit (builtins) lessThan nixVersion fetchurl;
163+
in
164+
if lessThan nixVersion "1.12" then
165+
fetchurl ({ inherit url; } // (optionalAttrs (name != null) { inherit name; }))
166+
else
167+
fetchurl attrs;
168+
169+
# Create the final "sources" from the config
170+
mkSources = config:
171+
mapAttrs
172+
(
173+
name: spec:
174+
if builtins.hasAttr "outPath" spec
175+
then
176+
abort
177+
"The values in sources.json should not have an 'outPath' attribute"
178+
else
179+
spec // { outPath = replace name (fetch config.pkgs name spec); }
180+
)
181+
config.sources;
182+
183+
# The "config" used by the fetchers
184+
mkConfig =
185+
{ sourcesFile ? if builtins.pathExists ./sources.json then ./sources.json else null
186+
, sources ? if sourcesFile == null then { } else builtins.fromJSON (builtins.readFile sourcesFile)
187+
, system ? builtins.currentSystem
188+
, pkgs ? mkPkgs sources system
189+
}: rec {
190+
# The sources, i.e. the attribute set of spec name to spec
191+
inherit sources;
192+
193+
# The "pkgs" (evaluated nixpkgs) to use for e.g. non-builtin fetchers
194+
inherit pkgs;
195+
};
196+
197+
in
198+
mkSources (mkConfig { }) // { __functor = _: settings: mkSources (mkConfig settings); }

0 commit comments

Comments
 (0)