Skip to content

Commit bd70c1a

Browse files
makuru-orgmakuru-org
committed
Moved nixos config part of ssh-deploy.nix to the corresponding storage folder.
1 parent 176764e commit bd70c1a

3 files changed

Lines changed: 428 additions & 0 deletions

File tree

‎flake.nix‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,12 @@
4242
./nixos-modules/nomad.nix
4343
./nixos-modules/users.nix
4444
./nixos-modules/ssh-deployOptions.nix
45+
(import ./nixos-modules/seaweedf/ssh-deploy.nix {
46+
inherit microvm nixpkgs;
47+
})
48+
(import ./nixos-modules/ceph/ssh-deploy.nix {
49+
inherit microvm nixpkgs;
50+
})
4551
{
4652
nixpkgs.overlays = [
4753
nix-cache-cut.overlays.default

‎nixos-modules/storage/ceph/ssh-deploy.nix‎

Lines changed: 211 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,211 @@
1+
{ microvm, nixpkgs }:
2+
3+
{ config, lib, pkgs, ... }:
4+
5+
let
6+
inherit (config.skyflake.storage.ceph) cephfs;
7+
8+
debugShell = lib.optionalString config.skyflake.debug ''
9+
set -x
10+
'';
11+
12+
deployCommand = with pkgs; writeScript "skyflake-ssh-deploy" ''
13+
#! ${runtimeShell} -e
14+
${debugShell}
15+
16+
PATH=${lib.makeBinPath ([
17+
git
18+
])}:$PATH
19+
20+
if [[ "$SSH_ORIGINAL_COMMAND" =~ ^git-receive-pack\ \'([\\-_a-zA-Z0-9]+)\'$ ]]; then
21+
REPO="''${BASH_REMATCH[1]}"
22+
if ! [ -e $REPO ]; then
23+
echo "Creating $REPO anew..." >&2
24+
mkdir $REPO
25+
cd $REPO
26+
${git}/bin/git init --bare -b main >/dev/null
27+
else
28+
echo "Updating existing $REPO" >&2
29+
cd $REPO
30+
fi
31+
32+
SYSTEMS=$(mktemp --tmpdir -d deploy-systems-XXXXXXXX)
33+
34+
cat > hooks/update <<END_OF_HOOK
35+
#! ${runtimeShell} -e
36+
${debugShell}
37+
38+
PATH=${lib.makeBinPath ([
39+
git nix
40+
config.services.nomad.package
41+
])}:\$PATH
42+
43+
REF="\$1"
44+
REV="\$3"
45+
46+
if [[ "\$REF" =~ ^refs/heads/([\\-_a-zA-Z0-9]+)$ ]]; then
47+
NAME="\''${BASH_REMATCH[1]}"
48+
else
49+
echo "Invalid ref \$REF"
50+
exit 1
51+
fi
52+
53+
if [[ "\$REV" =~ ^0+\$ ]]; then
54+
# Deleting branch, stop microvm
55+
nomad job stop -namespace "$USER-$REPO" "\$NAME" || true
56+
exit 0
57+
fi
58+
59+
# the branch doesn't exist yet but is required by a git+file:///
60+
# flakeref. instead, we're archiving into a blank directory and add that
61+
# to the /nix/store so it can be used as a flakeref.
62+
FLAKETMP=\$(mktemp --tmpdir -d deploy-flake-\$NAME-XXXX)
63+
git archive \$REV | tar xp -C "\$FLAKETMP"
64+
FLAKE=\$(nix store add-path "\$FLAKETMP" -n "$REPO")
65+
rm -r "\$FLAKETMP"
66+
67+
echo "Skyflake is cooking $REPO#\$NAME"
68+
nix build -f /etc/skyflake/vm/build-vm.nix \
69+
-o "$SYSTEMS/\$NAME" \
70+
--extra-substituters file://${cfg.binaryCachePath}/?trusted=1 \
71+
--arg nixpkgsRef "\"${nixpkgs}\"" \
72+
--arg system "\"${pkgs.system}\"" \
73+
--arg datacenters '${lib.generators.toPretty {} cfg.datacenters}' \
74+
--arg user "\"\$USER\"" \
75+
--arg repo "\"$REPO\"" \
76+
--arg flakeRef "\"\$FLAKE\"" \
77+
--arg vmName "\"\$NAME\"" \
78+
--arg microvmFlake "\"${microvm}\""
79+
80+
SYSTEM=\$(readlink "$SYSTEMS/\$NAME")
81+
# Copy to shared store
82+
sudo nix copy --to file://${cfg.binaryCachePath} --no-check-sigs "\$SYSTEM"
83+
# Register gcroot
84+
mkdir -p "${cfg.sharedGcrootsPath}/$USER/$REPO"
85+
rm -f "${cfg.sharedGcrootsPath}/$USER/$REPO/\$NAME"
86+
rm -f "${cfg.sharedGcrootsPath}/$USER/$REPO/\$NAME"
87+
ln -s "\$SYSTEM" "${cfg.sharedGcrootsPath}/$USER/$REPO/\$NAME"
88+
89+
END_OF_HOOK
90+
chmod a+x hooks/update
91+
92+
# Run git operation
93+
GIT_DIR=. git-receive-pack .
94+
95+
cd $SYSTEMS
96+
if [ -z "$(ls -1A)" ]; then
97+
echo "No systems were built."
98+
exit 0
99+
fi
100+
101+
echo "Skyflake is launching machines:" >&2
102+
nomad namespace apply "$USER-$REPO" >/dev/null
103+
104+
for NAME in * ; do
105+
SYSTEM=$(readlink $NAME)
106+
echo $SYSTEM >&2
107+
nomad run -detach "$SYSTEM" >&2
108+
109+
# Register gcroot
110+
mkdir -p "${cfg.sharedGcrootsPath}/$USER/$REPO"
111+
rm -f "${cfg.sharedGcrootsPath}/$USER/$REPO/$NAME"
112+
ln -s "$SYSTEM" "${cfg.sharedGcrootsPath}/$USER/$REPO/$NAME"
113+
done
114+
cd -
115+
rm -r $SYSTEMS
116+
echo All done >&2
117+
118+
elif [[ "$SSH_ORIGINAL_COMMAND" =~ ^git-upload-pack\ \'([\\-_a-zA-Z0-9]+)\'$ ]]; then
119+
REPO="''${BASH_REMATCH[1]}"
120+
exec git-upload-pack "$REPO"
121+
122+
elif [[ "$SSH_ORIGINAL_COMMAND" = status ]]; then
123+
NAMESPACES=$(nomad namespace list -t "{{ range . }}{{ .Name }}
124+
{{ end }}"|grep -e "^$USER-")
125+
for NAMESPACE in $NAMESPACES ; do
126+
nomad job status -namespace "$NAMESPACE"
127+
done
128+
129+
else
130+
echo "Invalid SSH command: $SSH_ORIGINAL_COMMAND" >&2
131+
exit 1
132+
fi
133+
'';
134+
135+
sshKeyOpts = [
136+
"command=\"${deployCommand}\""
137+
"no-port-forwarding"
138+
"no-X11-forwarding"
139+
"no-agent-forwarding"
140+
"no-pty"
141+
"no-user-rc"
142+
"restrict"
143+
];
144+
cfg = config.skyflake.deploy;
145+
146+
in {
147+
config = {
148+
skyflake.storage.ceph.cephfs = lib.mkIf config.skyflake.storage.ceph.enable {
149+
${config.skyflake.deploy.binaryCachePath}.mountSource = "/skyflake-internals${config.skyflake.deploy.binaryCachePath}";
150+
${config.skyflake.deploy.sharedGcrootsPath}.mountSource = "/skyflake-internals${config.skyflake.deploy.sharedGcrootsPath}";
151+
};
152+
153+
services.openssh.enable = true;
154+
155+
users.users = builtins.mapAttrs (_: userConfig: {
156+
openssh.authorizedKeys.keys = map (sshKey:
157+
"${lib.concatStringsSep "," sshKeyOpts} ${sshKey}"
158+
) userConfig.sshKeys;
159+
}) config.skyflake.users // {
160+
# stable uid is useful across network filesystems
161+
microvm.uid = config.skyflake.microvmUid;
162+
};
163+
164+
environment.etc."skyflake/vm".source = pkgs.substituteAllFiles {
165+
src = ../vm;
166+
files = [ "." ];
167+
inherit (config.skyflake.deploy) binaryCachePath customizationModule;
168+
};
169+
170+
# lets the hook use $binaryCachePath
171+
nix.settings.trusted-users = builtins.attrNames config.skyflake.users;
172+
173+
# allowing commands to copy to/from shared store
174+
security.sudo = {
175+
enable = true;
176+
extraRules = [ {
177+
groups = [ "users" ];
178+
commands = [ {
179+
command = ''/run/current-system/sw/bin/nix copy --to file\://${cfg.binaryCachePath} *'';
180+
options = [ "NOPASSWD" ];
181+
} ];
182+
} ];
183+
};
184+
185+
systemd.tmpfiles.rules =
186+
[
187+
# workDir for nomad jobs
188+
"d /run/microvms 0700 microvm kvm - -"
189+
"d ${cfg.binaryCachePath} 0777 root root - -"
190+
]
191+
++
192+
map (userName:
193+
"d ${config.skyflake.deploy.sharedGcrootsPath}/${userName} 0750 ${userName} root - -"
194+
) (builtins.attrNames config.skyflake.users);
195+
196+
systemd.services.skyflake-permissions = {
197+
wantedBy = [ "multi-user.target" ];
198+
after = [ "remote-fs.target" ];
199+
script = ''
200+
mkdir -p ${cfg.binaryCachePath}
201+
chmod 0777 ${cfg.binaryCachePath}
202+
203+
${lib.concatMapStrings (userName: ''
204+
D="${config.skyflake.deploy.sharedGcrootsPath}/${userName}"
205+
mkdir -p "$D"
206+
chown "${userName}" "$D"
207+
'') (builtins.attrNames config.skyflake.users)}
208+
'';
209+
};
210+
};
211+
}

0 commit comments

Comments
 (0)