Start of modernization of skyflake.

Author: makuru-org

example-server.nix

@@ -1,10 +1,10 @@
 { instance }:
 
-{ config, lib, pkgs, ... }:
+{ config, pkgs, ... }:
 
 {
   microvm = {
-    vcpu = 2;
+    vcpu = 4;
     mem = 4096;
 
     shares = [ {
@@ -34,7 +34,7 @@
   networking.hostName = "example${toString instance}";
   users.users.root.password = "";
 
-  # TODO:
+  # TODO:? Shoulld firewall be fixed?
   networking.firewall.enable = false;
 
   networking.useDHCP = false;
@@ -64,7 +64,9 @@
           IPv6AcceptRA = true;
         };
         addresses = [ {
-          addressConfig.Address = "fec0::${toString instance}/64";
+          # TODO: addressConfig needs to be removed.
+          # trace: warning: Using 'addressConfig' is deprecated! Move all attributes inside one level up and remove it.
+          addressConfig.Address = "fec0::${toString instance}/64"; # 
         } ];
       };
     };
@@ -78,22 +80,29 @@
       }) [ 1 2 3 ]
     );
 
-    storage.ceph = rec {
-      fsid = "8364da79-5e03-49ae-82ea-7d936278cb0f";
-      monKeyring = example/ceph.mon.keyring;
-      adminKeyring = example/ceph.client.admin.keyring;
-      osds = [ {
-        id = instance;
-        fsid = "8e4ae689-5c15-4381-bd75-19de743378e${toString instance}";
-        path = "/dev/vdb";
-        deviceClass = "ssd";
-        keyfile = toString (./example + "/osd.${toString instance}.keyring");
-      } ];
-      rbdPools.microvms = {
-        params = { size = 2; class = "ssd"; };
+
+    storage.seaweedfs = {
+      enable = true;
+      filer.db.etcd = {  
+        enable = true;
       };
-      cephfs.skyflake.metaParams = { size = 2; class = "ssd"; };
     };
+    #storage.ceph = {
+    #  fsid = "8364da79-5e03-49ae-82ea-7d936278cb0f";
+    #  monKeyring = example/ceph.mon.keyring;
+    #  adminKeyring = example/ceph.client.admin.keyring;
+    #  osds = [ {
+    #    id = instance;
+    #    fsid = "8e4ae689-5c15-4381-bd75-19de743378e${toString instance}";
+    #    path = "/dev/vdb";
+    #    deviceClass = "ssd";
+    #    keyfile = toString (./example + "/osd.${toString instance}.keyring");
+    #  } ];
+    #  rbdPools.microvms = {
+    #    params = { size = 2; class = "ssd"; };
+    #  };
+    #  cephfs.skyflake.metaParams = { size = 2; class = "ssd"; };
+    #};
 
     nomad = {
       servers = [ "example1" "example2" "example3" ];
@@ -107,6 +116,7 @@
         uid = 1000;
         sshKeys = [
           "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGJJTSJdpDh82486uPiMhhyhnci4tScp5uUe7156MBC8 astro"
+          "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPRRdToCDUupkkwI+crB3fGDwdBIFkDsBHjOImn+qsjg openpgp:0xE8D3D833"
         ];
       };
     };
@@ -115,4 +125,4 @@
   environment.systemPackages = with pkgs; [
     tcpdump
   ];
-}
+}

flake.lock

@@ -2,8 +2,12 @@
   description = "Hyperconverged Infratructure for NixOS";
 
   inputs = {
-    microvm.url = "github:astro/microvm.nix";
-    microvm.inputs.nixpkgs.follows = "nixpkgs";
+    nixpkgs.url = "github:nixos/nixpkgs/nixos-unstable";
+    microvm = {
+      url = "github:astro/microvm.nix";
+      inputs.nixpkgs.follows = "nixpkgs";
+    };
+
     nix-cache-cut.url = "github:astro/nix-cache-cut";
   };
 
@@ -14,14 +18,18 @@
       pkgs = nixpkgs.legacyPackages.${system};
 
     in {
+      formatter.${system} = pkgs.alejandra;
       packages.${system} = import ./pkgs/doc.nix {
         inherit pkgs self;
       };
 
       nixosModules = {
         default = {
           imports = [
-            ./nixos-modules/storage/ceph/server.nix
+            ./nixos-modules/storage/seaweedfs/options.nix
+            # ./nixos-modules/storage/seaweedfs/dbBackend/sqlite.nix
+            ./nixos-modules/storage/seaweedfs/dbBackend/etcd.nix
+            # ./nixos-modules/storage/ceph/server.nix
             ./nixos-modules/defaults.nix
             ./nixos-modules/nodes.nix
             ./nixos-modules/nomad.nix
@@ -71,9 +79,9 @@
 
           make-ceph = {
             type = "app";
-            program = toString (pkgs.callPackage ./pkgs/make-ceph.nix {});
+            #program = toString (pkgs.callPackage ./pkgs/make-ceph.nix {});
           };
 
         };
     };
-}
+}

flake.nix

@@ -41,7 +41,7 @@ in
   systemd.services.skyflake-install-cache-gc = {
     wantedBy = [ "multi-user.target" ];
     requires = [ "nomad.service" ];
-    path = with pkgs; [ nomad ];
+    path = [ config.services.nomad.package ];
     script = ''
       nomad run -detach ${jobFile}
     '';

nixos-modules/cache-cut.nix

@@ -9,7 +9,7 @@ in
     description = ''
       All cluster nodes with their addresses, or at least those who
       run coordination servers (eg. nomad servers, ceph server,
-      ...).
+       seaweedfs server ...).
     '';
     default = {};
     type = types.attrsOf (types.submodule {

nixos-modules/nodes.nix

@@ -42,7 +42,7 @@ in
   config = {
     services.nomad = {
       enable = true;
-      package = pkgs.nomad_1_4;
+      package = pkgs.nomad_1_6; # nomad 1.6 is the newest version under an foss license.
       dropPrivileges = false;
       enableDocker = false;
 
@@ -75,7 +75,7 @@ in
       # alternatives to the nomad web ui
       wander damon
       # needed for microvms
-      virtiofsd ceph
+      virtiofsd ceph seaweedfs
       jq kmod e2fsprogs
     ];
   };

nixos-modules/nomad.nix

@@ -3,7 +3,7 @@
 { config, lib, pkgs, ... }:
 
 let
-  inherit (config.skyflake.storage.ceph) cephfs;
+  inherit (config.skyflake.storage.seaweedfs) seaweedfsMount;
 
   debugShell = lib.optionalString config.skyflake.debug ''
     set -x
@@ -143,8 +143,6 @@ let
   ];
 
   cfg = config.skyflake.deploy;
-  gcCfg = config.skyflake.gc;
-
 in {
   options.skyflake = with lib; {
     deploy = {
@@ -158,7 +156,7 @@ in {
 
       binaryCachePath = mkOption {
         type = types.str;
-        default = cephfs.skyflake-binary-cache.mountPoint;
+        default = seaweedfsMount.skyflake-binary-cache.mountPoint;
         description = ''
           Directory which is mounted on all nodes that will be used to
           share the /nix/store with MicroVMs.
@@ -167,7 +165,7 @@ in {
 
       sharedGcrootsPath = mkOption {
         type = types.str;
-        default = cephfs.skyflake-gcroots.mountPoint;
+        default = seaweedfsMount.skyflake-gcroots.mountPoint;
         description = ''
           Directory which is mounted on all nodes, is linked from
           /nix/var/nix/gcroots/, and contains links to all currently
@@ -211,9 +209,9 @@ in {
         '';
     };
   };
-
+ 
   config = {
-    skyflake.storage.ceph.cephfs = {
+    skyflake.storage.seaweedfs.seaweedfsMount = {
       skyflake-binary-cache.mountPoint = "/var/lib/skyflake/binary-cache";
       skyflake-gcroots.mountPoint = "/nix/var/nix/gcroots/skyflake";
     };

nixos-modules/ssh-deploy.nix

@@ -51,6 +51,11 @@ let
 
 in {
   options.skyflake.storage.ceph = {
+    enable = lib.mkOption {
+      type = lib.types.bool;
+      default = false;
+      example = true;
+    };
     package = lib.mkPackageOption pkgs "ceph" { };
     fsid = lib.mkOption {
       type = lib.types.str;
@@ -100,7 +105,7 @@ in {
     };
     rbdPools = lib.mkOption {
       default = {};
-      type = with lib.types; attrsOf (submodule ({ name, ... }: {
+      type = with lib.types; attrsOf (submodule ({ ... }: {
         options = {
           params = poolParamsOpts;
         };
@@ -146,7 +151,9 @@ in {
         publicNetwork = clusterNetwork; #"0.0.0.0/0, ::/0";
         clusterNetwork = lib.concatStringsSep ", " (
           lib.concatMap ({ addresses ? [], ... }:
-            lib.concatMap ({ addressConfig ? {}, ... }:
+            # Needs to be changed because of 
+            # trace: warning: Using 'addressConfig' is deprecated! Move all attributes inside one level up and remove it.
+            lib.concatMap ({ addressConfig ? {}, ... }: 
               if addressConfig ? Address
               then [ addressConfig.Address ]
               else []
@@ -387,4 +394,4 @@ in {
       ) cfg.cephfs
     ));
   };
-}
+}