Feat support redhat (#88)

* feat: add RHEL/Rocky Linux 9 container OS support

Add OSType protobuf enum (UBUNTU_2404, ROCKY_9, RHEL_9) and --os-type
CLI flag to support Rocky Linux 9 (dev/test) and RHEL 9 (production)
as container OS alternatives to Ubuntu 24.04.

- New internal/ostype package: maps OSType to Incus images and OS family
- New internal/ospkg package: abstracts package management (apt vs dnf,
  adduser vs useradd, sudo vs wheel group, ssh vs sshd service)
- Dual stack definitions: RHEL variants for all stacks in stacks.yaml
- Refactored container/manager.go to use OS-aware provisioning
- OS type stored as container label for subsequent operations

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

* feat: add backend heartbeat metrics and Grafana dashboard

Push containarium_backend_healthy metric (1=up, 0=down) per backend to
VictoriaMetrics every 30s. Add status-history panel to the Grafana
containarium-overview dashboard showing backend heartbeat timeline.

- New containarium.backend.healthy OTel gauge in metrics collector
- FetchPeerHealth() on PeerMetricsFetcherAdapter reports peer health
- Grafana dashboard: "Backend Heartbeat" row with status-history panel
- Enriched /v1/backends endpoint with live peer system info
- Dashboard auto-updates on daemon restart (updateGrafanaDashboard)
- PostgreSQL: wait up to 2min at startup for availability; add
  Restart=on-failure systemd override inside core-postgres container

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

* fix: harden peer stability and SSH reliability

Peer node stability:
- ClamAV: wait for freshclam DB download before starting clamd; bump
  security container memory from 3GB to 4GB to prevent OOM kills
- Conntrack: increase event channel buffers from 256 to 8192; rate-limit
  "channel full" warning to once per 30s to stop log flooding
- Tunnel: increase yamux keepalive from 15s to 60s on both client and
  server so tunnel survives CPU-heavy workloads (builds, GPU training)
- Peers: add --sentinel-url for auto-update; fix restart policies

SSH reliability:
- Fix ForwardCreateContainer to use camelCase field names (sshKeys, not
  ssh_keys) matching gRPC-gateway protojson. This was silently dropping
  SSH keys when creating containers on peers.
- Reject unknown JSON fields in gRPC-gateway (DiscardUnknown: false) so
  field name mismatches fail loudly instead of silently
- Validate SSH public keys at API boundary and pre-write to reject
  placeholder strings like "YOUR_KEY"
- Fix jump server account unlock: use 'usermod -p *' instead of
  'passwd -d' which left accounts locked on Ubuntu 24.04
- Raise sshpiper failtoban threshold from 3 to 20 (ssh-agent tries
  multiple keys per connection, each counted as a failure)

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

---------

Co-authored-by: hsinhoyeh <yhh92u@gmail.com>
Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

Author: 3 people

api/swagger/containarium.swagger.json

@@ -3850,6 +3850,10 @@
         "backendId": {
           "type": "string",
           "title": "Backend ID this container runs on (e.g., \"gcp-spot\", \"tunnel-fts-5900x-gpu\")"
+        },
+        "osType": {
+          "$ref": "#/definitions/OSType",
+          "title": "Operating system type of the container"
         }
       },
       "title": "Container represents a complete container instance"
@@ -4025,6 +4029,10 @@
         "backendId": {
           "type": "string",
           "title": "Target backend ID for creation (empty = primary backend)"
+        },
+        "osType": {
+          "$ref": "#/definitions/OSType",
+          "title": "Operating system type (takes precedence over image when set)"
         }
       },
       "title": "CreateContainerRequest is the request to create a new container"
@@ -5222,6 +5230,18 @@
       },
       "title": "NetworkTopology represents the complete network visualization"
     },
+    "OSType": {
+      "type": "string",
+      "enum": [
+        "OS_TYPE_UNSPECIFIED",
+        "OS_TYPE_UBUNTU_2404",
+        "OS_TYPE_ROCKY_9",
+        "OS_TYPE_RHEL_9"
+      ],
+      "default": "OS_TYPE_UNSPECIFIED",
+      "description": "- OS_TYPE_UNSPECIFIED: Unspecified OS type (defaults to Ubuntu 24.04)\n - OS_TYPE_UBUNTU_2404: Ubuntu 24.04 LTS\n - OS_TYPE_ROCKY_9: Rocky Linux 9 (RHEL 9 rebuild, for dev/test)\n - OS_TYPE_RHEL_9: Red Hat Enterprise Linux 9 (for production, requires subscription)",
+      "title": "OSType represents the operating system type for a container"
+    },
     "PassthroughRoute": {
       "type": "object",
       "properties": {

go.mod

@@ -22,6 +22,7 @@ require (
 	go.opentelemetry.io/otel/metric v1.42.0
 	go.opentelemetry.io/otel/sdk v1.42.0
 	go.opentelemetry.io/otel/sdk/metric v1.42.0
+	golang.org/x/crypto v0.49.0
 	golang.org/x/term v0.41.0
 	google.golang.org/api v0.272.0
 	google.golang.org/genproto/googleapis/api v0.0.0-20260226221140-a57be14db171
@@ -86,7 +87,6 @@ require (
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 // indirect
 	go.opentelemetry.io/otel/trace v1.42.0 // indirect
 	go.opentelemetry.io/proto/otlp v1.9.0 // indirect
-	golang.org/x/crypto v0.49.0 // indirect
 	golang.org/x/net v0.52.0 // indirect
 	golang.org/x/oauth2 v0.36.0 // indirect
 	golang.org/x/sync v0.20.0 // indirect

internal/client/grpc.go

@@ -126,7 +126,7 @@ func (c *GRPCClient) ListContainers() ([]incus.ContainerInfo, error) {
 }
 
 // CreateContainer creates a container via gRPC
-func (c *GRPCClient) CreateContainer(username, image, cpu, memory, disk string, sshKeys []string, enablePodman bool, stack, gpu string) (*incus.ContainerInfo, error) {
+func (c *GRPCClient) CreateContainer(username, image, cpu, memory, disk string, sshKeys []string, enablePodman bool, stack, gpu string, osType pb.OSType) (*incus.ContainerInfo, error) {
 	ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute) // Container creation can take time (includes ultra-aggressive retry logic for google_guest_agent)
 	defer cancel()
 
@@ -142,6 +142,7 @@ func (c *GRPCClient) CreateContainer(username, image, cpu, memory, disk string,
 		EnablePodman: enablePodman,
 		Stack:        stack,
 		Gpu:          gpu,
+		OsType:       osType,
 	}
 
 	resp, err := c.client.CreateContainer(ctx, req)

internal/client/http.go

@@ -12,6 +12,7 @@ import (
 	"time"
 
 	"github.com/footprintai/containarium/internal/incus"
+	pb "github.com/footprintai/containarium/pkg/pb/containarium/v1"
 )
 
 // HTTPClient wraps an HTTP connection to the containarium REST API
@@ -206,7 +207,7 @@ func (c *HTTPClient) ListContainers() ([]incus.ContainerInfo, error) {
 }
 
 // CreateContainer creates a container via HTTP
-func (c *HTTPClient) CreateContainer(username, image, cpu, memory, disk string, sshKeys []string, enablePodman bool, stack, gpu string) (*incus.ContainerInfo, error) {
+func (c *HTTPClient) CreateContainer(username, image, cpu, memory, disk string, sshKeys []string, enablePodman bool, stack, gpu string, osType pb.OSType) (*incus.ContainerInfo, error) {
 	ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute)
 	defer cancel()
 
@@ -222,6 +223,7 @@ func (c *HTTPClient) CreateContainer(username, image, cpu, memory, disk string,
 		"enablePodman": enablePodman,
 		"stack":        stack,
 		"gpu":          gpu,
+		"osType":       osType,
 	}
 
 	resp, err := c.doRequest(ctx, http.MethodPost, "/v1/containers", reqBody)

internal/cmd/create.go

@@ -9,6 +9,8 @@ import (
 	"github.com/footprintai/containarium/internal/client"
 	"github.com/footprintai/containarium/internal/container"
 	"github.com/footprintai/containarium/internal/incus"
+	"github.com/footprintai/containarium/internal/ostype"
+	pb "github.com/footprintai/containarium/pkg/pb/containarium/v1"
 	"github.com/spf13/cobra"
 )
 
@@ -24,6 +26,7 @@ var (
 	forceRecreate  bool
 	stackID        string
 	gpuDevice      string
+	osTypeStr      string
 )
 
 var createCmd = &cobra.Command{
@@ -76,6 +79,7 @@ func init() {
 	createCmd.Flags().StringVar(&gpuDevice, "gpu", "", "GPU device ID for passthrough (e.g., '0' for first GPU, PCI address)")
 	createCmd.Flags().StringSliceVar(&labels, "labels", []string{}, "Labels in key=value format (can be specified multiple times)")
 	createCmd.Flags().BoolVar(&forceRecreate, "force", false, "Delete and recreate if container already exists")
+	createCmd.Flags().StringVar(&osTypeStr, "os-type", "", "Container OS type: ubuntu, rocky9, rhel9 (overrides --image)")
 }
 
 func runCreate(cmd *cobra.Command, args []string) error {
@@ -234,15 +238,18 @@ func runCreate(cmd *cobra.Command, args []string) error {
 	// Create container - use remote or local mode
 	var info *incus.ContainerInfo
 
+	// Parse OS type from flag
+	osType := ostype.OSTypeFromString(osTypeStr)
+
 	if httpMode && serverAddr != "" {
 		// Remote mode via HTTP
-		info, err = createRemoteHTTP(username, containerImage, cpuLimit, memoryLimit, diskLimit, sshKeys, enablePodman, stackID, gpuDevice)
+		info, err = createRemoteHTTP(username, containerImage, cpuLimit, memoryLimit, diskLimit, sshKeys, enablePodman, stackID, gpuDevice, osType)
 		if err != nil {
 			return fmt.Errorf("failed to create container via HTTP API: %w", err)
 		}
 	} else if serverAddr != "" {
 		// Remote mode via gRPC
-		info, err = createRemote(username, containerImage, cpuLimit, memoryLimit, diskLimit, sshKeys, enablePodman, stackID, gpuDevice)
+		info, err = createRemote(username, containerImage, cpuLimit, memoryLimit, diskLimit, sshKeys, enablePodman, stackID, gpuDevice, osType)
 		if err != nil {
 			return fmt.Errorf("failed to create container via remote server: %w", err)
 		}
@@ -251,7 +258,7 @@ func runCreate(cmd *cobra.Command, args []string) error {
 		if verbose {
 			fmt.Println("Creating container...")
 		}
-		info, err = createLocal(username, containerImage, cpuLimit, memoryLimit, diskLimit, staticIP, sshKeys, parsedLabels, enablePodman, stackID, gpuDevice)
+		info, err = createLocal(username, containerImage, cpuLimit, memoryLimit, diskLimit, staticIP, sshKeys, parsedLabels, enablePodman, stackID, gpuDevice, osType)
 		if err != nil {
 			// Cleanup jump server account on failure
 			_ = container.DeleteJumpServerAccount(username, false)
@@ -313,7 +320,7 @@ func runCreate(cmd *cobra.Command, args []string) error {
 }
 
 // createLocal creates a container using local Incus daemon
-func createLocal(username, image, cpu, memory, disk, staticIP string, sshKeys []string, labelMap map[string]string, enablePodman bool, stack, gpu string) (*incus.ContainerInfo, error) {
+func createLocal(username, image, cpu, memory, disk, staticIP string, sshKeys []string, labelMap map[string]string, enablePodman bool, stack, gpu string, osType pb.OSType) (*incus.ContainerInfo, error) {
 	mgr, err := container.New()
 	if err != nil {
 		return nil, fmt.Errorf("failed to connect to Incus: %w (is Incus running?)", err)
@@ -334,6 +341,7 @@ func createLocal(username, image, cpu, memory, disk, staticIP string, sshKeys []
 		AutoStart:              true,
 		Verbose:                verbose,
 		Stack:                  stack,
+		OSType:                 osType,
 	}
 
 	return mgr.Create(opts)
@@ -356,23 +364,23 @@ func parseLabels(labelSlice []string) map[string]string {
 }
 
 // createRemote creates a container using remote gRPC server
-func createRemote(username, image, cpu, memory, disk string, sshKeys []string, enablePodman bool, stack, gpu string) (*incus.ContainerInfo, error) {
+func createRemote(username, image, cpu, memory, disk string, sshKeys []string, enablePodman bool, stack, gpu string, osType pb.OSType) (*incus.ContainerInfo, error) {
 	grpcClient, err := client.NewGRPCClient(serverAddr, certsDir, insecure)
 	if err != nil {
 		return nil, err
 	}
 	defer grpcClient.Close()
 
-	return grpcClient.CreateContainer(username, image, cpu, memory, disk, sshKeys, enablePodman, stack, gpu)
+	return grpcClient.CreateContainer(username, image, cpu, memory, disk, sshKeys, enablePodman, stack, gpu, osType)
 }
 
 // createRemoteHTTP creates a container using remote HTTP API
-func createRemoteHTTP(username, image, cpu, memory, disk string, sshKeys []string, enablePodman bool, stack, gpu string) (*incus.ContainerInfo, error) {
+func createRemoteHTTP(username, image, cpu, memory, disk string, sshKeys []string, enablePodman bool, stack, gpu string, osType pb.OSType) (*incus.ContainerInfo, error) {
 	httpClient, err := client.NewHTTPClient(serverAddr, authToken)
 	if err != nil {
 		return nil, err
 	}
 	defer httpClient.Close()
 
-	return httpClient.CreateContainer(username, image, cpu, memory, disk, sshKeys, enablePodman, stack, gpu)
+	return httpClient.CreateContainer(username, image, cpu, memory, disk, sshKeys, enablePodman, stack, gpu, osType)
 }

internal/cmd/daemon.go

@@ -236,6 +236,36 @@ func runDaemon(cmd *cobra.Command, args []string) error {
 		}
 	}
 
+	// Wait for PostgreSQL to be reachable before proceeding — services that
+	// depend on it (SecurityService, AlertService, etc.) are registered based
+	// on PostgreSQL availability at startup. If PostgreSQL is temporarily down
+	// (e.g., container restarting), we wait up to 2 minutes rather than
+	// starting with those services permanently disabled.
+	if postgresConnString != "" {
+		log.Printf("Waiting for PostgreSQL to become reachable...")
+		waitCtx, waitCancel := context.WithTimeout(context.Background(), 2*time.Minute)
+		for {
+			pingPool, pingErr := pgxpool.New(waitCtx, postgresConnString)
+			if pingErr == nil {
+				if err := pingPool.Ping(waitCtx); err == nil {
+					pingPool.Close()
+					log.Printf("PostgreSQL is reachable")
+					break
+				}
+				pingPool.Close()
+			}
+			select {
+			case <-waitCtx.Done():
+				log.Printf("Warning: PostgreSQL not reachable after 2 minutes, proceeding anyway (some services will be disabled)")
+				goto pgWaitDone
+			case <-time.After(5 * time.Second):
+				log.Printf("  still waiting for PostgreSQL...")
+			}
+		}
+	pgWaitDone:
+		waitCancel()
+	}
+
 	// Load persisted daemon config from PostgreSQL (values saved by previous runs).
 	// CLI flags that were explicitly set always override DB values.
 	// Uses retry logic to handle post-restart races with PostgreSQL.

internal/container/cgroup_wrapper.go

@@ -3,6 +3,8 @@ package container
 import (
 	"fmt"
 	"log"
+
+	"github.com/footprintai/containarium/internal/ostype"
 )
 
 // cgroupWrapperScript returns a bash wrapper script that intercepts run/create
@@ -154,7 +156,14 @@ exec $REAL_RUNC "$@"
 // injected from the LXC container.
 func (m *Manager) installDockerOCIRuntime(containerName string) error {
 	// Step 1: Ensure jq is installed (needed by the runtime script)
-	if err := m.incus.Exec(containerName, []string{"apt-get", "install", "-y", "jq"}); err != nil {
+	// Detect OS family from container labels
+	jqInstallCmd := []string{"apt-get", "install", "-y", "jq"}
+	if info, err := m.incus.GetContainer(containerName); err == nil {
+		if osLabel, ok := info.Labels[ostype.OSTypeLabelKey]; ok && ostype.FamilyFromLabel(osLabel) == ostype.RHEL {
+			jqInstallCmd = []string{"dnf", "install", "-y", "jq"}
+		}
+	}
+	if err := m.incus.Exec(containerName, jqInstallCmd); err != nil {
 		return fmt.Errorf("failed to install jq: %w", err)
 	}
 

internal/container/collaborator.go

@@ -6,6 +6,8 @@ import (
 	"time"
 
 	"github.com/footprintai/containarium/internal/collaborator"
+	"github.com/footprintai/containarium/internal/ospkg"
+	"github.com/footprintai/containarium/internal/ostype"
 )
 
 // CollaboratorManager handles collaborator operations for containers
@@ -105,13 +107,21 @@ func (cm *CollaboratorManager) AddCollaborator(ownerUsername, collaboratorUserna
 
 // createCollaboratorUser creates a collaborator user inside the container
 func (cm *CollaboratorManager) createCollaboratorUser(containerName, ownerUsername, accountName, sshPublicKey string, grantSudo, grantContainerRuntime bool) error {
-	// Create user with bash shell (they need interactive access)
-	if err := cm.manager.incus.Exec(containerName, []string{
-		"adduser",
-		"--disabled-password",
-		"--gecos", fmt.Sprintf("Collaborator %s", accountName),
-		accountName,
-	}); err != nil {
+	// Detect OS family from container labels or by probing
+	info, err := cm.manager.incus.GetContainer(containerName)
+	family := ostype.Debian
+	if err == nil {
+		if osLabel, ok := info.Labels[ostype.OSTypeLabelKey]; ok {
+			family = ostype.FamilyFromLabel(osLabel)
+		} else {
+			family = ostype.DetectFamily(cm.manager.incus, containerName)
+		}
+	}
+	pkgMgr := ospkg.ForFamily(family)
+
+	// Create user (OS-aware: adduser on Debian, useradd on RHEL)
+	gecos := fmt.Sprintf("Collaborator %s", accountName)
+	if err := cm.manager.incus.Exec(containerName, pkgMgr.CreateUserCmd(accountName, gecos)); err != nil {
 		return fmt.Errorf("failed to create user: %w", err)
 	}
 

internal/container/jump_server.go

@@ -129,9 +129,13 @@ func EnsureJumpServerAccount(username string) error {
 		return fmt.Errorf("useradd failed: %w", err)
 	}
 
-	// Unlock account (useradd creates locked accounts, sshd rejects them)
+	// Unlock account (useradd creates locked accounts, sshd rejects them).
+	// Set password to '*' which means "no valid password" but account is not
+	// locked. This allows public key auth while preventing password login.
+	// Note: passwd -d sets an empty password which some distros reject;
+	// usermod -p '*' is the portable approach.
 	// #nosec G204 -- username validated by isValidUsername above
-	_ = exec.Command("passwd", "-d", username).Run()
+	_ = exec.Command("usermod", "-p", "*", username).Run()
 
 	// Set home dir permissions (sshd requires 755 or stricter)
 	_ = os.Chmod(fmt.Sprintf("/home/%s", username), 0755) // #nosec G302 -- sshd requires home dir to be world-readable