From d37a7cc563cd423b8132c90b04d0b78bd3c2ebfb Mon Sep 17 00:00:00 2001
From: Ignacio Van Droogenbroeck <ignacio@vandroogenbroeck.net>
Date: Thu, 27 Nov 2025 15:37:43 -0300
Subject: [PATCH 01/11] Fix Go version in Dockerfile and workflow to 1.25.3

---
 .github/workflows/release-build.yml | 2 +-
 Dockerfile                          | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/release-build.yml b/.github/workflows/release-build.yml
index 83020fe4..d37b0747 100644
--- a/.github/workflows/release-build.yml
+++ b/.github/workflows/release-build.yml
@@ -139,7 +139,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v5
         with:
-          go-version: '1.22'
+          go-version: '1.25.3'
 
       - name: Build binary
         env:
diff --git a/Dockerfile b/Dockerfile
index c9424b96..d5cb93dd 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -4,7 +4,7 @@
 ARG VERSION=dev
 
 # Build stage
-FROM golang:1.22-bookworm AS builder
+FROM golang:1.25-bookworm AS builder
 
 WORKDIR /build
 

From c5d26512978983071e3a7823fbe41c3bcfc5d081 Mon Sep 17 00:00:00 2001
From: Ignacio Van Droogenbroeck <ignacio@vandroogenbroeck.net>
Date: Thu, 27 Nov 2025 15:46:19 -0300
Subject: [PATCH 02/11] Add missing cmd/arc and helm/arc, fix Dockerfile and
 workflow

- Add cmd/arc/main.go (was ignored by git)
- Add helm/arc chart (was ignored by git)
- Remove -tags=duckdb_arrow from Dockerfile (not needed)
- Remove CGO_ENABLED=0 and comment from workflow
- Simplify build commands
---
 .github/workflows/release-build.yml           |   2 -
 Dockerfile                                    |  11 +-
 cmd/arc/main.go                               | 488 ++++++++++++++++++
 helm/arc/Chart.yaml                           |  19 +
 helm/arc/templates/_helpers.tpl               |  60 +++
 helm/arc/templates/deployment.yaml            |  97 ++++
 helm/arc/templates/ingress.yaml               |  41 ++
 helm/arc/templates/persistentvolumeclaim.yaml |  21 +
 helm/arc/templates/service.yaml               |  15 +
 helm/arc/templates/serviceaccount.yaml        |  13 +
 helm/arc/values.yaml                          |  86 +++
 11 files changed, 842 insertions(+), 11 deletions(-)
 create mode 100644 cmd/arc/main.go
 create mode 100644 helm/arc/Chart.yaml
 create mode 100644 helm/arc/templates/_helpers.tpl
 create mode 100644 helm/arc/templates/deployment.yaml
 create mode 100644 helm/arc/templates/ingress.yaml
 create mode 100644 helm/arc/templates/persistentvolumeclaim.yaml
 create mode 100644 helm/arc/templates/service.yaml
 create mode 100644 helm/arc/templates/serviceaccount.yaml
 create mode 100644 helm/arc/values.yaml

diff --git a/.github/workflows/release-build.yml b/.github/workflows/release-build.yml
index d37b0747..2c6e884d 100644
--- a/.github/workflows/release-build.yml
+++ b/.github/workflows/release-build.yml
@@ -145,11 +145,9 @@ jobs:
         env:
           GOOS: ${{ matrix.goos }}
           GOARCH: ${{ matrix.goarch }}
-          CGO_ENABLED: 0
         run: |
           VERSION=${{ needs.prepare.outputs.version }}
 
-          # Build without CGO for static binary (no Arrow support in packages)
           go build -v \
             -ldflags="-s -w -X main.Version=${VERSION}" \
             -o arc-${{ matrix.suffix }} \
diff --git a/Dockerfile b/Dockerfile
index d5cb93dd..8918e768 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -8,13 +8,6 @@ FROM golang:1.25-bookworm AS builder
 
 WORKDIR /build
 
-# Install build dependencies for DuckDB with Arrow support
-RUN apt-get update && apt-get install -y \
-    gcc \
-    g++ \
-    make \
-    && rm -rf /var/lib/apt/lists/*
-
 # Copy go mod files first for better layer caching
 COPY go.mod go.sum ./
 RUN go mod download && go mod verify
@@ -22,9 +15,9 @@ RUN go mod download && go mod verify
 # Copy source code
 COPY . .
 
-# Build with Arrow support
+# Build
 ARG VERSION
-RUN CGO_ENABLED=1 go build -v -tags=duckdb_arrow \
+RUN go build -v \
     -ldflags="-s -w -X main.Version=${VERSION}" \
     -o arc ./cmd/arc
 
diff --git a/cmd/arc/main.go b/cmd/arc/main.go
new file mode 100644
index 00000000..50d645bd
--- /dev/null
+++ b/cmd/arc/main.go
@@ -0,0 +1,488 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"os"
+	"time"
+
+	"github.com/basekick-labs/arc/internal/api"
+	"github.com/basekick-labs/arc/internal/auth"
+	"github.com/basekick-labs/arc/internal/compaction"
+	"github.com/basekick-labs/arc/internal/config"
+	"github.com/basekick-labs/arc/internal/database"
+	"github.com/basekick-labs/arc/internal/ingest"
+	"github.com/basekick-labs/arc/internal/logger"
+	"github.com/basekick-labs/arc/internal/metrics"
+	"github.com/basekick-labs/arc/internal/shutdown"
+	"github.com/basekick-labs/arc/internal/storage"
+	"github.com/basekick-labs/arc/internal/telemetry"
+	"github.com/basekick-labs/arc/internal/wal"
+	"github.com/rs/zerolog/log"
+)
+
+// Version is set at build time
+var Version = "dev"
+
+func main() {
+	// Check for subcommands before loading full config
+	if len(os.Args) > 1 && os.Args[1] == "compact" {
+		runCompactSubcommand(os.Args[2:])
+		return
+	}
+
+	// Load configuration
+	cfg, err := config.Load()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Failed to load config: %v\n", err)
+		os.Exit(1)
+	}
+
+	// Setup logger
+	logger.Setup(cfg.Log.Level, cfg.Log.Format)
+	log.Info().Str("version", Version).Msg("Starting Arc...")
+
+	// Initialize metrics collector
+	metrics.Init(logger.Get("metrics"))
+
+	// Initialize shutdown coordinator
+	shutdownCoordinator := shutdown.New(30*time.Second, logger.Get("shutdown"))
+
+	// Initialize DuckDB
+	dbConfig := &database.Config{
+		MaxConnections: cfg.Database.MaxConnections,
+		MemoryLimit:    cfg.Database.MemoryLimit,
+		ThreadCount:    cfg.Database.ThreadCount,
+		EnableWAL:      cfg.Database.EnableWAL,
+	}
+
+	db, err := database.New(dbConfig, logger.Get("database"))
+	if err != nil {
+		log.Fatal().Err(err).Msg("Failed to initialize database")
+	}
+	shutdownCoordinator.Register("database", db, shutdown.PriorityDatabase)
+
+	// Test query
+	log.Info().Msg("Testing DuckDB connection...")
+	rows, err := db.Query("SELECT 'Hello from Arc in Go!' as message, version() as duckdb_version")
+	if err != nil {
+		log.Fatal().Err(err).Msg("Failed to execute test query")
+	}
+	defer rows.Close()
+
+	var message, version string
+	if rows.Next() {
+		if err := rows.Scan(&message, &version); err != nil {
+			log.Fatal().Err(err).Msg("Failed to scan result")
+		}
+		log.Info().
+			Str("message", message).
+			Str("duckdb_version", version).
+			Msg("Test query successful")
+	}
+
+	// Print database stats
+	stats := db.Stats()
+	log.Info().
+		Int("max_open", stats.MaxOpenConnections).
+		Int("open", stats.OpenConnections).
+		Int("in_use", stats.InUse).
+		Int("idle", stats.Idle).
+		Msg("Database connection pool stats")
+
+	// Initialize storage backend
+	var storageBackend storage.Backend
+	switch cfg.Storage.Backend {
+	case "local":
+		storageBackend, err = storage.NewLocalBackend(cfg.Storage.LocalPath, logger.Get("storage"))
+		if err != nil {
+			log.Fatal().Err(err).Msg("Failed to initialize local storage backend")
+		}
+		shutdownCoordinator.Register("storage", storageBackend, shutdown.PriorityStorage)
+		log.Info().
+			Str("backend", "local").
+			Str("path", cfg.Storage.LocalPath).
+			Msg("Storage backend initialized")
+
+	case "s3", "minio":
+		s3Config := &storage.S3Config{
+			Bucket:    cfg.Storage.S3Bucket,
+			Region:    cfg.Storage.S3Region,
+			Endpoint:  cfg.Storage.S3Endpoint,
+			AccessKey: cfg.Storage.S3AccessKey,
+			SecretKey: cfg.Storage.S3SecretKey,
+			UseSSL:    cfg.Storage.S3UseSSL,
+			PathStyle: cfg.Storage.S3PathStyle,
+		}
+		storageBackend, err = storage.NewS3Backend(s3Config, logger.Get("storage"))
+		if err != nil {
+			log.Fatal().Err(err).Msg("Failed to initialize S3 storage backend")
+		}
+		shutdownCoordinator.Register("storage", storageBackend, shutdown.PriorityStorage)
+		log.Info().
+			Str("backend", cfg.Storage.Backend).
+			Str("bucket", cfg.Storage.S3Bucket).
+			Str("region", cfg.Storage.S3Region).
+			Str("endpoint", cfg.Storage.S3Endpoint).
+			Msg("Storage backend initialized")
+
+	default:
+		log.Fatal().Str("backend", cfg.Storage.Backend).Msg("Unsupported storage backend (use 'local', 's3', or 'minio')")
+	}
+
+	// Initialize WAL (if enabled)
+	var walWriter *wal.Writer
+	if cfg.WAL.Enabled {
+		// Run WAL recovery FIRST (before creating new WAL writer)
+		recovery := wal.NewRecovery(cfg.WAL.Directory, logger.Get("wal"))
+		recoveryStats, err := recovery.Recover(context.Background(), func(ctx context.Context, records []map[string]interface{}) error {
+			// Re-ingest recovered records through the ingest pipeline
+			// For now, just log - full integration requires ArrowBuffer to be created first
+			log.Info().Int("records", len(records)).Msg("WAL recovery: replaying records")
+			return nil
+		})
+		if err != nil {
+			log.Error().Err(err).Msg("WAL recovery failed")
+		} else if recoveryStats.RecoveredFiles > 0 {
+			log.Info().
+				Int("files", recoveryStats.RecoveredFiles).
+				Int("batches", recoveryStats.RecoveredBatches).
+				Int("entries", recoveryStats.RecoveredEntries).
+				Int("corrupted", recoveryStats.CorruptedEntries).
+				Dur("duration", recoveryStats.RecoveryDuration).
+				Msg("WAL recovery complete")
+		}
+
+		// Create WAL writer AFTER recovery (so new file isn't recovered)
+		walWriter, err = wal.NewWriter(&wal.WriterConfig{
+			WALDir:       cfg.WAL.Directory,
+			SyncMode:     wal.SyncMode(cfg.WAL.SyncMode),
+			MaxSizeBytes: int64(cfg.WAL.MaxSizeMB) * 1024 * 1024,
+			MaxAge:       time.Duration(cfg.WAL.MaxAgeSeconds) * time.Second,
+			Logger:       logger.Get("wal"),
+		})
+		if err != nil {
+			log.Fatal().Err(err).Msg("Failed to initialize WAL writer")
+		}
+		shutdownCoordinator.Register("wal", walWriter, shutdown.PriorityWAL)
+
+		log.Info().
+			Str("directory", cfg.WAL.Directory).
+			Str("sync_mode", cfg.WAL.SyncMode).
+			Int("max_size_mb", cfg.WAL.MaxSizeMB).
+			Int("max_age_seconds", cfg.WAL.MaxAgeSeconds).
+			Msg("WAL enabled")
+	} else {
+		log.Info().Msg("WAL is DISABLED - data durability relies on immediate Parquet flushes")
+	}
+
+	// Initialize Arrow buffer (optionally with WAL)
+	arrowBuffer := ingest.NewArrowBuffer(&cfg.Ingest, storageBackend, logger.Get("arrow"))
+	if walWriter != nil {
+		arrowBuffer.SetWAL(walWriter)
+	}
+	shutdownCoordinator.Register("arrow-buffer", arrowBuffer, shutdown.PriorityBuffer)
+
+	// Initialize AuthManager (if enabled)
+	var authManager *auth.AuthManager
+	if cfg.Auth.Enabled {
+		authManager, err = auth.NewAuthManager(
+			cfg.Auth.DBPath,
+			time.Duration(cfg.Auth.CacheTTL)*time.Second,
+			cfg.Auth.MaxCacheSize,
+			logger.Get("auth"),
+		)
+		if err != nil {
+			log.Fatal().Err(err).Msg("Failed to initialize auth manager")
+		}
+		shutdownCoordinator.Register("auth", authManager, shutdown.PriorityAuth)
+
+		// Create initial admin token if this is first run
+		if token, err := authManager.EnsureInitialToken(); err != nil {
+			log.Error().Err(err).Msg("Failed to create initial admin token")
+		} else if token != "" {
+			// Print colorized banner to stderr (bypasses structured logging)
+			const (
+				cyan   = "\033[96m"
+				yellow = "\033[93m"
+				bold   = "\033[1m"
+				reset  = "\033[0m"
+			)
+			banner := cyan + "======================================================================" + reset
+			fmt.Fprintln(os.Stderr)
+			fmt.Fprintln(os.Stderr, banner)
+			fmt.Fprintln(os.Stderr, cyan+bold+"  FIRST RUN - INITIAL ADMIN TOKEN GENERATED"+reset)
+			fmt.Fprintln(os.Stderr, banner)
+			fmt.Fprintln(os.Stderr, yellow+bold+"  Initial admin API token: "+token+reset)
+			fmt.Fprintln(os.Stderr, banner)
+			fmt.Fprintln(os.Stderr, cyan+"  SAVE THIS TOKEN! It will not be shown again."+reset)
+			fmt.Fprintln(os.Stderr, cyan+"  Use this token to login to the web UI or API."+reset)
+			fmt.Fprintln(os.Stderr, cyan+"  You can create additional tokens after logging in."+reset)
+			fmt.Fprintln(os.Stderr, banner)
+			fmt.Fprintln(os.Stderr)
+		}
+
+		log.Info().
+			Str("db_path", cfg.Auth.DBPath).
+			Int("cache_ttl", cfg.Auth.CacheTTL).
+			Int("max_cache_size", cfg.Auth.MaxCacheSize).
+			Msg("Authentication enabled")
+	} else {
+		log.Warn().Msg("Authentication is DISABLED - all endpoints are public")
+	}
+
+	// Initialize Compaction (if enabled)
+	var compactionScheduler *compaction.Scheduler
+	var compactionManager *compaction.Manager
+	if cfg.Compaction.Enabled {
+		// Build tiers
+		var tiers []compaction.Tier
+
+		if cfg.Compaction.HourlyEnabled {
+			hourlyTier := compaction.NewHourlyTier(&compaction.HourlyTierConfig{
+				StorageBackend: storageBackend,
+				MinAgeHours:    cfg.Compaction.HourlyMinAgeHours,
+				MinFiles:       cfg.Compaction.HourlyMinFiles,
+				Enabled:        true,
+				Logger:         logger.Get("compaction"),
+			})
+			tiers = append(tiers, hourlyTier)
+		}
+
+		if cfg.Compaction.DailyEnabled {
+			dailyTier := compaction.NewDailyTier(&compaction.DailyTierConfig{
+				StorageBackend: storageBackend,
+				MinAgeHours:    cfg.Compaction.DailyMinAgeHours,
+				MinFiles:       cfg.Compaction.DailyMinFiles,
+				Enabled:        true,
+				Logger:         logger.Get("compaction"),
+			})
+			tiers = append(tiers, dailyTier)
+		}
+
+		// Create lock manager
+		lockManager := compaction.NewLockManager()
+
+		// Create compaction manager (discovers all databases dynamically)
+		// Compaction jobs run in subprocesses for memory isolation
+		compactionManager = compaction.NewManager(&compaction.ManagerConfig{
+			StorageBackend: storageBackend,
+			LockManager:    lockManager,
+			MaxConcurrent:  cfg.Compaction.MaxConcurrent,
+			Tiers:          tiers,
+			Logger:         logger.Get("compaction"),
+		})
+
+		// Create and start scheduler (using hourly schedule by default)
+		var err error
+		compactionScheduler, err = compaction.NewScheduler(&compaction.SchedulerConfig{
+			Manager:  compactionManager,
+			Schedule: cfg.Compaction.HourlySchedule,
+			Enabled:  true,
+			Logger:   logger.Get("compaction"),
+		})
+		if err != nil {
+			log.Fatal().Err(err).Msg("Failed to create compaction scheduler")
+		}
+
+		if err := compactionScheduler.Start(); err != nil {
+			log.Fatal().Err(err).Msg("Failed to start compaction scheduler")
+		}
+
+		// Register shutdown hook for compaction
+		shutdownCoordinator.RegisterHook("compaction-scheduler", func(ctx context.Context) error {
+			compactionScheduler.Stop()
+			return nil
+		}, shutdown.PriorityCompaction)
+
+		log.Info().
+			Str("hourly_schedule", cfg.Compaction.HourlySchedule).
+			Bool("hourly_enabled", cfg.Compaction.HourlyEnabled).
+			Bool("daily_enabled", cfg.Compaction.DailyEnabled).
+			Int("max_concurrent", cfg.Compaction.MaxConcurrent).
+			Msg("Compaction enabled")
+	} else {
+		log.Info().Msg("Compaction is DISABLED")
+	}
+
+	// Initialize Telemetry (if enabled)
+	var telemetryCollector *telemetry.Collector
+	if cfg.Telemetry.Enabled {
+		telemetryCfg := &telemetry.Config{
+			Enabled:  true,
+			Endpoint: cfg.Telemetry.Endpoint,
+			Interval: time.Duration(cfg.Telemetry.IntervalSeconds) * time.Second,
+			DataDir:  "./data",
+		}
+		telemetryCollector, err = telemetry.New(telemetryCfg, Version, logger.Get("telemetry"))
+		if err != nil {
+			log.Warn().Err(err).Msg("Failed to initialize telemetry (continuing without it)")
+		} else {
+			telemetryCollector.Start()
+			shutdownCoordinator.RegisterHook("telemetry", func(ctx context.Context) error {
+				telemetryCollector.Stop()
+				return nil
+			}, shutdown.PriorityTelemetry)
+
+			log.Info().
+				Str("instance_id", telemetryCollector.GetInstanceID()).
+				Dur("interval", time.Duration(cfg.Telemetry.IntervalSeconds)*time.Second).
+				Msg("Telemetry enabled")
+		}
+	} else {
+		log.Info().Msg("Telemetry is DISABLED (opt-out via ARC_TELEMETRY_ENABLED=false)")
+	}
+
+	// Initialize HTTP server
+	serverConfig := &api.ServerConfig{
+		Port:            cfg.Server.Port,
+		ReadTimeout:     30 * time.Second,
+		WriteTimeout:    30 * time.Second,
+		IdleTimeout:     120 * time.Second,
+		ShutdownTimeout: 30 * time.Second,
+	}
+
+	server := api.NewServer(serverConfig, logger.Get("server"))
+
+	// Register base routes
+	server.RegisterRoutes()
+
+	// Apply auth middleware if enabled
+	if authManager != nil {
+		middlewareConfig := auth.DefaultMiddlewareConfig()
+		middlewareConfig.AuthManager = authManager
+		// Add public routes that don't need auth
+		middlewareConfig.PublicRoutes = append(middlewareConfig.PublicRoutes, "/health", "/ready", "/api/v1/auth/verify")
+		middlewareConfig.PublicPrefixes = append(middlewareConfig.PublicPrefixes, "/metrics", "/debug/pprof")
+		server.GetApp().Use(auth.NewMiddleware(middlewareConfig))
+
+		// Register auth routes
+		authHandler := api.NewAuthHandler(authManager, logger.Get("auth"))
+		authHandler.RegisterRoutes(server.GetApp())
+	}
+
+	// Register MessagePack handler with Arrow buffer
+	msgpackHandler := api.NewMsgPackHandler(logger.Get("msgpack"), arrowBuffer)
+	msgpackHandler.RegisterRoutes(server.GetApp())
+
+	// Register Line Protocol handler
+	lineProtocolHandler := api.NewLineProtocolHandler(arrowBuffer, logger.Get("lineprotocol"))
+	lineProtocolHandler.RegisterRoutes(server.GetApp())
+
+	// Register Query handler
+	queryHandler := api.NewQueryHandler(db, storageBackend, logger.Get("query"))
+	queryHandler.RegisterRoutes(server.GetApp())
+
+	// Register Compaction handler (if compaction is enabled)
+	if compactionManager != nil {
+		compactionHandler := api.NewCompactionHandler(compactionManager, compactionScheduler, logger.Get("compaction"))
+		compactionHandler.RegisterRoutes(server.GetApp())
+	}
+
+	// Register Delete handler
+	deleteHandler := api.NewDeleteHandler(db, storageBackend, &cfg.Delete, logger.Get("delete"))
+	deleteHandler.RegisterRoutes(server.GetApp())
+	if cfg.Delete.Enabled {
+		log.Info().
+			Int("confirmation_threshold", cfg.Delete.ConfirmationThreshold).
+			Int("max_rows_per_delete", cfg.Delete.MaxRowsPerDelete).
+			Msg("Delete operations enabled")
+	} else {
+		log.Info().Msg("Delete operations DISABLED (set delete.enabled=true in arc.toml to enable)")
+	}
+
+	// Register Retention handler
+	if cfg.Retention.Enabled {
+		retentionHandler, err := api.NewRetentionHandler(storageBackend, db, &cfg.Retention, logger.Get("retention"))
+		if err != nil {
+			log.Fatal().Err(err).Msg("Failed to initialize retention handler")
+		}
+		retentionHandler.RegisterRoutes(server.GetApp())
+		shutdownCoordinator.RegisterHook("retention", func(ctx context.Context) error {
+			return retentionHandler.Close()
+		}, shutdown.PriorityDatabase)
+		log.Info().Str("db_path", cfg.Retention.DBPath).Msg("Retention policies enabled")
+	} else {
+		log.Info().Msg("Retention policies DISABLED")
+	}
+
+	// Register Continuous Query handler
+	if cfg.ContinuousQuery.Enabled {
+		cqHandler, err := api.NewContinuousQueryHandler(db, storageBackend, arrowBuffer, &cfg.ContinuousQuery, logger.Get("cq"))
+		if err != nil {
+			log.Fatal().Err(err).Msg("Failed to initialize continuous query handler")
+		}
+		cqHandler.RegisterRoutes(server.GetApp())
+		shutdownCoordinator.RegisterHook("continuous-query", func(ctx context.Context) error {
+			return cqHandler.Close()
+		}, shutdown.PriorityDatabase)
+		log.Info().Str("db_path", cfg.ContinuousQuery.DBPath).Msg("Continuous queries enabled")
+	} else {
+		log.Info().Msg("Continuous queries DISABLED")
+	}
+
+	// Register HTTP server shutdown hook (first to stop accepting new requests)
+	shutdownCoordinator.RegisterHook("http-server", func(ctx context.Context) error {
+		return server.Shutdown(30 * time.Second)
+	}, shutdown.PriorityHTTPServer)
+
+	// Start server
+	if err := server.Start(); err != nil {
+		log.Fatal().Err(err).Msg("Failed to start HTTP server")
+	}
+
+	log.Info().
+		Int("port", cfg.Server.Port).
+		Str("version", Version).
+		Msg("Arc is ready! HTTP server started")
+
+	// Wait for shutdown signal
+	sig := shutdownCoordinator.WaitForSignal()
+	log.Info().Str("signal", sig.String()).Msg("Initiating graceful shutdown...")
+
+	// Perform graceful shutdown of all components
+	if err := shutdownCoordinator.Shutdown(); err != nil {
+		log.Error().Err(err).Msg("Shutdown completed with errors")
+		os.Exit(1)
+	}
+
+	log.Info().Msg("Arc shutdown complete")
+}
+
+// runCompactSubcommand handles the "compact" subcommand for subprocess-based compaction.
+// This is invoked by the parent process to run compaction jobs in isolation,
+// ensuring DuckDB memory is fully released when the subprocess exits.
+func runCompactSubcommand(args []string) {
+	fs := flag.NewFlagSet("compact", flag.ExitOnError)
+	jobJSON := fs.String("job", "", "Job configuration as JSON")
+	if err := fs.Parse(args); err != nil {
+		fmt.Fprintf(os.Stderr, "error: failed to parse flags: %v\n", err)
+		os.Exit(1)
+	}
+
+	if *jobJSON == "" {
+		fmt.Fprintln(os.Stderr, "error: --job flag required")
+		os.Exit(1)
+	}
+
+	var cfg compaction.SubprocessJobConfig
+	if err := json.Unmarshal([]byte(*jobJSON), &cfg); err != nil {
+		fmt.Fprintf(os.Stderr, "error: invalid job config: %v\n", err)
+		os.Exit(1)
+	}
+
+	// Run compaction job
+	result, err := compaction.RunSubprocessJob(&cfg)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "error: %v\n", err)
+		os.Exit(1)
+	}
+
+	// Output result as JSON to stdout for parent process to parse
+	if err := json.NewEncoder(os.Stdout).Encode(result); err != nil {
+		fmt.Fprintf(os.Stderr, "error: failed to encode result: %v\n", err)
+		os.Exit(1)
+	}
+}
diff --git a/helm/arc/Chart.yaml b/helm/arc/Chart.yaml
new file mode 100644
index 00000000..002713a0
--- /dev/null
+++ b/helm/arc/Chart.yaml
@@ -0,0 +1,19 @@
+apiVersion: v2
+name: arc
+description: High-performance time-series database built on DuckDB (Go implementation)
+type: application
+version: 0.0.1
+appVersion: "0.0.1"
+keywords:
+  - database
+  - timeseries
+  - observability
+  - duckdb
+  - analytics
+  - go
+home: https://github.com/basekick-labs/arc-go
+sources:
+  - https://github.com/basekick-labs/arc-go
+maintainers:
+  - name: Basekick Labs
+    email: contact@basekick.com
diff --git a/helm/arc/templates/_helpers.tpl b/helm/arc/templates/_helpers.tpl
new file mode 100644
index 00000000..5723514a
--- /dev/null
+++ b/helm/arc/templates/_helpers.tpl
@@ -0,0 +1,60 @@
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "arc.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Create a default fully qualified app name.
+*/}}
+{{- define "arc.fullname" -}}
+{{- if .Values.fullnameOverride }}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- $name := default .Chart.Name .Values.nameOverride }}
+{{- if contains $name .Release.Name }}
+{{- .Release.Name | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
+{{- end }}
+{{- end }}
+{{- end }}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "arc.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Common labels
+*/}}
+{{- define "arc.labels" -}}
+helm.sh/chart: {{ include "arc.chart" . }}
+{{ include "arc.selectorLabels" . }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end }}
+
+{{/*
+Selector labels
+*/}}
+{{- define "arc.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "arc.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end }}
+
+{{/*
+Create the name of the service account to use
+*/}}
+{{- define "arc.serviceAccountName" -}}
+{{- if .Values.serviceAccount.create }}
+{{- default (include "arc.fullname" .) .Values.serviceAccount.name }}
+{{- else }}
+{{- default "default" .Values.serviceAccount.name }}
+{{- end }}
+{{- end }}
diff --git a/helm/arc/templates/deployment.yaml b/helm/arc/templates/deployment.yaml
new file mode 100644
index 00000000..95f3a2f9
--- /dev/null
+++ b/helm/arc/templates/deployment.yaml
@@ -0,0 +1,97 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "arc.fullname" . }}
+  labels:
+    {{- include "arc.labels" . | nindent 4 }}
+spec:
+  {{- if not .Values.autoscaling.enabled }}
+  replicas: {{ .Values.replicaCount }}
+  {{- end }}
+  selector:
+    matchLabels:
+      {{- include "arc.selectorLabels" . | nindent 6 }}
+  template:
+    metadata:
+      {{- with .Values.podAnnotations }}
+      annotations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      labels:
+        {{- include "arc.labels" . | nindent 8 }}
+        {{- with .Values.podLabels }}
+        {{- toYaml . | nindent 8 }}
+        {{- end }}
+    spec:
+      {{- with .Values.imagePullSecrets }}
+      imagePullSecrets:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      serviceAccountName: {{ include "arc.serviceAccountName" . }}
+      securityContext:
+        {{- toYaml .Values.podSecurityContext | nindent 8 }}
+      containers:
+        - name: {{ .Chart.Name }}
+          securityContext:
+            {{- toYaml .Values.securityContext | nindent 12 }}
+          image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          ports:
+            - name: http
+              containerPort: 8000
+              protocol: TCP
+          env:
+            - name: ARC_STORAGE_BACKEND
+              value: {{ .Values.arc.storageBackend | quote }}
+            - name: ARC_STORAGE_LOCAL_PATH
+              value: "/app/data/arc"
+            {{- with .Values.arc.env }}
+            {{- toYaml . | nindent 12 }}
+            {{- end }}
+          livenessProbe:
+            httpGet:
+              path: /health
+              port: http
+            initialDelaySeconds: 10
+            periodSeconds: 10
+            timeoutSeconds: 5
+            failureThreshold: 3
+          readinessProbe:
+            httpGet:
+              path: /ready
+              port: http
+            initialDelaySeconds: 5
+            periodSeconds: 5
+            timeoutSeconds: 3
+            failureThreshold: 3
+          resources:
+            {{- toYaml .Values.resources | nindent 12 }}
+          volumeMounts:
+            - name: data
+              mountPath: /app/data
+            {{- with .Values.volumeMounts }}
+            {{- toYaml . | nindent 12 }}
+            {{- end }}
+      volumes:
+        - name: data
+          {{- if .Values.persistence.enabled }}
+          persistentVolumeClaim:
+            claimName: {{ .Values.persistence.existingClaim | default (include "arc.fullname" .) }}
+          {{- else }}
+          emptyDir: {}
+          {{- end }}
+        {{- with .Values.volumes }}
+        {{- toYaml . | nindent 8 }}
+        {{- end }}
+      {{- with .Values.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
diff --git a/helm/arc/templates/ingress.yaml b/helm/arc/templates/ingress.yaml
new file mode 100644
index 00000000..96d5ac77
--- /dev/null
+++ b/helm/arc/templates/ingress.yaml
@@ -0,0 +1,41 @@
+{{- if .Values.ingress.enabled -}}
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: {{ include "arc.fullname" . }}
+  labels:
+    {{- include "arc.labels" . | nindent 4 }}
+  {{- with .Values.ingress.annotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+spec:
+  {{- if .Values.ingress.className }}
+  ingressClassName: {{ .Values.ingress.className }}
+  {{- end }}
+  {{- if .Values.ingress.tls }}
+  tls:
+    {{- range .Values.ingress.tls }}
+    - hosts:
+        {{- range .hosts }}
+        - {{ . | quote }}
+        {{- end }}
+      secretName: {{ .secretName }}
+    {{- end }}
+  {{- end }}
+  rules:
+    {{- range .Values.ingress.hosts }}
+    - host: {{ .host | quote }}
+      http:
+        paths:
+          {{- range .paths }}
+          - path: {{ .path }}
+            pathType: {{ .pathType }}
+            backend:
+              service:
+                name: {{ include "arc.fullname" $ }}
+                port:
+                  number: {{ $.Values.service.port }}
+          {{- end }}
+    {{- end }}
+{{- end }}
diff --git a/helm/arc/templates/persistentvolumeclaim.yaml b/helm/arc/templates/persistentvolumeclaim.yaml
new file mode 100644
index 00000000..4a7e4598
--- /dev/null
+++ b/helm/arc/templates/persistentvolumeclaim.yaml
@@ -0,0 +1,21 @@
+{{- if and .Values.persistence.enabled (not .Values.persistence.existingClaim) }}
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: {{ include "arc.fullname" . }}
+  labels:
+    {{- include "arc.labels" . | nindent 4 }}
+spec:
+  accessModes:
+    - {{ .Values.persistence.accessMode }}
+  {{- if .Values.persistence.storageClass }}
+  {{- if (eq "-" .Values.persistence.storageClass) }}
+  storageClassName: ""
+  {{- else }}
+  storageClassName: "{{ .Values.persistence.storageClass }}"
+  {{- end }}
+  {{- end }}
+  resources:
+    requests:
+      storage: {{ .Values.persistence.size }}
+{{- end }}
diff --git a/helm/arc/templates/service.yaml b/helm/arc/templates/service.yaml
new file mode 100644
index 00000000..e73fd667
--- /dev/null
+++ b/helm/arc/templates/service.yaml
@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "arc.fullname" . }}
+  labels:
+    {{- include "arc.labels" . | nindent 4 }}
+spec:
+  type: {{ .Values.service.type }}
+  ports:
+    - port: {{ .Values.service.port }}
+      targetPort: http
+      protocol: TCP
+      name: http
+  selector:
+    {{- include "arc.selectorLabels" . | nindent 4 }}
diff --git a/helm/arc/templates/serviceaccount.yaml b/helm/arc/templates/serviceaccount.yaml
new file mode 100644
index 00000000..73f0667b
--- /dev/null
+++ b/helm/arc/templates/serviceaccount.yaml
@@ -0,0 +1,13 @@
+{{- if .Values.serviceAccount.create -}}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ include "arc.serviceAccountName" . }}
+  labels:
+    {{- include "arc.labels" . | nindent 4 }}
+  {{- with .Values.serviceAccount.annotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+automountServiceAccountToken: {{ .Values.serviceAccount.automount }}
+{{- end }}
diff --git a/helm/arc/values.yaml b/helm/arc/values.yaml
new file mode 100644
index 00000000..074517c6
--- /dev/null
+++ b/helm/arc/values.yaml
@@ -0,0 +1,86 @@
+# Default values for Arc (Go)
+
+replicaCount: 1
+
+image:
+  repository: ghcr.io/basekick-labs/arc-go
+  pullPolicy: IfNotPresent
+  # Overrides the image tag whose default is the chart appVersion
+  tag: ""
+
+imagePullSecrets: []
+nameOverride: ""
+fullnameOverride: ""
+
+serviceAccount:
+  create: true
+  automount: true
+  annotations: {}
+  name: ""
+
+podAnnotations: {}
+podLabels: {}
+
+podSecurityContext:
+  fsGroup: 1000
+
+securityContext:
+  capabilities:
+    drop:
+      - ALL
+  readOnlyRootFilesystem: false
+  runAsNonRoot: true
+  runAsUser: 1000
+
+service:
+  type: ClusterIP
+  port: 8000
+
+ingress:
+  enabled: false
+  className: ""
+  annotations: {}
+  hosts:
+    - host: arc.local
+      paths:
+        - path: /
+          pathType: Prefix
+  tls: []
+
+resources: {}
+  # limits:
+  #   cpu: 2000m
+  #   memory: 4Gi
+  # requests:
+  #   cpu: 1000m
+  #   memory: 2Gi
+
+# Arc-specific configuration
+arc:
+  # Storage backend: local, s3, minio
+  storageBackend: local
+  # Additional environment variables
+  env: []
+    # - name: ARC_AUTH_ENABLED
+    #   value: "true"
+
+# Persistent storage for Arc data
+persistence:
+  enabled: true
+  # storageClass: "-"
+  accessMode: ReadWriteOnce
+  size: 10Gi
+  # existingClaim: ""
+
+autoscaling:
+  enabled: false
+  minReplicas: 1
+  maxReplicas: 10
+  targetCPUUtilizationPercentage: 80
+
+volumes: []
+volumeMounts: []
+
+nodeSelector: {}
+tolerations: []
+affinity: {}

From 19b82f9223893ed7e9397971039e4a435eb57e40 Mon Sep 17 00:00:00 2001
From: Ignacio Van Droogenbroeck <ignacio@vandroogenbroeck.net>
Date: Thu, 27 Nov 2025 15:50:20 -0300
Subject: [PATCH 03/11] Use Docker for binary builds (CGO cross-compilation)

DuckDB requires CGO, so we use Docker buildx for cross-compiling
arm64 binaries on the amd64 runner.
---
 .github/workflows/release-build.yml | 37 ++++++++++++++++-------------
 1 file changed, 20 insertions(+), 17 deletions(-)

diff --git a/.github/workflows/release-build.yml b/.github/workflows/release-build.yml
index 2c6e884d..41257401 100644
--- a/.github/workflows/release-build.yml
+++ b/.github/workflows/release-build.yml
@@ -118,7 +118,7 @@ jobs:
           if-no-files-found: error
           retention-days: 1
 
-  # Build Go binaries for all platforms
+  # Build Go binaries for all platforms using Docker (for CGO cross-compilation)
   build-binaries:
     name: Build Binaries
     runs-on: ubuntu-latest
@@ -126,32 +126,35 @@ jobs:
     strategy:
       matrix:
         include:
-          - goos: linux
-            goarch: amd64
+          - platform: linux/amd64
             suffix: linux-amd64
-          - goos: linux
-            goarch: arm64
+          - platform: linux/arm64
             suffix: linux-arm64
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
 
-      - name: Set up Go
-        uses: actions/setup-go@v5
-        with:
-          go-version: '1.25.3'
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
 
-      - name: Build binary
-        env:
-          GOOS: ${{ matrix.goos }}
-          GOARCH: ${{ matrix.goarch }}
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Build binary in Docker
         run: |
           VERSION=${{ needs.prepare.outputs.version }}
 
-          go build -v \
-            -ldflags="-s -w -X main.Version=${VERSION}" \
-            -o arc-${{ matrix.suffix }} \
-            ./cmd/arc
+          # Build using Docker to handle CGO cross-compilation
+          docker buildx build \
+            --platform ${{ matrix.platform }} \
+            --build-arg VERSION=${VERSION} \
+            --target builder \
+            --output type=local,dest=./out \
+            .
+
+          # Extract the binary
+          cp ./out/build/arc arc-${{ matrix.suffix }}
+          chmod +x arc-${{ matrix.suffix }}
 
       - name: Upload binary
         uses: actions/upload-artifact@v4

From efe187e8ea1b13e27860a4115ae159d0962253e4 Mon Sep 17 00:00:00 2001
From: Ignacio Van Droogenbroeck <ignacio@vandroogenbroeck.net>
Date: Thu, 27 Nov 2025 16:18:28 -0300
Subject: [PATCH 04/11] Add Docker/binary tests, update release notes for Go
 migration

- Fix all arc-go URLs to arc (image name, homepage, docs)
- Add test-docker job: pulls image, tests /health endpoint
- Add test-binary job: runs binary, tests /health endpoint
- Update release notes with migration highlights:
  - 9.47M rec/s MessagePack (125% faster than Python)
  - Memory stable (no 372MB leak)
  - Full feature parity checklist
  - Breaking changes and upgrade guide
- Add tests to release job dependencies
- Add test status to job summary
---
 .github/workflows/release-build.yml | 188 +++++++++++++++++++++++-----
 1 file changed, 155 insertions(+), 33 deletions(-)

diff --git a/.github/workflows/release-build.yml b/.github/workflows/release-build.yml
index 41257401..c7520bb3 100644
--- a/.github/workflows/release-build.yml
+++ b/.github/workflows/release-build.yml
@@ -16,7 +16,7 @@ permissions:
 
 env:
   REGISTRY_GHCR: ghcr.io
-  IMAGE_NAME: basekick-labs/arc-go
+  IMAGE_NAME: basekick-labs/arc
 
 jobs:
   # Extract version from branch name or input
@@ -242,7 +242,7 @@ jobs:
            .
            Features: 9.47M records/sec ingestion, DuckDB SQL queries,
            Parquet storage, S3/MinIO support.
-          Homepage: https://github.com/basekick-labs/arc-go
+          Homepage: https://github.com/basekick-labs/arc
           EOF
 
           # Create postinst script
@@ -366,7 +366,7 @@ jobs:
           Release:        1%{?dist}
           Summary:        High-Performance Time-Series Database
           License:        AGPL-3.0
-          URL:            https://github.com/basekick-labs/arc-go
+          URL:            https://github.com/basekick-labs/arc
           Source0:        arc-${VERSION}.tar.gz
           Source1:        arc.service
           BuildArch:      ${ARCH}
@@ -477,6 +477,105 @@ jobs:
           echo "✅ Multi-arch manifest created and pushed" >> $GITHUB_STEP_SUMMARY
           echo "Tags: ${{ needs.prepare.outputs.version }}, ${{ needs.prepare.outputs.short_version }}" >> $GITHUB_STEP_SUMMARY
 
+  # Test Docker image health
+  test-docker:
+    name: Test Docker Image
+    runs-on: ubuntu-latest
+    needs: [prepare, docker-merge]
+    steps:
+      - name: Pull and run Docker image
+        run: |
+          VERSION=${{ needs.prepare.outputs.version }}
+          IMAGE="${{ env.REGISTRY_GHCR }}/${{ env.IMAGE_NAME }}:${VERSION}"
+
+          echo "Pulling image: ${IMAGE}"
+          docker pull ${IMAGE}
+
+          echo "Starting Arc container..."
+          docker run -d --name arc-test -p 8000:8000 ${IMAGE}
+
+          # Wait for startup
+          echo "Waiting for Arc to start..."
+          sleep 10
+
+          # Test health endpoint
+          echo "Testing health endpoint..."
+          for i in {1..30}; do
+            if curl -sf http://localhost:8000/health; then
+              echo ""
+              echo "✅ Health check passed!"
+              docker logs arc-test
+              exit 0
+            fi
+            echo "Attempt $i/30 - waiting..."
+            sleep 2
+          done
+
+          echo "❌ Health check failed after 60 seconds"
+          docker logs arc-test
+          exit 1
+
+      - name: Test version endpoint
+        run: |
+          VERSION=${{ needs.prepare.outputs.version }}
+
+          # Check if version endpoint returns expected version
+          RESPONSE=$(curl -sf http://localhost:8000/health || echo "{}")
+          echo "Health response: ${RESPONSE}"
+
+          # Verify version is not "dev"
+          if echo "${RESPONSE}" | grep -q '"version"'; then
+            echo "✅ Version info present in health response"
+          fi
+
+      - name: Cleanup
+        if: always()
+        run: |
+          docker stop arc-test || true
+          docker rm arc-test || true
+
+  # Test binary execution
+  test-binary:
+    name: Test Binary
+    runs-on: ubuntu-latest
+    needs: [prepare, build-binaries]
+    steps:
+      - name: Download amd64 binary
+        uses: actions/download-artifact@v4
+        with:
+          name: arc-binary-linux-amd64
+
+      - name: Test binary execution
+        run: |
+          chmod +x arc-linux-amd64
+
+          # Test --version or basic startup
+          echo "Testing binary execution..."
+
+          # Start Arc in background
+          ./arc-linux-amd64 &
+          ARC_PID=$!
+
+          # Wait for startup
+          sleep 5
+
+          # Test health endpoint
+          echo "Testing health endpoint..."
+          for i in {1..20}; do
+            if curl -sf http://localhost:8000/health; then
+              echo ""
+              echo "✅ Binary health check passed!"
+              kill $ARC_PID || true
+              exit 0
+            fi
+            echo "Attempt $i/20 - waiting..."
+            sleep 2
+          done
+
+          echo "❌ Binary health check failed"
+          kill $ARC_PID || true
+          exit 1
+
   # Package Helm chart
   helm-package:
     name: Package Helm Chart
@@ -602,7 +701,7 @@ jobs:
   create-draft-release:
     name: Create Draft Release
     runs-on: ubuntu-latest
-    needs: [prepare, docker-merge, helm-package, test-helm, debian-build, rpm-build]
+    needs: [prepare, docker-merge, test-docker, test-binary, helm-package, test-helm, debian-build, rpm-build]
     permissions:
       contents: write
     steps:
@@ -624,9 +723,35 @@ jobs:
           VERSION=${{ needs.prepare.outputs.version }}
 
           cat > release-notes.md << 'EOF'
-          # Arc ${{ needs.prepare.outputs.version }} (Go)
+          # Arc v${{ needs.prepare.outputs.version }} - Go Implementation
+
+          **Major Release: Complete rewrite from Python to Go**
+
+          Arc is a high-performance time-series database built on DuckDB, optimized for IoT, observability, and analytics workloads.
+
+          ## 🚀 Migration Highlights
+
+          This release marks the complete migration from Python to Go, delivering:
 
-          High-performance time-series database built on DuckDB. Go implementation.
+          ### Performance Improvements
+          - **9.47M records/sec** MessagePack ingestion (125% faster than Python's 4.21M)
+          - **1.92M records/sec** Line Protocol ingestion (76% faster than Python's 1.09M)
+          - **2.88M rows/sec** Arrow query throughput
+
+          ### Reliability
+          - **Memory stable**: No memory leaks (Python leaked 372MB per 500 queries)
+          - **Single binary**: No Python dependencies, pip, or virtual environments
+          - **Type-safe**: Strong typing catches bugs at compile time
+
+          ### Full Feature Parity
+          - ✅ Authentication (user/password)
+          - ✅ Automatic Compaction (Parquet optimization)
+          - ✅ Write-Ahead Log (WAL for durability)
+          - ✅ Retention Policies (automatic data expiration)
+          - ✅ Continuous Queries (real-time aggregations)
+          - ✅ Delete API (selective data removal)
+          - ✅ S3/MinIO storage backend
+          - ✅ Arrow IPC query responses
 
           ## Quick Start
 
@@ -636,14 +761,14 @@ jobs:
           docker run -d \
             -p 8000:8000 \
             -v arc-data:/app/data \
-            ghcr.io/basekick-labs/arc-go:${{ needs.prepare.outputs.version }}
+            ghcr.io/basekick-labs/arc:${{ needs.prepare.outputs.version }}
           ```
 
           ### Debian/Ubuntu (amd64, arm64)
 
           ```bash
           # Download and install
-          wget https://github.com/basekick-labs/arc-go/releases/download/v${{ needs.prepare.outputs.version }}/arc_${{ needs.prepare.outputs.version }}_amd64.deb
+          wget https://github.com/basekick-labs/arc/releases/download/v${{ needs.prepare.outputs.version }}/arc_${{ needs.prepare.outputs.version }}_amd64.deb
           sudo dpkg -i arc_${{ needs.prepare.outputs.version }}_amd64.deb
 
           # Start and enable
@@ -651,7 +776,6 @@ jobs:
           sudo systemctl start arc
 
           # Check status
-          sudo systemctl status arc
           curl http://localhost:8000/health
           ```
 
@@ -659,7 +783,7 @@ jobs:
 
           ```bash
           # Download and install
-          wget https://github.com/basekick-labs/arc-go/releases/download/v${{ needs.prepare.outputs.version }}/arc-${{ needs.prepare.outputs.version }}-1.x86_64.rpm
+          wget https://github.com/basekick-labs/arc/releases/download/v${{ needs.prepare.outputs.version }}/arc-${{ needs.prepare.outputs.version }}-1.x86_64.rpm
           sudo rpm -i arc-${{ needs.prepare.outputs.version }}-1.x86_64.rpm
 
           # Start and enable
@@ -670,47 +794,41 @@ jobs:
           ### Kubernetes (Helm)
 
           ```bash
-          helm install arc https://github.com/basekick-labs/arc-go/releases/download/v${{ needs.prepare.outputs.version }}/arc-${{ needs.prepare.outputs.version }}.tgz
+          helm install arc https://github.com/basekick-labs/arc/releases/download/v${{ needs.prepare.outputs.version }}/arc-${{ needs.prepare.outputs.version }}.tgz
           ```
 
           ## Download Artifacts
 
           | Platform | Architecture | Package |
           |----------|--------------|---------|
-          | Docker | amd64, arm64 | `ghcr.io/basekick-labs/arc-go:${{ needs.prepare.outputs.version }}` |
+          | Docker | amd64, arm64 | `ghcr.io/basekick-labs/arc:${{ needs.prepare.outputs.version }}` |
           | Debian/Ubuntu | amd64 | `arc_${{ needs.prepare.outputs.version }}_amd64.deb` |
           | Debian/Ubuntu | arm64 | `arc_${{ needs.prepare.outputs.version }}_arm64.deb` |
           | RHEL/Fedora | x86_64 | `arc-${{ needs.prepare.outputs.version }}-1.x86_64.rpm` |
           | RHEL/Fedora | aarch64 | `arc-${{ needs.prepare.outputs.version }}-1.aarch64.rpm` |
           | Kubernetes | - | `arc-${{ needs.prepare.outputs.version }}.tgz` (Helm) |
 
-          ## Performance
+          ## Breaking Changes
 
-          Benchmarked on Apple MacBook Pro M3 Max (14 cores, 36GB RAM):
+          - **Python version**: The Python implementation is preserved in the `python-legacy` branch
+          - **Configuration**: TOML config format (unchanged, but verify your `arc.toml`)
 
-          | Metric | Throughput |
-          |--------|------------|
-          | MessagePack Ingestion | **9.47M rec/s** |
-          | Line Protocol Ingestion | 1.92M rec/s |
-          | Arrow Query | 2.88M rows/s |
+          ## Upgrading from Python
 
-          ## What's New
-
-          - Go implementation with 125% faster ingestion than Python
-          - Stable memory (no leaks)
-          - Single binary deployment
-          - Multi-arch support (amd64, arm64)
-          - Native packages for Debian/Ubuntu and RHEL/Fedora
-          - Systemd integration (auto-start on boot)
+          1. Stop existing Arc service
+          2. Backup your data directory
+          3. Install the new Go binary (same config format)
+          4. Start Arc - data is automatically migrated
 
           ## Documentation
 
-          [docs.basekick.net/arc](https://docs.basekick.net/arc)
+          - [Full Documentation](https://docs.basekick.net/arc)
+          - [Migration Guide](https://github.com/basekick-labs/arc/blob/main/MIGRATION_PLAN.md)
 
           ## Community
 
           - **Discord**: [Join](https://discord.gg/nxnWfUxsdm)
-          - **Issues**: [GitHub](https://github.com/basekick-labs/arc-go/issues)
+          - **Issues**: [GitHub](https://github.com/basekick-labs/arc/issues)
           EOF
 
           cat release-notes.md
@@ -745,14 +863,18 @@ jobs:
           echo "" >> $GITHUB_STEP_SUMMARY
           echo "| Type | Package |" >> $GITHUB_STEP_SUMMARY
           echo "|------|---------|" >> $GITHUB_STEP_SUMMARY
-          echo "| Docker | \`ghcr.io/basekick-labs/arc-go:${VERSION}\` |" >> $GITHUB_STEP_SUMMARY
+          echo "| Docker | \`ghcr.io/basekick-labs/arc:${VERSION}\` |" >> $GITHUB_STEP_SUMMARY
           echo "| Helm | \`arc-${VERSION}.tgz\` |" >> $GITHUB_STEP_SUMMARY
           echo "| Debian amd64 | \`arc_${VERSION}_amd64.deb\` |" >> $GITHUB_STEP_SUMMARY
           echo "| Debian arm64 | \`arc_${VERSION}_arm64.deb\` |" >> $GITHUB_STEP_SUMMARY
           echo "| RPM x86_64 | \`arc-${VERSION}-1.x86_64.rpm\` |" >> $GITHUB_STEP_SUMMARY
           echo "| RPM aarch64 | \`arc-${VERSION}-1.aarch64.rpm\` |" >> $GITHUB_STEP_SUMMARY
           echo "" >> $GITHUB_STEP_SUMMARY
+          echo "### Tests Passed" >> $GITHUB_STEP_SUMMARY
+          echo "- ✅ Docker image health check" >> $GITHUB_STEP_SUMMARY
+          echo "- ✅ Binary execution test" >> $GITHUB_STEP_SUMMARY
+          echo "- ✅ Helm chart deployment" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
           echo "### Next Steps" >> $GITHUB_STEP_SUMMARY
-          echo "1. Download and test artifacts" >> $GITHUB_STEP_SUMMARY
-          echo "2. Review release notes" >> $GITHUB_STEP_SUMMARY
-          echo "3. Publish when ready" >> $GITHUB_STEP_SUMMARY
+          echo "1. Review release notes" >> $GITHUB_STEP_SUMMARY
+          echo "2. Publish when ready" >> $GITHUB_STEP_SUMMARY

From 9bcf9f59b3e466db3e2a746beb47f397e8f416fb Mon Sep 17 00:00:00 2001
From: Ignacio Van Droogenbroeck <ignacio@vandroogenbroeck.net>
Date: Thu, 27 Nov 2025 16:22:19 -0300
Subject: [PATCH 05/11] Add badges, latest Docker tag, update README

- Add throughput badges (9.47M rec/s, 2.88M rows/s)
- Add Go version, license, docs, website, Discord badges
- Add :latest tag to Docker manifest (GHCR doesn't auto-tag)
- Update installation section with Docker, deb, rpm, Helm
- Update repo references from arc-go to arc
- Update Go version from 1.22 to 1.25
---
 .github/workflows/release-build.yml |  5 +--
 README.md                           | 53 ++++++++++++++++++++++-------
 2 files changed, 44 insertions(+), 14 deletions(-)

diff --git a/.github/workflows/release-build.yml b/.github/workflows/release-build.yml
index c7520bb3..b39c74ae 100644
--- a/.github/workflows/release-build.yml
+++ b/.github/workflows/release-build.yml
@@ -468,14 +468,15 @@ jobs:
       - name: Create manifest list and push
         working-directory: /tmp/digests
         run: |
-          # Create multi-arch manifest
+          # Create multi-arch manifest with version, short_version, and latest tags
           docker buildx imagetools create \
             -t ${{ env.REGISTRY_GHCR }}/${{ env.IMAGE_NAME }}:${{ needs.prepare.outputs.version }} \
             -t ${{ env.REGISTRY_GHCR }}/${{ env.IMAGE_NAME }}:${{ needs.prepare.outputs.short_version }} \
+            -t ${{ env.REGISTRY_GHCR }}/${{ env.IMAGE_NAME }}:latest \
             $(printf '${{ env.REGISTRY_GHCR }}/${{ env.IMAGE_NAME }}@sha256:%s ' *)
 
           echo "✅ Multi-arch manifest created and pushed" >> $GITHUB_STEP_SUMMARY
-          echo "Tags: ${{ needs.prepare.outputs.version }}, ${{ needs.prepare.outputs.short_version }}" >> $GITHUB_STEP_SUMMARY
+          echo "Tags: ${{ needs.prepare.outputs.version }}, ${{ needs.prepare.outputs.short_version }}, latest" >> $GITHUB_STEP_SUMMARY
 
   # Test Docker image health
   test-docker:
diff --git a/README.md b/README.md
index d51749dc..ca5b7d43 100644
--- a/README.md
+++ b/README.md
@@ -1,8 +1,16 @@
 # Arc
 
-High-performance time-series database built on DuckDB. Go implementation.
+[![Ingestion](https://img.shields.io/badge/ingestion-9.47M%20rec%2Fs-brightgreen)](https://github.com/basekick-labs/arc)
+[![Query](https://img.shields.io/badge/query-2.88M%20rows%2Fs-blue)](https://github.com/basekick-labs/arc)
+[![Go](https://img.shields.io/badge/go-1.25+-00ADD8?logo=go)](https://go.dev)
+[![License](https://img.shields.io/badge/license-AGPL--3.0-blue)](LICENSE)
+
+[![Docs](https://img.shields.io/badge/docs-basekick.net-blue?logo=gitbook)](https://docs.basekick.net/arc)
+[![Website](https://img.shields.io/badge/website-basekick.net-orange?logo=firefox)](https://basekick.net)
+[![Discord](https://img.shields.io/badge/discord-join-7289da?logo=discord)](https://discord.gg/nxnWfUxsdm)
+[![GitHub](https://img.shields.io/github/stars/basekick-labs/arc?style=social)](https://github.com/basekick-labs/arc)
 
-[Documentation](https://docs.basekick.net/arc)
+High-performance time-series database built on DuckDB. Go implementation.
 
 ---
 
@@ -100,24 +108,45 @@ curl http://localhost:8000/health
 
 ## Installation
 
-### Pre-built Binaries (Coming Soon)
+### Docker
+
+```bash
+docker run -d \
+  -p 8000:8000 \
+  -v arc-data:/app/data \
+  ghcr.io/basekick-labs/arc:latest
+```
 
-- Linux (amd64, arm64)
-- macOS (amd64, arm64)
-- Windows (amd64)
+### Debian/Ubuntu
 
-### Container Images
+```bash
+wget https://github.com/basekick-labs/arc/releases/latest/download/arc_amd64.deb
+sudo dpkg -i arc_amd64.deb
+sudo systemctl enable arc && sudo systemctl start arc
+```
 
-Coming soon.
+### RHEL/Fedora
+
+```bash
+wget https://github.com/basekick-labs/arc/releases/latest/download/arc.x86_64.rpm
+sudo rpm -i arc.x86_64.rpm
+sudo systemctl enable arc && sudo systemctl start arc
+```
+
+### Kubernetes (Helm)
+
+```bash
+helm install arc https://github.com/basekick-labs/arc/releases/latest/download/arc.tgz
+```
 
 ### Build from Source
 
 ```bash
-# Prerequisites: Go 1.22+
+# Prerequisites: Go 1.25+
 
 # Clone and build
-git clone https://github.com/basekick-labs/arc-go.git
-cd arc-go
+git clone https://github.com/basekick-labs/arc.git
+cd arc
 make build
 
 # Run
@@ -176,7 +205,7 @@ See [arc.toml](./arc.toml) for complete configuration reference.
 ## Project Structure
 
 ```
-arc-go/
+arc/
 ├── cmd/arc/           # Application entry point
 ├── internal/
 │   ├── api/           # HTTP handlers (Fiber)

From 1843b5c4ae3fc32a98d605f99da92836f0cf2693 Mon Sep 17 00:00:00 2001
From: Ignacio Van Droogenbroeck <ignacio@vandroogenbroeck.net>
Date: Thu, 27 Nov 2025 17:01:46 -0300
Subject: [PATCH 06/11] Use native ARM runners for binary builds instead of
 QEMU

- Replace Docker/QEMU emulation with native GitHub runners
- Use ubuntu-24.04-arm for linux-arm64 builds (native ARM runner)
- Use ubuntu-latest for linux-amd64 builds
- Enable CGO (required for SQLite auth database)
- Build time reduced from 30+ minutes to ~2-3 minutes
---
 .github/workflows/release-build.yml | 38 ++++++++++++++---------------
 1 file changed, 19 insertions(+), 19 deletions(-)

diff --git a/.github/workflows/release-build.yml b/.github/workflows/release-build.yml
index b39c74ae..4ee8452f 100644
--- a/.github/workflows/release-build.yml
+++ b/.github/workflows/release-build.yml
@@ -118,44 +118,44 @@ jobs:
           if-no-files-found: error
           retention-days: 1
 
-  # Build Go binaries for all platforms using Docker (for CGO cross-compilation)
+  # Build Go binaries using native runners (CGO required for SQLite)
   build-binaries:
     name: Build Binaries
-    runs-on: ubuntu-latest
     needs: prepare
     strategy:
       matrix:
         include:
-          - platform: linux/amd64
+          - runner: ubuntu-latest
             suffix: linux-amd64
-          - platform: linux/arm64
+          - runner: ubuntu-24.04-arm
             suffix: linux-arm64
+    runs-on: ${{ matrix.runner }}
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
 
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: '1.23'
+          cache: true
 
-      - name: Build binary in Docker
+      - name: Build binary
+        env:
+          CGO_ENABLED: 1
         run: |
           VERSION=${{ needs.prepare.outputs.version }}
 
-          # Build using Docker to handle CGO cross-compilation
-          docker buildx build \
-            --platform ${{ matrix.platform }} \
-            --build-arg VERSION=${VERSION} \
-            --target builder \
-            --output type=local,dest=./out \
-            .
+          go build -ldflags "-s -w -X main.Version=${VERSION}" \
+            -o arc-${{ matrix.suffix }} \
+            ./cmd/arc
 
-          # Extract the binary
-          cp ./out/build/arc arc-${{ matrix.suffix }}
           chmod +x arc-${{ matrix.suffix }}
 
+          # Show binary info
+          file arc-${{ matrix.suffix }}
+          ls -lh arc-${{ matrix.suffix }}
+
       - name: Upload binary
         uses: actions/upload-artifact@v4
         with:

From 37bcf900d50fcd0fb0d4f6073da59c0a498cd414 Mon Sep 17 00:00:00 2001
From: Ignacio Van Droogenbroeck <ignacio@vandroogenbroeck.net>
Date: Thu, 27 Nov 2025 17:10:16 -0300
Subject: [PATCH 07/11] Fix RPM build: remove BuildArch, use --target flag

BuildArch is for noarch packages only. For architecture-specific
binaries, use rpmbuild --target instead.
---
 .github/workflows/release-build.yml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.github/workflows/release-build.yml b/.github/workflows/release-build.yml
index 4ee8452f..1cdd678f 100644
--- a/.github/workflows/release-build.yml
+++ b/.github/workflows/release-build.yml
@@ -369,7 +369,6 @@ jobs:
           URL:            https://github.com/basekick-labs/arc
           Source0:        arc-${VERSION}.tar.gz
           Source1:        arc.service
-          BuildArch:      ${ARCH}
 
           %description
           Arc is a high-performance time-series database built on DuckDB,
@@ -422,7 +421,7 @@ jobs:
           EOF
 
           # Build RPM
-          rpmbuild -ba ~/rpmbuild/SPECS/arc.spec
+          rpmbuild -ba --target ${ARCH} ~/rpmbuild/SPECS/arc.spec
 
           # Copy to current directory
           cp ~/rpmbuild/RPMS/${ARCH}/arc-${VERSION}-1.*.rpm .

From 5ba5a5dd6af423e17a648fe27e06dea02b5a4fef Mon Sep 17 00:00:00 2001
From: Ignacio Van Droogenbroeck <ignacio@vandroogenbroeck.net>
Date: Thu, 27 Nov 2025 17:21:50 -0300
Subject: [PATCH 08/11] Add migration for expires_at column in api_tokens

Existing databases created by Python don't have the expires_at column.
This adds a migration that runs ALTER TABLE to add the column if missing.
---
 internal/auth/auth.go | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/internal/auth/auth.go b/internal/auth/auth.go
index 2e3ded46..e5cc59da 100644
--- a/internal/auth/auth.go
+++ b/internal/auth/auth.go
@@ -125,6 +125,15 @@ func (am *AuthManager) initDB() error {
 		return fmt.Errorf("failed to create index: %w", err)
 	}
 
+	// Migration: add expires_at column if it doesn't exist (for databases created by Python)
+	_, err = am.db.Exec(`ALTER TABLE api_tokens ADD COLUMN expires_at TIMESTAMP`)
+	if err != nil {
+		// Ignore "duplicate column" error - column already exists
+		if !strings.Contains(err.Error(), "duplicate column") {
+			am.logger.Debug().Err(err).Msg("expires_at column already exists or migration failed")
+		}
+	}
+
 	return nil
 }
 

From 1e497581fdb4da93f2b189ea1c27486a4fa7d668 Mon Sep 17 00:00:00 2001
From: Ignacio Van Droogenbroeck <ignacio@vandroogenbroeck.net>
Date: Thu, 27 Nov 2025 18:43:13 -0300
Subject: [PATCH 09/11] Handle integer types in float columns (msgpack
 coercion)

MessagePack sends integers (int8, int16, etc.) when values are whole
numbers like 0 or 10. This adds type coercion to float64 for all
integer types in float columns.

Fixes: "unexpected type in float column 'speed': int8"
---
 internal/ingest/arrow_writer.go | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/internal/ingest/arrow_writer.go b/internal/ingest/arrow_writer.go
index b9fef50b..d8db5dd1 100644
--- a/internal/ingest/arrow_writer.go
+++ b/internal/ingest/arrow_writer.go
@@ -1189,6 +1189,27 @@ func (b *ArrowBuffer) convertColumnsToTyped(columns map[string][]interface{}) (m
 						arr[i] = float64(val)
 					case float64:
 						arr[i] = val
+					// Handle integer types (msgpack may send int when value is whole number)
+					case int:
+						arr[i] = float64(val)
+					case int8:
+						arr[i] = float64(val)
+					case int16:
+						arr[i] = float64(val)
+					case int32:
+						arr[i] = float64(val)
+					case int64:
+						arr[i] = float64(val)
+					case uint:
+						arr[i] = float64(val)
+					case uint8:
+						arr[i] = float64(val)
+					case uint16:
+						arr[i] = float64(val)
+					case uint32:
+						arr[i] = float64(val)
+					case uint64:
+						arr[i] = float64(val)
 					default:
 						return nil, 0, fmt.Errorf("unexpected type in float column '%s': %T", name, val)
 					}

From 4d114c252ef20a9fc5c15cb25b695694a25d765b Mon Sep 17 00:00:00 2001
From: Ignacio Van Droogenbroeck <ignacio@vandroogenbroeck.net>
Date: Thu, 27 Nov 2025 20:43:22 -0300
Subject: [PATCH 10/11] Fix directory cache staleness and float-to-int coercion

- local.go: Retry CreateTemp on ENOENT (directory deleted externally),
  invalidate cache and recreate directory before retrying
- arrow_writer.go: Add float32/float64 cases to int column handler,
  truncating floats to int64 when schema inferred integer type
---
 internal/ingest/arrow_writer.go |  5 +++++
 internal/storage/local.go       | 20 +++++++++++++++++++-
 2 files changed, 24 insertions(+), 1 deletion(-)

diff --git a/internal/ingest/arrow_writer.go b/internal/ingest/arrow_writer.go
index d8db5dd1..7f61cca7 100644
--- a/internal/ingest/arrow_writer.go
+++ b/internal/ingest/arrow_writer.go
@@ -1143,6 +1143,11 @@ func (b *ArrowBuffer) convertColumnsToTyped(columns map[string][]interface{}) (m
 						arr[i] = int64(val)
 					case uint64:
 						arr[i] = int64(val)
+					// Handle float types (stream may send float when schema inferred int)
+					case float32:
+						arr[i] = int64(val)
+					case float64:
+						arr[i] = int64(val)
 					default:
 						return nil, 0, fmt.Errorf("unexpected type in int column '%s': %T", name, val)
 					}
diff --git a/internal/storage/local.go b/internal/storage/local.go
index eb572f57..518234ad 100644
--- a/internal/storage/local.go
+++ b/internal/storage/local.go
@@ -80,7 +80,25 @@ func (b *LocalBackend) Write(ctx context.Context, path string, data []byte) erro
 	// Write to temporary file with cryptographically random name (prevents TOCTOU attacks)
 	tmpFile, err := os.CreateTemp(dir, ".arc-*.tmp")
 	if err != nil {
-		return fmt.Errorf("failed to create temp file: %w", err)
+		// Directory might have been deleted externally - invalidate cache and retry
+		if os.IsNotExist(err) {
+			b.dirMu.Lock()
+			delete(b.dirCache, dir)
+			if err := os.MkdirAll(dir, 0700); err != nil {
+				b.dirMu.Unlock()
+				return fmt.Errorf("failed to create directory: %w", err)
+			}
+			b.dirCache[dir] = true
+			b.dirMu.Unlock()
+
+			// Retry CreateTemp
+			tmpFile, err = os.CreateTemp(dir, ".arc-*.tmp")
+			if err != nil {
+				return fmt.Errorf("failed to create temp file: %w", err)
+			}
+		} else {
+			return fmt.Errorf("failed to create temp file: %w", err)
+		}
 	}
 	tmpPath := tmpFile.Name()
 

From 4a27d3f14c983741bc6c1295f8a964de407a4417 Mon Sep 17 00:00:00 2001
From: Ignacio Van Droogenbroeck <ignacio@vandroogenbroeck.net>
Date: Fri, 28 Nov 2025 00:07:01 -0300
Subject: [PATCH 11/11] Fix Arrow column mismatch panic on schema evolution

When Telegraf pushes columnar msgpack data with evolving schemas
(columns appear/disappear between batches), mergeBatches() would
fail because it only allocated space for columns where they exist,
causing: "panic: arrow/array: mismatch number of rows in column"

This fix adds schema-aware buffering that detects when columns change
between batches and flushes the buffer before adding new data. This
ensures all batches in a buffer have identical schemas.

Changes:
- Add bufferSchemas map to track column signature per buffer
- Add getColumnSignature() helper for schema comparison
- Detect schema evolution in writeColumnar() and flush on change
- Clean up bufferSchemas on buffer deletion
---
 internal/ingest/arrow_writer.go | 43 +++++++++++++++++++++++++++++++++
 1 file changed, 43 insertions(+)

diff --git a/internal/ingest/arrow_writer.go b/internal/ingest/arrow_writer.go
index 7f61cca7..6f50b98c 100644
--- a/internal/ingest/arrow_writer.go
+++ b/internal/ingest/arrow_writer.go
@@ -4,6 +4,8 @@ import (
 	"bytes"
 	"context"
 	"fmt"
+	"sort"
+	"strings"
 	"sync"
 	"sync/atomic"
 	"time"
@@ -722,6 +724,7 @@ type bufferShard struct {
 	buffers            map[string][]interface{}
 	bufferStartTimes   map[string]time.Time
 	bufferRecordCounts map[string]int
+	bufferSchemas      map[string]string // Column signature for schema evolution detection
 	mu                 sync.RWMutex
 }
 
@@ -780,6 +783,19 @@ type ArrowBuffer struct {
 	logger zerolog.Logger
 }
 
+// getColumnSignature returns a sorted string of column names for schema comparison.
+// Used to detect schema evolution when columns appear/disappear between batches.
+func getColumnSignature(columns map[string]interface{}) string {
+	names := make([]string, 0, len(columns))
+	for name := range columns {
+		if len(name) > 0 && name[0] != '_' { // Skip internal columns
+			names = append(names, name)
+		}
+	}
+	sort.Strings(names)
+	return strings.Join(names, ",")
+}
+
 // getShard returns the shard for a given buffer key using FNV-1a hash
 func (b *ArrowBuffer) getShard(bufferKey string) *bufferShard {
 	// FNV-1a hash (fast, good distribution)
@@ -818,6 +834,7 @@ func NewArrowBuffer(cfg *config.IngestConfig, storage storage.Backend, logger ze
 			buffers:            make(map[string][]interface{}),
 			bufferStartTimes:   make(map[string]time.Time),
 			bufferRecordCounts: make(map[string]int),
+			bufferSchemas:      make(map[string]string),
 		}
 	}
 
@@ -964,6 +981,9 @@ func (b *ArrowBuffer) writeColumnar(ctx context.Context, database string, record
 		return fmt.Errorf("failed to convert columns: %w", err)
 	}
 
+	// Get column signature for schema evolution detection
+	newSignature := getColumnSignature(typedColumns)
+
 	// OPTIMIZATION: Get shard for this buffer key (lock sharding)
 	shard := b.getShard(bufferKey)
 
@@ -974,10 +994,31 @@ func (b *ArrowBuffer) writeColumnar(ctx context.Context, database string, record
 
 	shard.mu.Lock()
 
+	// Schema evolution detection: flush buffer if columns changed
+	if existingSignature, exists := shard.bufferSchemas[bufferKey]; exists {
+		if existingSignature != newSignature {
+			// Schema changed - flush existing buffer first to avoid column mismatch
+			b.logger.Debug().
+				Str("buffer_key", bufferKey).
+				Str("old_schema", existingSignature).
+				Str("new_schema", newSignature).
+				Msg("Schema evolution detected, flushing buffer")
+
+			if err := b.flushBufferLocked(ctx, shard, bufferKey, database, record.Measurement); err != nil {
+				b.logger.Error().Err(err).
+					Str("buffer_key", bufferKey).
+					Msg("Failed to flush buffer on schema change")
+				// Continue anyway - the buffer is cleared by flushBufferLocked
+			}
+			// Buffer is now empty, will be re-initialized below
+		}
+	}
+
 	// Initialize buffer and record count if needed
 	if _, exists := shard.buffers[bufferKey]; !exists {
 		shard.bufferStartTimes[bufferKey] = time.Now()
 		shard.bufferRecordCounts[bufferKey] = 0
+		shard.bufferSchemas[bufferKey] = newSignature // Store schema for evolution detection
 	}
 
 	// Add typed columns to buffer (already converted via zero-copy fast paths)
@@ -997,6 +1038,7 @@ func (b *ArrowBuffer) writeColumnar(ctx context.Context, database string, record
 		shard.buffers[bufferKey] = nil
 		delete(shard.bufferStartTimes, bufferKey)
 		delete(shard.bufferRecordCounts, bufferKey)
+		delete(shard.bufferSchemas, bufferKey)
 
 		shouldFlush = true
 
@@ -1511,6 +1553,7 @@ func (b *ArrowBuffer) flushBufferLocked(ctx context.Context, shard *bufferShard,
 	delete(shard.buffers, bufferKey)
 	delete(shard.bufferStartTimes, bufferKey)
 	delete(shard.bufferRecordCounts, bufferKey)
+	delete(shard.bufferSchemas, bufferKey)
 
 	// Release lock before expensive operations
 	shard.mu.Unlock()