From 11d282f5d608869407eaca8ba994c038efa36c28 Mon Sep 17 00:00:00 2001 From: ben-fornefeld Date: Wed, 18 Mar 2026 16:49:46 -0700 Subject: [PATCH 1/4] refactor: split dashboard-api queries & migrations --- .github/actions/start-services/action.yml | 1 + .github/workflows/out-of-order-migrations.yml | 48 ++-- iac/modules/job-api/jobs/api.hcl | 2 + .../job-dashboard-api/jobs/dashboard-api.hcl | 24 ++ iac/modules/job-dashboard-api/main.tf | 1 + iac/modules/job-dashboard-api/variables.tf | 4 + iac/provider-gcp/nomad/main.tf | 1 + packages/api/Makefile | 2 +- packages/dashboard-api/Dockerfile | 5 +- packages/dashboard-api/Makefile | 13 +- .../dashboard-api/internal/handlers/build.go | 4 +- .../internal/handlers/builds_list.go | 18 +- .../internal/handlers/builds_statuses.go | 4 +- .../internal/handlers/sandbox_record.go | 4 +- .../internal/handlers/sandbox_record_test.go | 8 +- .../dashboard-api/internal/handlers/store.go | 6 +- packages/dashboard-api/main.go | 29 ++- packages/db/Makefile | 18 ++ packages/db/client/migration.go | 41 ++- packages/db/pkg/dashboard/client.go | 46 ++++ ..._env_defaults_and_team_profile_picture.sql | 24 ++ packages/db/pkg/dashboard/queries/db.go | 32 +++ .../dashboard}/queries/get_build_info.sql.go | 2 +- .../queries/get_builds_paginated.sql.go | 2 +- .../queries/get_builds_statuses.sql.go | 2 +- .../queries/get_sandbox_record.sql.go | 2 +- packages/db/pkg/dashboard/queries/models.go | 245 ++++++++++++++++++ .../sql_queries}/builds/get_build_info.sql | 0 .../builds/get_builds_paginated.sql | 0 .../builds/get_builds_statuses.sql | 0 .../sandboxes/get_sandbox_record.sql | 0 packages/db/pkg/testutils/queries/models.go | 230 ++++++++++++++++ packages/db/scripts/migrator.go | 15 +- packages/db/sqlc.yaml | 54 ++++ scripts/get-latest-migration.sh | 16 +- 35 files changed, 828 insertions(+), 75 deletions(-) create mode 100644 packages/db/pkg/dashboard/client.go create mode 100644 packages/db/pkg/dashboard/migrations/20260318140000_dashboard_add_env_defaults_and_team_profile_picture.sql create mode 100644 packages/db/pkg/dashboard/queries/db.go rename packages/db/{ => pkg/dashboard}/queries/get_build_info.sql.go (98%) rename packages/db/{ => pkg/dashboard}/queries/get_builds_paginated.sql.go (99%) rename packages/db/{ => pkg/dashboard}/queries/get_builds_statuses.sql.go (98%) rename packages/db/{ => pkg/dashboard}/queries/get_sandbox_record.sql.go (98%) create mode 100644 packages/db/pkg/dashboard/queries/models.go rename packages/db/{queries => pkg/dashboard/sql_queries}/builds/get_build_info.sql (100%) rename packages/db/{queries => pkg/dashboard/sql_queries}/builds/get_builds_paginated.sql (100%) rename packages/db/{queries => pkg/dashboard/sql_queries}/builds/get_builds_statuses.sql (100%) rename packages/db/{queries => pkg/dashboard/sql_queries}/sandboxes/get_sandbox_record.sql (100%) diff --git a/.github/actions/start-services/action.yml b/.github/actions/start-services/action.yml index 4cc7c396ab..e130cd0dc9 100644 --- a/.github/actions/start-services/action.yml +++ b/.github/actions/start-services/action.yml @@ -37,6 +37,7 @@ runs: echo "SUPABASE_JWT_SECRETS=${SUPABASE_JWT_SECRET}" >> .env.test set -x make migrate + make -C packages/db migrate-dashboard make -C tests/integration seed shell: bash diff --git a/.github/workflows/out-of-order-migrations.yml b/.github/workflows/out-of-order-migrations.yml index 5184e67e0a..e164fe29dc 100644 --- a/.github/workflows/out-of-order-migrations.yml +++ b/.github/workflows/out-of-order-migrations.yml @@ -15,24 +15,32 @@ jobs: - name: Compare migrations run: | - # Get all migration versions from main - git ls-tree -r origin/main --name-only | grep '^packages/db/migrations/' | grep -oE '[0-9]{14}' | sort -n > main_versions.txt - - # Find the highest version number from main - HIGHEST_MAIN=$(tail -n1 main_versions.txt) - echo "Highest main migration version: $HIGHEST_MAIN" - - # Find newly added migration files in this PR - NEW_FILES=$(git diff --name-status origin/main -- packages/db/migrations/ | grep '^A' | awk '{print $2}') - - for file in $NEW_FILES; do - version=$(basename "$file" | grep -oE '^[0-9]{14}') - echo "Checking new migration version: $version" - if [ "$version" -le "$HIGHEST_MAIN" ]; then - echo "❌ Migration $file is out of order! ($version <= $HIGHEST_MAIN)" - exit 1 - fi - done - + check_stream() { + local dir="$1" + local stream="$2" + local versions_file="$3" + + # Get all migration versions from main + git ls-tree -r origin/main --name-only | grep "^${dir}/" | grep -oE '[0-9]{14}' | sort -n > "$versions_file" + + # Find the highest version number from main + HIGHEST_MAIN=$(tail -n1 "$versions_file") + echo "Highest main ${stream} migration version: $HIGHEST_MAIN" + + # Find newly added migration files in this PR + NEW_FILES=$(git diff --name-status origin/main -- "${dir}/" | grep '^A' | awk '{print $2}') + + for file in $NEW_FILES; do + version=$(basename "$file" | grep -oE '^[0-9]{14}') + echo "Checking new ${stream} migration version: $version" + if [ "$version" -le "$HIGHEST_MAIN" ]; then + echo "❌ Migration $file is out of order! ($version <= $HIGHEST_MAIN)" + exit 1 + fi + done + } + + check_stream "packages/db/migrations" "core" "core_main_versions.txt" + check_stream "packages/db/pkg/dashboard/migrations" "dashboard" "dashboard_main_versions.txt" + echo "✅ All new migrations are in correct order." - diff --git a/iac/modules/job-api/jobs/api.hcl b/iac/modules/job-api/jobs/api.hcl index 972464bf5c..e5e4af6fd5 100644 --- a/iac/modules/job-api/jobs/api.hcl +++ b/iac/modules/job-api/jobs/api.hcl @@ -176,6 +176,8 @@ job "api" { env { POSTGRES_CONNECTION_STRING="${postgres_connection_string}" + MIGRATIONS_DIR="./migrations" + MIGRATIONS_TABLE="_migrations" } config { diff --git a/iac/modules/job-dashboard-api/jobs/dashboard-api.hcl b/iac/modules/job-dashboard-api/jobs/dashboard-api.hcl index f37d681cc9..9357c7fdd2 100644 --- a/iac/modules/job-dashboard-api/jobs/dashboard-api.hcl +++ b/iac/modules/job-dashboard-api/jobs/dashboard-api.hcl @@ -90,5 +90,29 @@ job "dashboard-api" { ports = ["api"] } } + + task "db-migrator" { + driver = "docker" + + env { + POSTGRES_CONNECTION_STRING = "${postgres_connection_string}" + MIGRATIONS_DIR = "./pkg/dashboard/migrations" + MIGRATIONS_TABLE = "_migrations_dashboard" + } + + config { + image = "${db_migrator_docker_image}" + } + + resources { + cpu = 250 + memory = 128 + } + + lifecycle { + hook = "prestart" + sidecar = false + } + } } } diff --git a/iac/modules/job-dashboard-api/main.tf b/iac/modules/job-dashboard-api/main.tf index d1c455e3d1..992e682e76 100644 --- a/iac/modules/job-dashboard-api/main.tf +++ b/iac/modules/job-dashboard-api/main.tf @@ -15,6 +15,7 @@ resource "nomad_job" "dashboard_api" { auth_db_read_replica_connection_string = var.auth_db_read_replica_connection_string clickhouse_connection_string = var.clickhouse_connection_string supabase_jwt_secrets = var.supabase_jwt_secrets + db_migrator_docker_image = var.db_migrator_docker_image subdomain = "dashboard-api" diff --git a/iac/modules/job-dashboard-api/variables.tf b/iac/modules/job-dashboard-api/variables.tf index 8b77e1c9a6..7500556e42 100644 --- a/iac/modules/job-dashboard-api/variables.tf +++ b/iac/modules/job-dashboard-api/variables.tf @@ -14,6 +14,10 @@ variable "image" { type = string } +variable "db_migrator_docker_image" { + type = string +} + variable "count_instances" { type = number } diff --git a/iac/provider-gcp/nomad/main.tf b/iac/provider-gcp/nomad/main.tf index bd48f2dcc9..28c2472abd 100644 --- a/iac/provider-gcp/nomad/main.tf +++ b/iac/provider-gcp/nomad/main.tf @@ -129,6 +129,7 @@ module "dashboard_api" { environment = var.environment image = data.google_artifact_registry_docker_image.dashboard_api_image[0].self_link + db_migrator_docker_image = data.google_artifact_registry_docker_image.db_migrator_image.self_link postgres_connection_string = data.google_secret_manager_secret_version.postgres_connection_string.secret_data auth_db_connection_string = data.google_secret_manager_secret_version.postgres_connection_string.secret_data diff --git a/packages/api/Makefile b/packages/api/Makefile index 9c6b3c7238..41da367c65 100644 --- a/packages/api/Makefile +++ b/packages/api/Makefile @@ -4,7 +4,7 @@ PREFIX := $(strip $(subst ",,$(PREFIX))) HOSTNAME := $(shell hostname 2> /dev/null || hostnamectl hostname 2> /dev/null) $(if $(HOSTNAME),,$(error Failed to determine hostname: both 'hostname' and 'hostnamectl' failed)) -expectedMigration := $(shell ./../../scripts/get-latest-migration.sh) +expectedMigration := $(shell ./../../scripts/get-latest-migration.sh core) ifeq ($(PROVIDER),aws) REGISTRY_PREFIX := $(AWS_ACCOUNT_ID).dkr.ecr.$(AWS_REGION).amazonaws.com/$(PREFIX)core diff --git a/packages/dashboard-api/Dockerfile b/packages/dashboard-api/Dockerfile index 2ca5a1525a..a7d3e8e288 100644 --- a/packages/dashboard-api/Dockerfile +++ b/packages/dashboard-api/Dockerfile @@ -43,8 +43,9 @@ COPY ./dashboard-api/Makefile ./dashboard-api/Makefile WORKDIR /build/dashboard-api ARG COMMIT_SHA -ARG EXPECTED_MIGRATION_TIMESTAMP -RUN --mount=type=cache,target=/root/.cache/go-build make build COMMIT_SHA=${COMMIT_SHA} EXPECTED_MIGRATION_TIMESTAMP=${EXPECTED_MIGRATION_TIMESTAMP} +ARG EXPECTED_CORE_MIGRATION_TIMESTAMP +ARG EXPECTED_DASHBOARD_MIGRATION_TIMESTAMP +RUN --mount=type=cache,target=/root/.cache/go-build make build COMMIT_SHA=${COMMIT_SHA} EXPECTED_CORE_MIGRATION_TIMESTAMP=${EXPECTED_CORE_MIGRATION_TIMESTAMP} EXPECTED_DASHBOARD_MIGRATION_TIMESTAMP=${EXPECTED_DASHBOARD_MIGRATION_TIMESTAMP} RUN chmod +x /build/dashboard-api/bin/dashboard-api FROM alpine:${ALPINE_VERSION} diff --git a/packages/dashboard-api/Makefile b/packages/dashboard-api/Makefile index 5ce6efda2a..c8f8e3c149 100644 --- a/packages/dashboard-api/Makefile +++ b/packages/dashboard-api/Makefile @@ -2,7 +2,8 @@ ENV := $(shell cat ../../.last_used_env || echo "not-set") -include ../../.env.${ENV} PREFIX := $(strip $(subst ",,$(PREFIX))) -expectedMigration := $(shell ./../../scripts/get-latest-migration.sh) +expectedCoreMigration := $(shell ./../../scripts/get-latest-migration.sh core) +expectedDashboardMigration := $(shell ./../../scripts/get-latest-migration.sh dashboard) ifeq ($(PROVIDER),aws) IMAGE_REGISTRY := $(AWS_ACCOUNT_ID).dkr.ecr.$(AWS_REGION).amazonaws.com/$(PREFIX)core/dashboard-api @@ -21,14 +22,16 @@ generate: build: # Allow for passing commit sha directly for docker builds $(eval COMMIT_SHA ?= $(shell git rev-parse --short HEAD)) - $(eval EXPECTED_MIGRATION_TIMESTAMP ?= $(expectedMigration)) - CGO_ENABLED=0 go build -o bin/dashboard-api -ldflags "-X=main.commitSHA=$(COMMIT_SHA) -X=main.expectedMigrationTimestamp=$(EXPECTED_MIGRATION_TIMESTAMP)" . + $(eval EXPECTED_CORE_MIGRATION_TIMESTAMP ?= $(expectedCoreMigration)) + $(eval EXPECTED_DASHBOARD_MIGRATION_TIMESTAMP ?= $(expectedDashboardMigration)) + CGO_ENABLED=0 go build -o bin/dashboard-api -ldflags "-X=main.commitSHA=$(COMMIT_SHA) -X=main.expectedCoreMigrationTimestamp=$(EXPECTED_CORE_MIGRATION_TIMESTAMP) -X=main.expectedDashboardMigrationTimestamp=$(EXPECTED_DASHBOARD_MIGRATION_TIMESTAMP)" . .PHONY: build-and-upload build-and-upload: $(eval COMMIT_SHA := $(shell git rev-parse --short HEAD)) - $(eval EXPECTED_MIGRATION_TIMESTAMP := $(expectedMigration)) - @ docker build --platform linux/amd64 --tag $(IMAGE_REGISTRY) --push --build-arg COMMIT_SHA="$(COMMIT_SHA)" --build-arg EXPECTED_MIGRATION_TIMESTAMP="$(EXPECTED_MIGRATION_TIMESTAMP)" -f ./Dockerfile .. + $(eval EXPECTED_CORE_MIGRATION_TIMESTAMP := $(expectedCoreMigration)) + $(eval EXPECTED_DASHBOARD_MIGRATION_TIMESTAMP := $(expectedDashboardMigration)) + @ docker build --platform linux/amd64 --tag $(IMAGE_REGISTRY) --push --build-arg COMMIT_SHA="$(COMMIT_SHA)" --build-arg EXPECTED_CORE_MIGRATION_TIMESTAMP="$(EXPECTED_CORE_MIGRATION_TIMESTAMP)" --build-arg EXPECTED_DASHBOARD_MIGRATION_TIMESTAMP="$(EXPECTED_DASHBOARD_MIGRATION_TIMESTAMP)" -f ./Dockerfile .. .PHONY: run run: diff --git a/packages/dashboard-api/internal/handlers/build.go b/packages/dashboard-api/internal/handlers/build.go index 90237884e2..4f82ee7a41 100644 --- a/packages/dashboard-api/internal/handlers/build.go +++ b/packages/dashboard-api/internal/handlers/build.go @@ -11,7 +11,7 @@ import ( "github.com/e2b-dev/infra/packages/auth/pkg/auth" "github.com/e2b-dev/infra/packages/dashboard-api/internal/api" dashboardutils "github.com/e2b-dev/infra/packages/dashboard-api/internal/utils" - "github.com/e2b-dev/infra/packages/db/queries" + dashboardqueries "github.com/e2b-dev/infra/packages/db/pkg/dashboard/queries" "github.com/e2b-dev/infra/packages/shared/pkg/logger" "github.com/e2b-dev/infra/packages/shared/pkg/telemetry" ) @@ -22,7 +22,7 @@ func (s *APIStore) GetBuildsBuildId(c *gin.Context, buildId api.BuildId) { teamID := auth.MustGetTeamInfo(c).Team.ID telemetry.SetAttributes(ctx, telemetry.WithTeamID(teamID.String()), telemetry.WithBuildID(buildId.String())) - row, err := s.db.GetBuildInfoByTeamAndBuildID(ctx, queries.GetBuildInfoByTeamAndBuildIDParams{ + row, err := s.db.GetBuildInfoByTeamAndBuildID(ctx, dashboardqueries.GetBuildInfoByTeamAndBuildIDParams{ TeamID: teamID, BuildID: buildId, }) diff --git a/packages/dashboard-api/internal/handlers/builds_list.go b/packages/dashboard-api/internal/handlers/builds_list.go index 077e7de7b9..4eecf4f949 100644 --- a/packages/dashboard-api/internal/handlers/builds_list.go +++ b/packages/dashboard-api/internal/handlers/builds_list.go @@ -14,8 +14,8 @@ import ( "github.com/e2b-dev/infra/packages/auth/pkg/auth" "github.com/e2b-dev/infra/packages/dashboard-api/internal/api" dashboardutils "github.com/e2b-dev/infra/packages/dashboard-api/internal/utils" + dashboardqueries "github.com/e2b-dev/infra/packages/db/pkg/dashboard/queries" dbtypes "github.com/e2b-dev/infra/packages/db/pkg/types" - "github.com/e2b-dev/infra/packages/db/queries" "github.com/e2b-dev/infra/packages/shared/pkg/logger" "github.com/e2b-dev/infra/packages/shared/pkg/telemetry" ) @@ -109,7 +109,7 @@ func (s *APIStore) listBuildRows( statuses := buildStatusGroupsToStrings(statusGroups) if buildIDOrTemplate == nil || strings.TrimSpace(*buildIDOrTemplate) == "" { - rows, err := s.db.GetTeamBuildsPage(ctx, queries.GetTeamBuildsPageParams{ + rows, err := s.db.GetTeamBuildsPage(ctx, dashboardqueries.GetTeamBuildsPageParams{ TeamID: teamID, CursorCreatedAt: cursorTime, CursorID: cursorID, @@ -126,7 +126,7 @@ func (s *APIStore) listBuildRows( filter := strings.TrimSpace(*buildIDOrTemplate) filterUUID, parseErr := uuid.Parse(filter) if parseErr == nil { - byBuildIDRows, byBuildIDErr := s.db.GetTeamBuildsPageByBuildID(ctx, queries.GetTeamBuildsPageByBuildIDParams{ + byBuildIDRows, byBuildIDErr := s.db.GetTeamBuildsPageByBuildID(ctx, dashboardqueries.GetTeamBuildsPageByBuildIDParams{ TeamID: teamID, BuildID: filterUUID, CursorCreatedAt: cursorTime, @@ -144,7 +144,7 @@ func (s *APIStore) listBuildRows( // templateIDs are not UUIDs if parseErr != nil { - byTemplateIDRows, byTemplateIDErr := s.db.GetTeamBuildsPageByTemplateID(ctx, queries.GetTeamBuildsPageByTemplateIDParams{ + byTemplateIDRows, byTemplateIDErr := s.db.GetTeamBuildsPageByTemplateID(ctx, dashboardqueries.GetTeamBuildsPageByTemplateIDParams{ TemplateID: filter, TeamID: teamID, CursorCreatedAt: cursorTime, @@ -160,7 +160,7 @@ func (s *APIStore) listBuildRows( } } - byTemplateAliasRows, byTemplateAliasErr := s.db.GetTeamBuildsPageByTemplateAlias(ctx, queries.GetTeamBuildsPageByTemplateAliasParams{ + byTemplateAliasRows, byTemplateAliasErr := s.db.GetTeamBuildsPageByTemplateAlias(ctx, dashboardqueries.GetTeamBuildsPageByTemplateAliasParams{ TemplateAlias: filter, TeamID: teamID, CursorCreatedAt: cursorTime, @@ -233,7 +233,7 @@ func parseCursorTime(value string) (time.Time, error) { return time.Parse(time.RFC3339, value) } -func mapBuildRows(rows []queries.GetTeamBuildsPageRow) []listBuildRow { +func mapBuildRows(rows []dashboardqueries.GetTeamBuildsPageRow) []listBuildRow { out := make([]listBuildRow, 0, len(rows)) for _, row := range rows { out = append(out, listBuildRow{ @@ -250,7 +250,7 @@ func mapBuildRows(rows []queries.GetTeamBuildsPageRow) []listBuildRow { return out } -func mapBuildRowsByBuildID(rows []queries.GetTeamBuildsPageByBuildIDRow) []listBuildRow { +func mapBuildRowsByBuildID(rows []dashboardqueries.GetTeamBuildsPageByBuildIDRow) []listBuildRow { out := make([]listBuildRow, 0, len(rows)) for _, row := range rows { out = append(out, listBuildRow{ @@ -267,7 +267,7 @@ func mapBuildRowsByBuildID(rows []queries.GetTeamBuildsPageByBuildIDRow) []listB return out } -func mapBuildRowsByTemplateID(rows []queries.GetTeamBuildsPageByTemplateIDRow) []listBuildRow { +func mapBuildRowsByTemplateID(rows []dashboardqueries.GetTeamBuildsPageByTemplateIDRow) []listBuildRow { out := make([]listBuildRow, 0, len(rows)) for _, row := range rows { out = append(out, listBuildRow{ @@ -284,7 +284,7 @@ func mapBuildRowsByTemplateID(rows []queries.GetTeamBuildsPageByTemplateIDRow) [ return out } -func mapBuildRowsByTemplateAlias(rows []queries.GetTeamBuildsPageByTemplateAliasRow) []listBuildRow { +func mapBuildRowsByTemplateAlias(rows []dashboardqueries.GetTeamBuildsPageByTemplateAliasRow) []listBuildRow { out := make([]listBuildRow, 0, len(rows)) for _, row := range rows { out = append(out, listBuildRow{ diff --git a/packages/dashboard-api/internal/handlers/builds_statuses.go b/packages/dashboard-api/internal/handlers/builds_statuses.go index b49e38be51..b003acf184 100644 --- a/packages/dashboard-api/internal/handlers/builds_statuses.go +++ b/packages/dashboard-api/internal/handlers/builds_statuses.go @@ -9,7 +9,7 @@ import ( "github.com/e2b-dev/infra/packages/auth/pkg/auth" "github.com/e2b-dev/infra/packages/dashboard-api/internal/api" dashboardutils "github.com/e2b-dev/infra/packages/dashboard-api/internal/utils" - "github.com/e2b-dev/infra/packages/db/queries" + dashboardqueries "github.com/e2b-dev/infra/packages/db/pkg/dashboard/queries" "github.com/e2b-dev/infra/packages/shared/pkg/logger" "github.com/e2b-dev/infra/packages/shared/pkg/telemetry" ) @@ -32,7 +32,7 @@ func (s *APIStore) GetBuildsStatuses(c *gin.Context, params api.GetBuildsStatuse return } - p := queries.GetBuildsStatusesByTeamParams{ + p := dashboardqueries.GetBuildsStatusesByTeamParams{ TeamID: teamID, BuildIds: params.BuildIds, } diff --git a/packages/dashboard-api/internal/handlers/sandbox_record.go b/packages/dashboard-api/internal/handlers/sandbox_record.go index 823c597ff2..9fe3103b07 100644 --- a/packages/dashboard-api/internal/handlers/sandbox_record.go +++ b/packages/dashboard-api/internal/handlers/sandbox_record.go @@ -12,7 +12,7 @@ import ( "github.com/e2b-dev/infra/packages/auth/pkg/auth" "github.com/e2b-dev/infra/packages/dashboard-api/internal/api" - "github.com/e2b-dev/infra/packages/db/queries" + dashboardqueries "github.com/e2b-dev/infra/packages/db/pkg/dashboard/queries" "github.com/e2b-dev/infra/packages/shared/pkg/logger" "github.com/e2b-dev/infra/packages/shared/pkg/telemetry" ) @@ -30,7 +30,7 @@ func (s *APIStore) GetSandboxesSandboxIDRecord(c *gin.Context, sandboxID api.San teamID := auth.MustGetTeamInfo(c).Team.ID telemetry.SetAttributes(ctx, telemetry.WithTeamID(teamID.String()), telemetry.WithSandboxID(sandboxID)) - row, err := s.db.GetSandboxRecordByTeamAndSandboxID(ctx, queries.GetSandboxRecordByTeamAndSandboxIDParams{ + row, err := s.db.GetSandboxRecordByTeamAndSandboxID(ctx, dashboardqueries.GetSandboxRecordByTeamAndSandboxIDParams{ TeamID: teamID, SandboxID: sandboxID, CreatedAfter: time.Now().UTC().Add(-sandboxRecordRetention), diff --git a/packages/dashboard-api/internal/handlers/sandbox_record_test.go b/packages/dashboard-api/internal/handlers/sandbox_record_test.go index 838795311e..31cba7eada 100644 --- a/packages/dashboard-api/internal/handlers/sandbox_record_test.go +++ b/packages/dashboard-api/internal/handlers/sandbox_record_test.go @@ -15,9 +15,9 @@ import ( "github.com/e2b-dev/infra/packages/auth/pkg/auth" authtypes "github.com/e2b-dev/infra/packages/auth/pkg/types" "github.com/e2b-dev/infra/packages/dashboard-api/internal/api" - sqlcdb "github.com/e2b-dev/infra/packages/db/client" authqueries "github.com/e2b-dev/infra/packages/db/pkg/auth/queries" - "github.com/e2b-dev/infra/packages/db/queries" + dashboarddb "github.com/e2b-dev/infra/packages/db/pkg/dashboard" + dashboardqueries "github.com/e2b-dev/infra/packages/db/pkg/dashboard/queries" ) type noRowsDB struct{} @@ -56,8 +56,8 @@ func TestGetSandboxesSandboxIDRecordReturns404WhenRecordRetentionNotMet(t *testi }) store := &APIStore{ - db: &sqlcdb.Client{ - Queries: queries.New(noRowsDB{}), + db: &dashboarddb.Client{ + Queries: dashboardqueries.New(noRowsDB{}), }, } diff --git a/packages/dashboard-api/internal/handlers/store.go b/packages/dashboard-api/internal/handlers/store.go index 0cb45346a3..e0356d6fa7 100644 --- a/packages/dashboard-api/internal/handlers/store.go +++ b/packages/dashboard-api/internal/handlers/store.go @@ -12,8 +12,8 @@ import ( clickhouse "github.com/e2b-dev/infra/packages/clickhouse/pkg" "github.com/e2b-dev/infra/packages/dashboard-api/internal/api" "github.com/e2b-dev/infra/packages/dashboard-api/internal/cfg" - sqlcdb "github.com/e2b-dev/infra/packages/db/client" authdb "github.com/e2b-dev/infra/packages/db/pkg/auth" + dashboarddb "github.com/e2b-dev/infra/packages/db/pkg/dashboard" "github.com/e2b-dev/infra/packages/shared/pkg/apierrors" ) @@ -21,13 +21,13 @@ var _ api.ServerInterface = (*APIStore)(nil) type APIStore struct { config cfg.Config - db *sqlcdb.Client + db *dashboarddb.Client authDB *authdb.Client clickhouse clickhouse.Clickhouse authService *sharedauth.AuthService[*types.Team] } -func NewAPIStore(config cfg.Config, db *sqlcdb.Client, authDB *authdb.Client, ch clickhouse.Clickhouse, authService *sharedauth.AuthService[*types.Team]) *APIStore { +func NewAPIStore(config cfg.Config, db *dashboarddb.Client, authDB *authdb.Client, ch clickhouse.Clickhouse, authService *sharedauth.AuthService[*types.Team]) *APIStore { return &APIStore{ config: config, db: db, diff --git a/packages/dashboard-api/main.go b/packages/dashboard-api/main.go index 4b40634473..4502076909 100644 --- a/packages/dashboard-api/main.go +++ b/packages/dashboard-api/main.go @@ -32,6 +32,7 @@ import ( "github.com/e2b-dev/infra/packages/dashboard-api/internal/handlers" sqlcdb "github.com/e2b-dev/infra/packages/db/client" authdb "github.com/e2b-dev/infra/packages/db/pkg/auth" + dashboarddb "github.com/e2b-dev/infra/packages/db/pkg/dashboard" "github.com/e2b-dev/infra/packages/db/pkg/pool" e2benv "github.com/e2b-dev/infra/packages/shared/pkg/env" "github.com/e2b-dev/infra/packages/shared/pkg/logger" @@ -50,8 +51,9 @@ const ( ) var ( - commitSHA string - expectedMigrationTimestamp string + commitSHA string + expectedCoreMigrationTimestamp string + expectedDashboardMigrationTimestamp string ) func run() int { @@ -93,22 +95,33 @@ func run() int { l.Info(ctx, "Starting dashboard-api service...", zap.String("commit_sha", commitSHA), zap.String("instance_id", serviceInstanceID)) - expectedMigration, err := strconv.ParseInt(expectedMigrationTimestamp, 10, 64) + expectedCoreMigration, err := strconv.ParseInt(expectedCoreMigrationTimestamp, 10, 64) if err != nil { - l.Warn(ctx, "Failed to parse expected migration timestamp", zap.Error(err)) - expectedMigration = 0 + l.Warn(ctx, "Failed to parse expected core migration timestamp", zap.Error(err)) + expectedCoreMigration = 0 } - err = sqlcdb.CheckMigrationVersion(ctx, config.PostgresConnectionString, expectedMigration) + expectedDashboardMigration, err := strconv.ParseInt(expectedDashboardMigrationTimestamp, 10, 64) if err != nil { - l.Fatal(ctx, "failed to check migration version", zap.Error(err)) + l.Warn(ctx, "Failed to parse expected dashboard migration timestamp", zap.Error(err)) + expectedDashboardMigration = 0 + } + + err = sqlcdb.CheckMigrationVersionWithTable(ctx, config.PostgresConnectionString, "_migrations", expectedCoreMigration) + if err != nil { + l.Fatal(ctx, "failed to check core migration version", zap.Error(err)) + } + + err = sqlcdb.CheckMigrationVersionWithTable(ctx, config.PostgresConnectionString, "_migrations_dashboard", expectedDashboardMigration) + if err != nil { + l.Fatal(ctx, "failed to check dashboard migration version", zap.Error(err)) } if !e2benv.IsDebug() { gin.SetMode(gin.ReleaseMode) } - db, err := sqlcdb.NewClient( + db, err := dashboarddb.NewClient( ctx, config.PostgresConnectionString, pool.WithMaxConnections(8), diff --git a/packages/db/Makefile b/packages/db/Makefile index e5f1d67439..56d8bd5134 100644 --- a/packages/db/Makefile +++ b/packages/db/Makefile @@ -4,6 +4,8 @@ PREFIX := $(strip $(subst ",,$(PREFIX))) goose := GOOSE_DRIVER=postgres GOOSE_DBSTRING=$(POSTGRES_CONNECTION_STRING) go tool goose -table "_migrations" -dir "migrations" goose-local := GOOSE_DBSTRING=postgres://postgres:postgres@localhost:5432/postgres?sslmode=disable go tool goose -table "_migrations" -dir "migrations" postgres +goose-dashboard := GOOSE_DRIVER=postgres GOOSE_DBSTRING=$(POSTGRES_CONNECTION_STRING) go tool goose -table "_migrations_dashboard" -dir "pkg/dashboard/migrations" +goose-dashboard-local := GOOSE_DBSTRING=postgres://postgres:postgres@localhost:5432/postgres?sslmode=disable go tool goose -table "_migrations_dashboard" -dir "pkg/dashboard/migrations" postgres .PHONY: migrate @@ -18,6 +20,18 @@ migrate-local: @$(goose-local) up @echo "Done" +.PHONY: migrate-dashboard +migrate-dashboard: + @echo "Applying Postgres migration *$(notdir $@)* for ${ENV}" + @$(goose-dashboard) up + @echo "Done" + +.PHONY: migrate-dashboard-local +migrate-dashboard-local: + @echo "Applying Postgres migration *$(notdir $@)* for ${ENV}" + @$(goose-dashboard-local) up + @echo "Done" + .PHONY: migrate/up migrate/up: migrate @@ -44,6 +58,10 @@ endif status: @$(goose) status +.PHONY: status-dashboard +status-dashboard: + @$(goose-dashboard) status + .PHONY: generate generate: rm -rf queries/*.go diff --git a/packages/db/client/migration.go b/packages/db/client/migration.go index 619659a4b1..db808f9682 100644 --- a/packages/db/client/migration.go +++ b/packages/db/client/migration.go @@ -4,6 +4,8 @@ import ( "context" "database/sql" "fmt" + "regexp" + "sync" _ "github.com/lib/pq" //nolint:blank-imports "github.com/pressly/goose/v3" @@ -12,13 +14,26 @@ import ( "github.com/e2b-dev/infra/packages/shared/pkg/logger" ) -const trackingTable = "_migrations" +const coreTrackingTable = "_migrations" + +var ( + migrationTablePattern = regexp.MustCompile(`^[a-zA-Z_][a-zA-Z0-9_]*$`) + gooseTableMu sync.Mutex +) func init() { - goose.SetTableName(trackingTable) + goose.SetTableName(coreTrackingTable) } func CheckMigrationVersion(ctx context.Context, connectionString string, expectedMigration int64) error { + return CheckMigrationVersionWithTable(ctx, connectionString, coreTrackingTable, expectedMigration) +} + +func CheckMigrationVersionWithTable(ctx context.Context, connectionString, table string, expectedMigration int64) error { + if !migrationTablePattern.MatchString(table) { + return fmt.Errorf("invalid migration table name: %s", table) + } + db, err := sql.Open("postgres", connectionString) if err != nil { return fmt.Errorf("failed to connect: %w", err) @@ -30,18 +45,32 @@ func CheckMigrationVersion(ctx context.Context, connectionString string, expecte } }() - version, err := goose.GetDBVersion(db) + version, err := getDBVersionWithTable(db, table) if err != nil { - return fmt.Errorf("failed to get database version: %w", err) + return fmt.Errorf("failed to get database version from %s: %w", table, err) } // Check if the database version is less than the expected migration // We allow higher versions to account for future migrations and rollbacks if version < expectedMigration { - return fmt.Errorf("database version %d is less than expected %d", version, expectedMigration) + return fmt.Errorf("database version %d in %s is less than expected %d", version, table, expectedMigration) } - logger.L().Info(ctx, "Database version", zap.Int64("version", version), zap.Int64("expected_migration", expectedMigration)) + logger.L().Info(ctx, "Database version", + zap.String("table", table), + zap.Int64("version", version), + zap.Int64("expected_migration", expectedMigration), + ) return nil } + +func getDBVersionWithTable(db *sql.DB, table string) (int64, error) { + gooseTableMu.Lock() + defer gooseTableMu.Unlock() + + goose.SetTableName(table) + defer goose.SetTableName(coreTrackingTable) + + return goose.GetDBVersion(db) +} diff --git a/packages/db/pkg/dashboard/client.go b/packages/db/pkg/dashboard/client.go new file mode 100644 index 0000000000..d0c390457e --- /dev/null +++ b/packages/db/pkg/dashboard/client.go @@ -0,0 +1,46 @@ +package dashboarddb + +import ( + "context" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + _ "github.com/lib/pq" //nolint:blank-imports + + dashboardqueries "github.com/e2b-dev/infra/packages/db/pkg/dashboard/queries" + "github.com/e2b-dev/infra/packages/db/pkg/pool" +) + +const poolName = "dashboard" + +type Client struct { + *dashboardqueries.Queries + + conn *pgxpool.Pool +} + +func NewClient(ctx context.Context, databaseURL string, options ...pool.Option) (*Client, error) { + dbClient, connPool, err := pool.New(ctx, databaseURL, poolName, options...) + if err != nil { + return nil, err + } + + return &Client{Queries: dashboardqueries.New(dbClient), conn: connPool}, nil +} + +func (db *Client) Close() error { + db.conn.Close() + + return nil +} + +func (db *Client) WithTx(ctx context.Context) (*Client, pgx.Tx, error) { + tx, err := db.conn.BeginTx(ctx, pgx.TxOptions{}) + if err != nil { + return nil, nil, err + } + + client := &Client{Queries: db.Queries.WithTx(tx), conn: db.conn} + + return client, tx, nil +} diff --git a/packages/db/pkg/dashboard/migrations/20260318140000_dashboard_add_env_defaults_and_team_profile_picture.sql b/packages/db/pkg/dashboard/migrations/20260318140000_dashboard_add_env_defaults_and_team_profile_picture.sql new file mode 100644 index 0000000000..e1216f22e8 --- /dev/null +++ b/packages/db/pkg/dashboard/migrations/20260318140000_dashboard_add_env_defaults_and_team_profile_picture.sql @@ -0,0 +1,24 @@ +-- +goose Up +-- +goose StatementBegin + +CREATE TABLE IF NOT EXISTS public.env_defaults ( + env_id TEXT PRIMARY KEY REFERENCES public.envs(id), + description TEXT +); + +ALTER TABLE public.env_defaults ENABLE ROW LEVEL SECURITY; + +CREATE INDEX IF NOT EXISTS env_defaults_env_id_idx ON public.env_defaults(env_id); + +ALTER TABLE public.teams ADD COLUMN IF NOT EXISTS profile_picture_url TEXT; + +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin + +ALTER TABLE public.teams DROP COLUMN IF EXISTS profile_picture_url; +DROP INDEX IF EXISTS env_defaults_env_id_idx; +DROP TABLE IF EXISTS public.env_defaults; + +-- +goose StatementEnd diff --git a/packages/db/pkg/dashboard/queries/db.go b/packages/db/pkg/dashboard/queries/db.go new file mode 100644 index 0000000000..8da2a63008 --- /dev/null +++ b/packages/db/pkg/dashboard/queries/db.go @@ -0,0 +1,32 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.29.0 + +package dashboardqueries + +import ( + "context" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" +) + +type DBTX interface { + Exec(context.Context, string, ...interface{}) (pgconn.CommandTag, error) + Query(context.Context, string, ...interface{}) (pgx.Rows, error) + QueryRow(context.Context, string, ...interface{}) pgx.Row +} + +func New(db DBTX) *Queries { + return &Queries{db: db} +} + +type Queries struct { + db DBTX +} + +func (q *Queries) WithTx(tx pgx.Tx) *Queries { + return &Queries{ + db: tx, + } +} diff --git a/packages/db/queries/get_build_info.sql.go b/packages/db/pkg/dashboard/queries/get_build_info.sql.go similarity index 98% rename from packages/db/queries/get_build_info.sql.go rename to packages/db/pkg/dashboard/queries/get_build_info.sql.go index b374608be4..bf2134b727 100644 --- a/packages/db/queries/get_build_info.sql.go +++ b/packages/db/pkg/dashboard/queries/get_build_info.sql.go @@ -3,7 +3,7 @@ // sqlc v1.29.0 // source: get_build_info.sql -package queries +package dashboardqueries import ( "context" diff --git a/packages/db/queries/get_builds_paginated.sql.go b/packages/db/pkg/dashboard/queries/get_builds_paginated.sql.go similarity index 99% rename from packages/db/queries/get_builds_paginated.sql.go rename to packages/db/pkg/dashboard/queries/get_builds_paginated.sql.go index bd4e3451e8..e85155b74a 100644 --- a/packages/db/queries/get_builds_paginated.sql.go +++ b/packages/db/pkg/dashboard/queries/get_builds_paginated.sql.go @@ -3,7 +3,7 @@ // sqlc v1.29.0 // source: get_builds_paginated.sql -package queries +package dashboardqueries import ( "context" diff --git a/packages/db/queries/get_builds_statuses.sql.go b/packages/db/pkg/dashboard/queries/get_builds_statuses.sql.go similarity index 98% rename from packages/db/queries/get_builds_statuses.sql.go rename to packages/db/pkg/dashboard/queries/get_builds_statuses.sql.go index 057a354f5b..31146620f3 100644 --- a/packages/db/queries/get_builds_statuses.sql.go +++ b/packages/db/pkg/dashboard/queries/get_builds_statuses.sql.go @@ -3,7 +3,7 @@ // sqlc v1.29.0 // source: get_builds_statuses.sql -package queries +package dashboardqueries import ( "context" diff --git a/packages/db/queries/get_sandbox_record.sql.go b/packages/db/pkg/dashboard/queries/get_sandbox_record.sql.go similarity index 98% rename from packages/db/queries/get_sandbox_record.sql.go rename to packages/db/pkg/dashboard/queries/get_sandbox_record.sql.go index 891331d970..e5264aad4e 100644 --- a/packages/db/queries/get_sandbox_record.sql.go +++ b/packages/db/pkg/dashboard/queries/get_sandbox_record.sql.go @@ -3,7 +3,7 @@ // sqlc v1.29.0 // source: get_sandbox_record.sql -package queries +package dashboardqueries import ( "context" diff --git a/packages/db/pkg/dashboard/queries/models.go b/packages/db/pkg/dashboard/queries/models.go new file mode 100644 index 0000000000..8bc439df6e --- /dev/null +++ b/packages/db/pkg/dashboard/queries/models.go @@ -0,0 +1,245 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.29.0 + +package dashboardqueries + +import ( + "time" + + "github.com/e2b-dev/infra/packages/db/pkg/types" + "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgtype" +) + +type AccessToken struct { + UserID uuid.UUID + CreatedAt time.Time + ID uuid.UUID + // sensitive + AccessTokenHash string + Name string + AccessTokenPrefix string + AccessTokenLength int32 + AccessTokenMaskPrefix string + AccessTokenMaskSuffix string +} + +type ActiveTemplateBuild struct { + BuildID uuid.UUID + TeamID uuid.UUID + TemplateID string + Tags []string + CreatedAt time.Time +} + +type Addon struct { + ID uuid.UUID + TeamID uuid.UUID + Name string + Description *string + ExtraConcurrentSandboxes int64 + ExtraConcurrentTemplateBuilds int64 + ExtraMaxVcpu int64 + ExtraMaxRamMb int64 + ExtraDiskMb int64 + ValidFrom time.Time + ValidTo *time.Time + AddedBy uuid.UUID + IdempotencyKey *string +} + +type AuthUser struct { + ID uuid.UUID + Email string +} + +type BillingSandboxLog struct { + SandboxID string + EnvID string + Vcpu int64 + RamMb int64 + TotalDiskSizeMb int64 + StartedAt time.Time + StoppedAt *time.Time + CreatedAt time.Time + TeamID uuid.UUID +} + +type Cluster struct { + ID uuid.UUID + Endpoint string + EndpointTls bool + Token string + SandboxProxyDomain *string +} + +type Env struct { + ID string + CreatedAt time.Time + UpdatedAt time.Time + Public bool + BuildCount int32 + // Number of times the env was spawned + SpawnCount int64 + // Timestamp of the last time the env was spawned + LastSpawnedAt *time.Time + TeamID uuid.UUID + CreatedBy *uuid.UUID + ClusterID *uuid.UUID + Source string +} + +type EnvAlias struct { + Alias string + IsRenamable bool + EnvID string + Namespace *string + ID uuid.UUID +} + +type EnvBuild struct { + ID uuid.UUID + CreatedAt time.Time + UpdatedAt time.Time + FinishedAt *time.Time + Status types.BuildStatus + Dockerfile *string + StartCmd *string + Vcpu int64 + RamMb int64 + FreeDiskSizeMb int64 + TotalDiskSizeMb *int64 + KernelVersion string + FirecrackerVersion string + EnvID *string + EnvdVersion *string + ReadyCmd *string + ClusterNodeID *string + Reason types.BuildReason + Version *string + CpuArchitecture *string + CpuFamily *string + CpuModel *string + CpuModelName *string + CpuFlags []string + StatusGroup types.BuildStatusGroup + TeamID *uuid.UUID +} + +type EnvBuildAssignment struct { + ID uuid.UUID + EnvID string + BuildID uuid.UUID + Tag string + Source string + CreatedAt pgtype.Timestamptz +} + +type EnvDefault struct { + EnvID string + Description *string +} + +type Snapshot struct { + CreatedAt pgtype.Timestamptz + EnvID string + SandboxID string + ID uuid.UUID + Metadata types.JSONBStringMap + BaseEnvID string + SandboxStartedAt pgtype.Timestamptz + EnvSecure bool + OriginNodeID string + AllowInternetAccess *bool + AutoPause bool + TeamID uuid.UUID + Config types.JSONBStringMap +} + +type SnapshotTemplate struct { + EnvID string + SandboxID string + CreatedAt pgtype.Timestamptz + OriginNodeID *string + BuildID *uuid.UUID +} + +type Team struct { + ID uuid.UUID + CreatedAt time.Time + IsBlocked bool + Name string + Tier string + Email string + IsBanned bool + BlockedReason *string + ClusterID *uuid.UUID + SandboxSchedulingLabels []string + ProfilePictureUrl *string + Slug string +} + +type TeamApiKey struct { + CreatedAt time.Time + TeamID uuid.UUID + UpdatedAt *time.Time + Name string + LastUsed *time.Time + CreatedBy *uuid.UUID + ID uuid.UUID + // sensitive + ApiKeyHash string + ApiKeyPrefix string + ApiKeyLength int32 + ApiKeyMaskPrefix string + ApiKeyMaskSuffix string +} + +type TeamLimit struct { + ID uuid.UUID + MaxLengthHours int64 + ConcurrentSandboxes int32 + ConcurrentTemplateBuilds int32 + MaxVcpu int32 + MaxRamMb int32 + DiskMb int32 +} + +type Tier struct { + ID string + Name string + DiskMb int64 + // The number of instances the team can run concurrently + ConcurrentInstances int64 + MaxLengthHours int64 + MaxVcpu int64 + MaxRamMb int64 + // The number of concurrent template builds the team can run + ConcurrentTemplateBuilds int64 +} + +type User struct { + CreatedAt time.Time + UpdatedAt time.Time + ID uuid.UUID + Email string +} + +type UsersTeam struct { + ID int64 + UserID uuid.UUID + TeamID uuid.UUID + IsDefault bool + AddedBy *uuid.UUID + CreatedAt pgtype.Timestamp + UuidID uuid.UUID +} + +type Volume struct { + ID uuid.UUID + TeamID uuid.UUID + Name string + VolumeType string + CreatedAt pgtype.Timestamptz +} diff --git a/packages/db/queries/builds/get_build_info.sql b/packages/db/pkg/dashboard/sql_queries/builds/get_build_info.sql similarity index 100% rename from packages/db/queries/builds/get_build_info.sql rename to packages/db/pkg/dashboard/sql_queries/builds/get_build_info.sql diff --git a/packages/db/queries/builds/get_builds_paginated.sql b/packages/db/pkg/dashboard/sql_queries/builds/get_builds_paginated.sql similarity index 100% rename from packages/db/queries/builds/get_builds_paginated.sql rename to packages/db/pkg/dashboard/sql_queries/builds/get_builds_paginated.sql diff --git a/packages/db/queries/builds/get_builds_statuses.sql b/packages/db/pkg/dashboard/sql_queries/builds/get_builds_statuses.sql similarity index 100% rename from packages/db/queries/builds/get_builds_statuses.sql rename to packages/db/pkg/dashboard/sql_queries/builds/get_builds_statuses.sql diff --git a/packages/db/queries/sandboxes/get_sandbox_record.sql b/packages/db/pkg/dashboard/sql_queries/sandboxes/get_sandbox_record.sql similarity index 100% rename from packages/db/queries/sandboxes/get_sandbox_record.sql rename to packages/db/pkg/dashboard/sql_queries/sandboxes/get_sandbox_record.sql diff --git a/packages/db/pkg/testutils/queries/models.go b/packages/db/pkg/testutils/queries/models.go index 9caff681c2..7cf7acc09e 100644 --- a/packages/db/pkg/testutils/queries/models.go +++ b/packages/db/pkg/testutils/queries/models.go @@ -3,3 +3,233 @@ // sqlc v1.29.0 package queries + +import ( + "github.com/jackc/pgx/v5/pgtype" +) + +type AccessToken struct { + UserID pgtype.UUID + CreatedAt pgtype.Timestamptz + ID pgtype.UUID + // sensitive + AccessTokenHash string + Name string + AccessTokenPrefix string + AccessTokenLength int32 + AccessTokenMaskPrefix string + AccessTokenMaskSuffix string +} + +type ActiveTemplateBuild struct { + BuildID pgtype.UUID + TeamID pgtype.UUID + TemplateID string + Tags []string + CreatedAt pgtype.Timestamptz +} + +type Addon struct { + ID pgtype.UUID + TeamID pgtype.UUID + Name string + Description pgtype.Text + ExtraConcurrentSandboxes int64 + ExtraConcurrentTemplateBuilds int64 + ExtraMaxVcpu int64 + ExtraMaxRamMb int64 + ExtraDiskMb int64 + ValidFrom pgtype.Timestamptz + ValidTo pgtype.Timestamptz + AddedBy pgtype.UUID + IdempotencyKey pgtype.Text +} + +type AuthUser struct { + ID pgtype.UUID + Email string +} + +type BillingSandboxLog struct { + SandboxID string + EnvID string + Vcpu int64 + RamMb int64 + TotalDiskSizeMb int64 + StartedAt pgtype.Timestamptz + StoppedAt pgtype.Timestamptz + CreatedAt pgtype.Timestamptz + TeamID pgtype.UUID +} + +type Cluster struct { + ID pgtype.UUID + Endpoint string + EndpointTls bool + Token string + SandboxProxyDomain pgtype.Text +} + +type Env struct { + ID string + CreatedAt pgtype.Timestamptz + UpdatedAt pgtype.Timestamptz + Public bool + BuildCount int32 + // Number of times the env was spawned + SpawnCount int64 + // Timestamp of the last time the env was spawned + LastSpawnedAt pgtype.Timestamptz + TeamID pgtype.UUID + CreatedBy pgtype.UUID + ClusterID pgtype.UUID + Source string +} + +type EnvAlias struct { + Alias string + IsRenamable bool + EnvID string + Namespace pgtype.Text + ID pgtype.UUID +} + +type EnvBuild struct { + ID pgtype.UUID + CreatedAt pgtype.Timestamptz + UpdatedAt pgtype.Timestamptz + FinishedAt pgtype.Timestamptz + Status string + Dockerfile pgtype.Text + StartCmd pgtype.Text + Vcpu int64 + RamMb int64 + FreeDiskSizeMb int64 + TotalDiskSizeMb pgtype.Int8 + KernelVersion string + FirecrackerVersion string + EnvID pgtype.Text + EnvdVersion pgtype.Text + ReadyCmd pgtype.Text + ClusterNodeID pgtype.Text + Reason []byte + Version pgtype.Text + CpuArchitecture pgtype.Text + CpuFamily pgtype.Text + CpuModel pgtype.Text + CpuModelName pgtype.Text + CpuFlags []string + StatusGroup string + TeamID pgtype.UUID +} + +type EnvBuildAssignment struct { + ID pgtype.UUID + EnvID string + BuildID pgtype.UUID + Tag string + Source string + CreatedAt pgtype.Timestamptz +} + +type Snapshot struct { + CreatedAt pgtype.Timestamptz + EnvID string + SandboxID string + ID pgtype.UUID + Metadata []byte + BaseEnvID string + SandboxStartedAt pgtype.Timestamptz + EnvSecure bool + OriginNodeID string + AllowInternetAccess pgtype.Bool + AutoPause bool + TeamID pgtype.UUID + Config []byte +} + +type SnapshotTemplate struct { + EnvID string + SandboxID string + CreatedAt pgtype.Timestamptz + OriginNodeID pgtype.Text + BuildID pgtype.UUID +} + +type Team struct { + ID pgtype.UUID + CreatedAt pgtype.Timestamptz + IsBlocked bool + Name string + Tier string + Email string + IsBanned bool + BlockedReason pgtype.Text + ClusterID pgtype.UUID + SandboxSchedulingLabels []string + Slug string +} + +type TeamApiKey struct { + CreatedAt pgtype.Timestamptz + TeamID pgtype.UUID + UpdatedAt pgtype.Timestamptz + Name string + LastUsed pgtype.Timestamptz + CreatedBy pgtype.UUID + ID pgtype.UUID + // sensitive + ApiKeyHash string + ApiKeyPrefix string + ApiKeyLength int32 + ApiKeyMaskPrefix string + ApiKeyMaskSuffix string +} + +type TeamLimit struct { + ID pgtype.UUID + MaxLengthHours int64 + ConcurrentSandboxes int32 + ConcurrentTemplateBuilds int32 + MaxVcpu int32 + MaxRamMb int32 + DiskMb int32 +} + +type Tier struct { + ID string + Name string + DiskMb int64 + // The number of instances the team can run concurrently + ConcurrentInstances int64 + MaxLengthHours int64 + MaxVcpu int64 + MaxRamMb int64 + // The number of concurrent template builds the team can run + ConcurrentTemplateBuilds int64 +} + +type User struct { + CreatedAt pgtype.Timestamptz + UpdatedAt pgtype.Timestamptz + ID pgtype.UUID + Email string +} + +type UsersTeam struct { + ID int64 + UserID pgtype.UUID + TeamID pgtype.UUID + IsDefault bool + AddedBy pgtype.UUID + CreatedAt pgtype.Timestamp + UuidID pgtype.UUID +} + +type Volume struct { + ID pgtype.UUID + TeamID pgtype.UUID + Name string + VolumeType string + CreatedAt pgtype.Timestamptz +} diff --git a/packages/db/scripts/migrator.go b/packages/db/scripts/migrator.go index 72bcc1756d..0a74e7ee7f 100644 --- a/packages/db/scripts/migrator.go +++ b/packages/db/scripts/migrator.go @@ -8,6 +8,7 @@ import ( "os" "time" + "github.com/e2b-dev/infra/packages/shared/pkg/env" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgxpool" "github.com/jackc/pgx/v5/stdlib" @@ -18,8 +19,8 @@ import ( ) const ( - trackingTable = "_migrations" - migrationsDir = "./migrations" + defaultTrackingTable = "_migrations" + defaultMigrationsDir = "./migrations" authMigrationVersion = 20000101000000 statementTimeout = 3 * time.Hour @@ -28,8 +29,10 @@ const ( func main() { fmt.Printf("Starting migrations...\n") ctx := context.Background() + trackingTable := env.GetEnv("MIGRATIONS_TABLE", defaultTrackingTable) + migrationsDir := env.GetEnv("MIGRATIONS_DIR", defaultMigrationsDir) - dbString := os.Getenv("POSTGRES_CONNECTION_STRING") + dbString := env.GetEnv("POSTGRES_CONNECTION_STRING", "") if dbString == "" { log.Fatal("Database connection string is required. Set POSTGRES_CONNECTION_STRING env var.") } @@ -69,9 +72,9 @@ func main() { } fmt.Printf("Current DB version: %d\n", version) - if version < authMigrationVersion { + if trackingTable == defaultTrackingTable && version < authMigrationVersion { fmt.Println("Creating auth.users table...") - err = setupAuthSchema(ctx, db, version) + err = setupAuthSchema(ctx, db, version, trackingTable) if err != nil { log.Fatalf("failed to ensure auth.users table: %v", err) } @@ -107,7 +110,7 @@ func main() { fmt.Println("Migrations applied successfully.") } -func setupAuthSchema(ctx context.Context, db *sql.DB, version int64) error { +func setupAuthSchema(ctx context.Context, db *sql.DB, version int64, trackingTable string) error { rows, err := db.QueryContext(ctx, `SELECT EXISTS (SELECT 1 FROM information_schema.tables WHERE table_schema = 'auth' AND table_name = 'users')`) if err != nil { return fmt.Errorf("failed to query: %w", err) diff --git a/packages/db/sqlc.yaml b/packages/db/sqlc.yaml index fdb318004a..1f97f78867 100644 --- a/packages/db/sqlc.yaml +++ b/packages/db/sqlc.yaml @@ -105,8 +105,62 @@ sql: - db_type: "jsonb" go_type: "github.com/e2b-dev/infra/packages/db/pkg/types.JSONBStringMap" + - engine: "postgresql" + queries: "pkg/dashboard/sql_queries/**" + schema: + - "migrations" + - "pkg/dashboard/migrations" + - "schema" + gen: + go: + emit_pointers_for_null_types: true + package: "dashboardqueries" + out: "pkg/dashboard/queries/" + sql_package: "pgx/v5" + overrides: + - column: "public.env_builds.status" + go_type: "github.com/e2b-dev/infra/packages/db/pkg/types.BuildStatus" + - column: "public.env_builds.status_group" + go_type: "github.com/e2b-dev/infra/packages/db/pkg/types.BuildStatusGroup" + - column: "public.env_builds.reason" + go_type: "github.com/e2b-dev/infra/packages/db/pkg/types.BuildReason" + - db_type: "uuid" + go_type: + import: "github.com/google/uuid" + type: "UUID" + - db_type: "uuid" + nullable: true + go_type: + import: "github.com/google/uuid" + type: "UUID" + pointer: true + + - db_type: "pg_catalog.numeric" + go_type: "github.com/shopspring/decimal.Decimal" + - db_type: "pg_catalog.numeric" + nullable: true + go_type: "*github.com/shopspring/decimal.Decimal" + + - db_type: "timestamptz" + go_type: "time.Time" + - db_type: "timestamptz" + go_type: + import: "time" + type: "Time" + pointer: true + nullable: true + + - db_type: "jsonb" + go_type: "github.com/e2b-dev/infra/packages/db/pkg/types.JSONBStringMap" + nullable: true + - db_type: "jsonb" + go_type: "github.com/e2b-dev/infra/packages/db/pkg/types.JSONBStringMap" + - engine: "postgresql" queries: "pkg/testutils/*.sql" + schema: + - "migrations" + - "schema" gen: go: package: "queries" diff --git a/scripts/get-latest-migration.sh b/scripts/get-latest-migration.sh index 9498caf870..7886fdd219 100755 --- a/scripts/get-latest-migration.sh +++ b/scripts/get-latest-migration.sh @@ -3,5 +3,19 @@ set -euo pipefail cd "$(git rev-parse --show-toplevel)" -latest_version=$(git ls-tree --name-only HEAD -- packages/db/migrations/ | sed 's|.*/||' | sed 's/_.*//' | sort | tail -n 1) +scope="${1:-core}" +case "$scope" in + core) + migration_dir="packages/db/migrations" + ;; + dashboard) + migration_dir="packages/db/pkg/dashboard/migrations" + ;; + *) + echo "unsupported scope: $scope" >&2 + exit 1 + ;; +esac + +latest_version=$(git ls-tree --name-only HEAD -- "${migration_dir}/" | sed 's|.*/||' | sed 's/_.*//' | sort | tail -n 1) echo "$latest_version" \ No newline at end of file From ac0469ee6ab7c6517128f2f8ff48fc82cb90a7c7 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 19 Mar 2026 01:32:57 +0000 Subject: [PATCH 2/4] chore: auto-commit generated changes --- iac/provider-gcp/nomad/main.tf | 2 +- packages/db/scripts/migrator.go | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/iac/provider-gcp/nomad/main.tf b/iac/provider-gcp/nomad/main.tf index 28c2472abd..e317215069 100644 --- a/iac/provider-gcp/nomad/main.tf +++ b/iac/provider-gcp/nomad/main.tf @@ -128,7 +128,7 @@ module "dashboard_api" { update_stanza = var.dashboard_api_count > 1 environment = var.environment - image = data.google_artifact_registry_docker_image.dashboard_api_image[0].self_link + image = data.google_artifact_registry_docker_image.dashboard_api_image[0].self_link db_migrator_docker_image = data.google_artifact_registry_docker_image.db_migrator_image.self_link postgres_connection_string = data.google_secret_manager_secret_version.postgres_connection_string.secret_data diff --git a/packages/db/scripts/migrator.go b/packages/db/scripts/migrator.go index 0a74e7ee7f..59a76a08eb 100644 --- a/packages/db/scripts/migrator.go +++ b/packages/db/scripts/migrator.go @@ -8,7 +8,6 @@ import ( "os" "time" - "github.com/e2b-dev/infra/packages/shared/pkg/env" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgxpool" "github.com/jackc/pgx/v5/stdlib" @@ -16,6 +15,8 @@ import ( "github.com/pressly/goose/v3" "github.com/pressly/goose/v3/database" "github.com/pressly/goose/v3/lock" + + "github.com/e2b-dev/infra/packages/shared/pkg/env" ) const ( From b5e223a9bb5cabce76fa74fd664c5776be8844b9 Mon Sep 17 00:00:00 2001 From: ben-fornefeld Date: Wed, 18 Mar 2026 18:34:08 -0700 Subject: [PATCH 3/4] chore: revert unecessary sqlc.yaml changes --- packages/db/sqlc.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/packages/db/sqlc.yaml b/packages/db/sqlc.yaml index 1f97f78867..230d2196d7 100644 --- a/packages/db/sqlc.yaml +++ b/packages/db/sqlc.yaml @@ -158,9 +158,6 @@ sql: - engine: "postgresql" queries: "pkg/testutils/*.sql" - schema: - - "migrations" - - "schema" gen: go: package: "queries" From 6b32cb47d1b545b4e2f5cb9c35c7edad0fbf8e9c Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 19 Mar 2026 01:39:18 +0000 Subject: [PATCH 4/4] chore: auto-commit generated changes --- packages/db/pkg/testutils/queries/models.go | 230 -------------------- 1 file changed, 230 deletions(-) diff --git a/packages/db/pkg/testutils/queries/models.go b/packages/db/pkg/testutils/queries/models.go index 7cf7acc09e..9caff681c2 100644 --- a/packages/db/pkg/testutils/queries/models.go +++ b/packages/db/pkg/testutils/queries/models.go @@ -3,233 +3,3 @@ // sqlc v1.29.0 package queries - -import ( - "github.com/jackc/pgx/v5/pgtype" -) - -type AccessToken struct { - UserID pgtype.UUID - CreatedAt pgtype.Timestamptz - ID pgtype.UUID - // sensitive - AccessTokenHash string - Name string - AccessTokenPrefix string - AccessTokenLength int32 - AccessTokenMaskPrefix string - AccessTokenMaskSuffix string -} - -type ActiveTemplateBuild struct { - BuildID pgtype.UUID - TeamID pgtype.UUID - TemplateID string - Tags []string - CreatedAt pgtype.Timestamptz -} - -type Addon struct { - ID pgtype.UUID - TeamID pgtype.UUID - Name string - Description pgtype.Text - ExtraConcurrentSandboxes int64 - ExtraConcurrentTemplateBuilds int64 - ExtraMaxVcpu int64 - ExtraMaxRamMb int64 - ExtraDiskMb int64 - ValidFrom pgtype.Timestamptz - ValidTo pgtype.Timestamptz - AddedBy pgtype.UUID - IdempotencyKey pgtype.Text -} - -type AuthUser struct { - ID pgtype.UUID - Email string -} - -type BillingSandboxLog struct { - SandboxID string - EnvID string - Vcpu int64 - RamMb int64 - TotalDiskSizeMb int64 - StartedAt pgtype.Timestamptz - StoppedAt pgtype.Timestamptz - CreatedAt pgtype.Timestamptz - TeamID pgtype.UUID -} - -type Cluster struct { - ID pgtype.UUID - Endpoint string - EndpointTls bool - Token string - SandboxProxyDomain pgtype.Text -} - -type Env struct { - ID string - CreatedAt pgtype.Timestamptz - UpdatedAt pgtype.Timestamptz - Public bool - BuildCount int32 - // Number of times the env was spawned - SpawnCount int64 - // Timestamp of the last time the env was spawned - LastSpawnedAt pgtype.Timestamptz - TeamID pgtype.UUID - CreatedBy pgtype.UUID - ClusterID pgtype.UUID - Source string -} - -type EnvAlias struct { - Alias string - IsRenamable bool - EnvID string - Namespace pgtype.Text - ID pgtype.UUID -} - -type EnvBuild struct { - ID pgtype.UUID - CreatedAt pgtype.Timestamptz - UpdatedAt pgtype.Timestamptz - FinishedAt pgtype.Timestamptz - Status string - Dockerfile pgtype.Text - StartCmd pgtype.Text - Vcpu int64 - RamMb int64 - FreeDiskSizeMb int64 - TotalDiskSizeMb pgtype.Int8 - KernelVersion string - FirecrackerVersion string - EnvID pgtype.Text - EnvdVersion pgtype.Text - ReadyCmd pgtype.Text - ClusterNodeID pgtype.Text - Reason []byte - Version pgtype.Text - CpuArchitecture pgtype.Text - CpuFamily pgtype.Text - CpuModel pgtype.Text - CpuModelName pgtype.Text - CpuFlags []string - StatusGroup string - TeamID pgtype.UUID -} - -type EnvBuildAssignment struct { - ID pgtype.UUID - EnvID string - BuildID pgtype.UUID - Tag string - Source string - CreatedAt pgtype.Timestamptz -} - -type Snapshot struct { - CreatedAt pgtype.Timestamptz - EnvID string - SandboxID string - ID pgtype.UUID - Metadata []byte - BaseEnvID string - SandboxStartedAt pgtype.Timestamptz - EnvSecure bool - OriginNodeID string - AllowInternetAccess pgtype.Bool - AutoPause bool - TeamID pgtype.UUID - Config []byte -} - -type SnapshotTemplate struct { - EnvID string - SandboxID string - CreatedAt pgtype.Timestamptz - OriginNodeID pgtype.Text - BuildID pgtype.UUID -} - -type Team struct { - ID pgtype.UUID - CreatedAt pgtype.Timestamptz - IsBlocked bool - Name string - Tier string - Email string - IsBanned bool - BlockedReason pgtype.Text - ClusterID pgtype.UUID - SandboxSchedulingLabels []string - Slug string -} - -type TeamApiKey struct { - CreatedAt pgtype.Timestamptz - TeamID pgtype.UUID - UpdatedAt pgtype.Timestamptz - Name string - LastUsed pgtype.Timestamptz - CreatedBy pgtype.UUID - ID pgtype.UUID - // sensitive - ApiKeyHash string - ApiKeyPrefix string - ApiKeyLength int32 - ApiKeyMaskPrefix string - ApiKeyMaskSuffix string -} - -type TeamLimit struct { - ID pgtype.UUID - MaxLengthHours int64 - ConcurrentSandboxes int32 - ConcurrentTemplateBuilds int32 - MaxVcpu int32 - MaxRamMb int32 - DiskMb int32 -} - -type Tier struct { - ID string - Name string - DiskMb int64 - // The number of instances the team can run concurrently - ConcurrentInstances int64 - MaxLengthHours int64 - MaxVcpu int64 - MaxRamMb int64 - // The number of concurrent template builds the team can run - ConcurrentTemplateBuilds int64 -} - -type User struct { - CreatedAt pgtype.Timestamptz - UpdatedAt pgtype.Timestamptz - ID pgtype.UUID - Email string -} - -type UsersTeam struct { - ID int64 - UserID pgtype.UUID - TeamID pgtype.UUID - IsDefault bool - AddedBy pgtype.UUID - CreatedAt pgtype.Timestamp - UuidID pgtype.UUID -} - -type Volume struct { - ID pgtype.UUID - TeamID pgtype.UUID - Name string - VolumeType string - CreatedAt pgtype.Timestamptz -}