Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .github/actions/start-services/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ runs:
echo "SUPABASE_JWT_SECRETS=${SUPABASE_JWT_SECRET}" >> .env.test
set -x
make migrate
make -C packages/db migrate-dashboard
make -C tests/integration seed
shell: bash

Expand Down
48 changes: 28 additions & 20 deletions .github/workflows/out-of-order-migrations.yml
Original file line number Diff line number Diff line change
Expand Up @@ -15,24 +15,32 @@ jobs:

- name: Compare migrations
run: |
# Get all migration versions from main
git ls-tree -r origin/main --name-only | grep '^packages/db/migrations/' | grep -oE '[0-9]{14}' | sort -n > main_versions.txt

# Find the highest version number from main
HIGHEST_MAIN=$(tail -n1 main_versions.txt)
echo "Highest main migration version: $HIGHEST_MAIN"

# Find newly added migration files in this PR
NEW_FILES=$(git diff --name-status origin/main -- packages/db/migrations/ | grep '^A' | awk '{print $2}')

for file in $NEW_FILES; do
version=$(basename "$file" | grep -oE '^[0-9]{14}')
echo "Checking new migration version: $version"
if [ "$version" -le "$HIGHEST_MAIN" ]; then
echo "❌ Migration $file is out of order! ($version <= $HIGHEST_MAIN)"
exit 1
fi
done

check_stream() {
local dir="$1"
local stream="$2"
local versions_file="$3"

# Get all migration versions from main
git ls-tree -r origin/main --name-only | grep "^${dir}/" | grep -oE '[0-9]{14}' | sort -n > "$versions_file"

# Find the highest version number from main
HIGHEST_MAIN=$(tail -n1 "$versions_file")
echo "Highest main ${stream} migration version: $HIGHEST_MAIN"

# Find newly added migration files in this PR
NEW_FILES=$(git diff --name-status origin/main -- "${dir}/" | grep '^A' | awk '{print $2}')

for file in $NEW_FILES; do
version=$(basename "$file" | grep -oE '^[0-9]{14}')
echo "Checking new ${stream} migration version: $version"
if [ "$version" -le "$HIGHEST_MAIN" ]; then
echo "❌ Migration $file is out of order! ($version <= $HIGHEST_MAIN)"
exit 1
fi
done
}

check_stream "packages/db/migrations" "core" "core_main_versions.txt"
check_stream "packages/db/pkg/dashboard/migrations" "dashboard" "dashboard_main_versions.txt"

echo "✅ All new migrations are in correct order."

2 changes: 2 additions & 0 deletions iac/modules/job-api/jobs/api.hcl
Original file line number Diff line number Diff line change
Expand Up @@ -176,6 +176,8 @@ job "api" {

env {
POSTGRES_CONNECTION_STRING="${postgres_connection_string}"
MIGRATIONS_DIR="./migrations"
MIGRATIONS_TABLE="_migrations"
}

config {
Expand Down
24 changes: 24 additions & 0 deletions iac/modules/job-dashboard-api/jobs/dashboard-api.hcl
Original file line number Diff line number Diff line change
Expand Up @@ -90,5 +90,29 @@ job "dashboard-api" {
ports = ["api"]
}
}

task "db-migrator" {
driver = "docker"

env {
POSTGRES_CONNECTION_STRING = "${postgres_connection_string}"
MIGRATIONS_DIR = "./pkg/dashboard/migrations"
MIGRATIONS_TABLE = "_migrations_dashboard"
}

config {
image = "${db_migrator_docker_image}"
}

resources {
cpu = 250
memory = 128
}

lifecycle {
hook = "prestart"
sidecar = false
}
}
}
}
1 change: 1 addition & 0 deletions iac/modules/job-dashboard-api/main.tf
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ resource "nomad_job" "dashboard_api" {
auth_db_read_replica_connection_string = var.auth_db_read_replica_connection_string
clickhouse_connection_string = var.clickhouse_connection_string
supabase_jwt_secrets = var.supabase_jwt_secrets
db_migrator_docker_image = var.db_migrator_docker_image

subdomain = "dashboard-api"

Expand Down
4 changes: 4 additions & 0 deletions iac/modules/job-dashboard-api/variables.tf
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,10 @@ variable "image" {
type = string
}

variable "db_migrator_docker_image" {
type = string
}

variable "count_instances" {
type = number
}
Expand Down
3 changes: 2 additions & 1 deletion iac/provider-gcp/nomad/main.tf
Original file line number Diff line number Diff line change
Expand Up @@ -128,7 +128,8 @@ module "dashboard_api" {
update_stanza = var.dashboard_api_count > 1
environment = var.environment

image = data.google_artifact_registry_docker_image.dashboard_api_image[0].self_link
image = data.google_artifact_registry_docker_image.dashboard_api_image[0].self_link
db_migrator_docker_image = data.google_artifact_registry_docker_image.db_migrator_image.self_link

postgres_connection_string = data.google_secret_manager_secret_version.postgres_connection_string.secret_data
auth_db_connection_string = data.google_secret_manager_secret_version.postgres_connection_string.secret_data
Expand Down
2 changes: 1 addition & 1 deletion packages/api/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ PREFIX := $(strip $(subst ",,$(PREFIX)))
HOSTNAME := $(shell hostname 2> /dev/null || hostnamectl hostname 2> /dev/null)
$(if $(HOSTNAME),,$(error Failed to determine hostname: both 'hostname' and 'hostnamectl' failed))

expectedMigration := $(shell ./../../scripts/get-latest-migration.sh)
expectedMigration := $(shell ./../../scripts/get-latest-migration.sh core)

ifeq ($(PROVIDER),aws)
REGISTRY_PREFIX := $(AWS_ACCOUNT_ID).dkr.ecr.$(AWS_REGION).amazonaws.com/$(PREFIX)core
Expand Down
5 changes: 3 additions & 2 deletions packages/dashboard-api/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -43,8 +43,9 @@ COPY ./dashboard-api/Makefile ./dashboard-api/Makefile
WORKDIR /build/dashboard-api

ARG COMMIT_SHA
ARG EXPECTED_MIGRATION_TIMESTAMP
RUN --mount=type=cache,target=/root/.cache/go-build make build COMMIT_SHA=${COMMIT_SHA} EXPECTED_MIGRATION_TIMESTAMP=${EXPECTED_MIGRATION_TIMESTAMP}
ARG EXPECTED_CORE_MIGRATION_TIMESTAMP
ARG EXPECTED_DASHBOARD_MIGRATION_TIMESTAMP
RUN --mount=type=cache,target=/root/.cache/go-build make build COMMIT_SHA=${COMMIT_SHA} EXPECTED_CORE_MIGRATION_TIMESTAMP=${EXPECTED_CORE_MIGRATION_TIMESTAMP} EXPECTED_DASHBOARD_MIGRATION_TIMESTAMP=${EXPECTED_DASHBOARD_MIGRATION_TIMESTAMP}
RUN chmod +x /build/dashboard-api/bin/dashboard-api

FROM alpine:${ALPINE_VERSION}
Expand Down
13 changes: 8 additions & 5 deletions packages/dashboard-api/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,8 @@ ENV := $(shell cat ../../.last_used_env || echo "not-set")
-include ../../.env.${ENV}
PREFIX := $(strip $(subst ",,$(PREFIX)))

expectedMigration := $(shell ./../../scripts/get-latest-migration.sh)
expectedCoreMigration := $(shell ./../../scripts/get-latest-migration.sh core)
expectedDashboardMigration := $(shell ./../../scripts/get-latest-migration.sh dashboard)

ifeq ($(PROVIDER),aws)
IMAGE_REGISTRY := $(AWS_ACCOUNT_ID).dkr.ecr.$(AWS_REGION).amazonaws.com/$(PREFIX)core/dashboard-api
Expand All @@ -21,14 +22,16 @@ generate:
build:
# Allow for passing commit sha directly for docker builds
$(eval COMMIT_SHA ?= $(shell git rev-parse --short HEAD))
$(eval EXPECTED_MIGRATION_TIMESTAMP ?= $(expectedMigration))
CGO_ENABLED=0 go build -o bin/dashboard-api -ldflags "-X=main.commitSHA=$(COMMIT_SHA) -X=main.expectedMigrationTimestamp=$(EXPECTED_MIGRATION_TIMESTAMP)" .
$(eval EXPECTED_CORE_MIGRATION_TIMESTAMP ?= $(expectedCoreMigration))
$(eval EXPECTED_DASHBOARD_MIGRATION_TIMESTAMP ?= $(expectedDashboardMigration))
CGO_ENABLED=0 go build -o bin/dashboard-api -ldflags "-X=main.commitSHA=$(COMMIT_SHA) -X=main.expectedCoreMigrationTimestamp=$(EXPECTED_CORE_MIGRATION_TIMESTAMP) -X=main.expectedDashboardMigrationTimestamp=$(EXPECTED_DASHBOARD_MIGRATION_TIMESTAMP)" .

.PHONY: build-and-upload
build-and-upload:
$(eval COMMIT_SHA := $(shell git rev-parse --short HEAD))
$(eval EXPECTED_MIGRATION_TIMESTAMP := $(expectedMigration))
@ docker build --platform linux/amd64 --tag $(IMAGE_REGISTRY) --push --build-arg COMMIT_SHA="$(COMMIT_SHA)" --build-arg EXPECTED_MIGRATION_TIMESTAMP="$(EXPECTED_MIGRATION_TIMESTAMP)" -f ./Dockerfile ..
$(eval EXPECTED_CORE_MIGRATION_TIMESTAMP := $(expectedCoreMigration))
$(eval EXPECTED_DASHBOARD_MIGRATION_TIMESTAMP := $(expectedDashboardMigration))
@ docker build --platform linux/amd64 --tag $(IMAGE_REGISTRY) --push --build-arg COMMIT_SHA="$(COMMIT_SHA)" --build-arg EXPECTED_CORE_MIGRATION_TIMESTAMP="$(EXPECTED_CORE_MIGRATION_TIMESTAMP)" --build-arg EXPECTED_DASHBOARD_MIGRATION_TIMESTAMP="$(EXPECTED_DASHBOARD_MIGRATION_TIMESTAMP)" -f ./Dockerfile ..

.PHONY: run
run:
Expand Down
4 changes: 2 additions & 2 deletions packages/dashboard-api/internal/handlers/build.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ import (
"github.com/e2b-dev/infra/packages/auth/pkg/auth"
"github.com/e2b-dev/infra/packages/dashboard-api/internal/api"
dashboardutils "github.com/e2b-dev/infra/packages/dashboard-api/internal/utils"
"github.com/e2b-dev/infra/packages/db/queries"
dashboardqueries "github.com/e2b-dev/infra/packages/db/pkg/dashboard/queries"
"github.com/e2b-dev/infra/packages/shared/pkg/logger"
"github.com/e2b-dev/infra/packages/shared/pkg/telemetry"
)
Expand All @@ -22,7 +22,7 @@ func (s *APIStore) GetBuildsBuildId(c *gin.Context, buildId api.BuildId) {
teamID := auth.MustGetTeamInfo(c).Team.ID
telemetry.SetAttributes(ctx, telemetry.WithTeamID(teamID.String()), telemetry.WithBuildID(buildId.String()))

row, err := s.db.GetBuildInfoByTeamAndBuildID(ctx, queries.GetBuildInfoByTeamAndBuildIDParams{
row, err := s.db.GetBuildInfoByTeamAndBuildID(ctx, dashboardqueries.GetBuildInfoByTeamAndBuildIDParams{
Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I know these queries are called only in the dashboard-api, but lets keep them colocated to the database (source), rather than usage. -> they should stay in core

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

good point

TeamID: teamID,
BuildID: buildId,
})
Expand Down
18 changes: 9 additions & 9 deletions packages/dashboard-api/internal/handlers/builds_list.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,8 @@ import (
"github.com/e2b-dev/infra/packages/auth/pkg/auth"
"github.com/e2b-dev/infra/packages/dashboard-api/internal/api"
dashboardutils "github.com/e2b-dev/infra/packages/dashboard-api/internal/utils"
dashboardqueries "github.com/e2b-dev/infra/packages/db/pkg/dashboard/queries"
dbtypes "github.com/e2b-dev/infra/packages/db/pkg/types"
"github.com/e2b-dev/infra/packages/db/queries"
"github.com/e2b-dev/infra/packages/shared/pkg/logger"
"github.com/e2b-dev/infra/packages/shared/pkg/telemetry"
)
Expand Down Expand Up @@ -109,7 +109,7 @@ func (s *APIStore) listBuildRows(
statuses := buildStatusGroupsToStrings(statusGroups)

if buildIDOrTemplate == nil || strings.TrimSpace(*buildIDOrTemplate) == "" {
rows, err := s.db.GetTeamBuildsPage(ctx, queries.GetTeamBuildsPageParams{
rows, err := s.db.GetTeamBuildsPage(ctx, dashboardqueries.GetTeamBuildsPageParams{
TeamID: teamID,
CursorCreatedAt: cursorTime,
CursorID: cursorID,
Expand All @@ -126,7 +126,7 @@ func (s *APIStore) listBuildRows(
filter := strings.TrimSpace(*buildIDOrTemplate)
filterUUID, parseErr := uuid.Parse(filter)
if parseErr == nil {
byBuildIDRows, byBuildIDErr := s.db.GetTeamBuildsPageByBuildID(ctx, queries.GetTeamBuildsPageByBuildIDParams{
byBuildIDRows, byBuildIDErr := s.db.GetTeamBuildsPageByBuildID(ctx, dashboardqueries.GetTeamBuildsPageByBuildIDParams{
TeamID: teamID,
BuildID: filterUUID,
CursorCreatedAt: cursorTime,
Expand All @@ -144,7 +144,7 @@ func (s *APIStore) listBuildRows(

// templateIDs are not UUIDs
if parseErr != nil {
byTemplateIDRows, byTemplateIDErr := s.db.GetTeamBuildsPageByTemplateID(ctx, queries.GetTeamBuildsPageByTemplateIDParams{
byTemplateIDRows, byTemplateIDErr := s.db.GetTeamBuildsPageByTemplateID(ctx, dashboardqueries.GetTeamBuildsPageByTemplateIDParams{
TemplateID: filter,
TeamID: teamID,
CursorCreatedAt: cursorTime,
Expand All @@ -160,7 +160,7 @@ func (s *APIStore) listBuildRows(
}
}

byTemplateAliasRows, byTemplateAliasErr := s.db.GetTeamBuildsPageByTemplateAlias(ctx, queries.GetTeamBuildsPageByTemplateAliasParams{
byTemplateAliasRows, byTemplateAliasErr := s.db.GetTeamBuildsPageByTemplateAlias(ctx, dashboardqueries.GetTeamBuildsPageByTemplateAliasParams{
TemplateAlias: filter,
TeamID: teamID,
CursorCreatedAt: cursorTime,
Expand Down Expand Up @@ -233,7 +233,7 @@ func parseCursorTime(value string) (time.Time, error) {
return time.Parse(time.RFC3339, value)
}

func mapBuildRows(rows []queries.GetTeamBuildsPageRow) []listBuildRow {
func mapBuildRows(rows []dashboardqueries.GetTeamBuildsPageRow) []listBuildRow {
out := make([]listBuildRow, 0, len(rows))
for _, row := range rows {
out = append(out, listBuildRow{
Expand All @@ -250,7 +250,7 @@ func mapBuildRows(rows []queries.GetTeamBuildsPageRow) []listBuildRow {
return out
}

func mapBuildRowsByBuildID(rows []queries.GetTeamBuildsPageByBuildIDRow) []listBuildRow {
func mapBuildRowsByBuildID(rows []dashboardqueries.GetTeamBuildsPageByBuildIDRow) []listBuildRow {
out := make([]listBuildRow, 0, len(rows))
for _, row := range rows {
out = append(out, listBuildRow{
Expand All @@ -267,7 +267,7 @@ func mapBuildRowsByBuildID(rows []queries.GetTeamBuildsPageByBuildIDRow) []listB
return out
}

func mapBuildRowsByTemplateID(rows []queries.GetTeamBuildsPageByTemplateIDRow) []listBuildRow {
func mapBuildRowsByTemplateID(rows []dashboardqueries.GetTeamBuildsPageByTemplateIDRow) []listBuildRow {
out := make([]listBuildRow, 0, len(rows))
for _, row := range rows {
out = append(out, listBuildRow{
Expand All @@ -284,7 +284,7 @@ func mapBuildRowsByTemplateID(rows []queries.GetTeamBuildsPageByTemplateIDRow) [
return out
}

func mapBuildRowsByTemplateAlias(rows []queries.GetTeamBuildsPageByTemplateAliasRow) []listBuildRow {
func mapBuildRowsByTemplateAlias(rows []dashboardqueries.GetTeamBuildsPageByTemplateAliasRow) []listBuildRow {
out := make([]listBuildRow, 0, len(rows))
for _, row := range rows {
out = append(out, listBuildRow{
Expand Down
4 changes: 2 additions & 2 deletions packages/dashboard-api/internal/handlers/builds_statuses.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import (
"github.com/e2b-dev/infra/packages/auth/pkg/auth"
"github.com/e2b-dev/infra/packages/dashboard-api/internal/api"
dashboardutils "github.com/e2b-dev/infra/packages/dashboard-api/internal/utils"
"github.com/e2b-dev/infra/packages/db/queries"
dashboardqueries "github.com/e2b-dev/infra/packages/db/pkg/dashboard/queries"
"github.com/e2b-dev/infra/packages/shared/pkg/logger"
"github.com/e2b-dev/infra/packages/shared/pkg/telemetry"
)
Expand All @@ -32,7 +32,7 @@ func (s *APIStore) GetBuildsStatuses(c *gin.Context, params api.GetBuildsStatuse
return
}

p := queries.GetBuildsStatusesByTeamParams{
p := dashboardqueries.GetBuildsStatusesByTeamParams{
TeamID: teamID,
BuildIds: params.BuildIds,
}
Expand Down
4 changes: 2 additions & 2 deletions packages/dashboard-api/internal/handlers/sandbox_record.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ import (

"github.com/e2b-dev/infra/packages/auth/pkg/auth"
"github.com/e2b-dev/infra/packages/dashboard-api/internal/api"
"github.com/e2b-dev/infra/packages/db/queries"
dashboardqueries "github.com/e2b-dev/infra/packages/db/pkg/dashboard/queries"
"github.com/e2b-dev/infra/packages/shared/pkg/logger"
"github.com/e2b-dev/infra/packages/shared/pkg/telemetry"
)
Expand All @@ -30,7 +30,7 @@ func (s *APIStore) GetSandboxesSandboxIDRecord(c *gin.Context, sandboxID api.San
teamID := auth.MustGetTeamInfo(c).Team.ID
telemetry.SetAttributes(ctx, telemetry.WithTeamID(teamID.String()), telemetry.WithSandboxID(sandboxID))

row, err := s.db.GetSandboxRecordByTeamAndSandboxID(ctx, queries.GetSandboxRecordByTeamAndSandboxIDParams{
row, err := s.db.GetSandboxRecordByTeamAndSandboxID(ctx, dashboardqueries.GetSandboxRecordByTeamAndSandboxIDParams{
TeamID: teamID,
SandboxID: sandboxID,
CreatedAfter: time.Now().UTC().Add(-sandboxRecordRetention),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,9 @@ import (
"github.com/e2b-dev/infra/packages/auth/pkg/auth"
authtypes "github.com/e2b-dev/infra/packages/auth/pkg/types"
"github.com/e2b-dev/infra/packages/dashboard-api/internal/api"
sqlcdb "github.com/e2b-dev/infra/packages/db/client"
authqueries "github.com/e2b-dev/infra/packages/db/pkg/auth/queries"
"github.com/e2b-dev/infra/packages/db/queries"
dashboarddb "github.com/e2b-dev/infra/packages/db/pkg/dashboard"
dashboardqueries "github.com/e2b-dev/infra/packages/db/pkg/dashboard/queries"
)

type noRowsDB struct{}
Expand Down Expand Up @@ -56,8 +56,8 @@ func TestGetSandboxesSandboxIDRecordReturns404WhenRecordRetentionNotMet(t *testi
})

store := &APIStore{
db: &sqlcdb.Client{
Queries: queries.New(noRowsDB{}),
db: &dashboarddb.Client{
Queries: dashboardqueries.New(noRowsDB{}),
},
}

Expand Down
6 changes: 3 additions & 3 deletions packages/dashboard-api/internal/handlers/store.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,22 +12,22 @@ import (
clickhouse "github.com/e2b-dev/infra/packages/clickhouse/pkg"
"github.com/e2b-dev/infra/packages/dashboard-api/internal/api"
"github.com/e2b-dev/infra/packages/dashboard-api/internal/cfg"
sqlcdb "github.com/e2b-dev/infra/packages/db/client"
authdb "github.com/e2b-dev/infra/packages/db/pkg/auth"
dashboarddb "github.com/e2b-dev/infra/packages/db/pkg/dashboard"
"github.com/e2b-dev/infra/packages/shared/pkg/apierrors"
)

var _ api.ServerInterface = (*APIStore)(nil)

type APIStore struct {
config cfg.Config
db *sqlcdb.Client
db *dashboarddb.Client
authDB *authdb.Client
clickhouse clickhouse.Clickhouse
authService *sharedauth.AuthService[*types.Team]
}

func NewAPIStore(config cfg.Config, db *sqlcdb.Client, authDB *authdb.Client, ch clickhouse.Clickhouse, authService *sharedauth.AuthService[*types.Team]) *APIStore {
func NewAPIStore(config cfg.Config, db *dashboarddb.Client, authDB *authdb.Client, ch clickhouse.Clickhouse, authService *sharedauth.AuthService[*types.Team]) *APIStore {
return &APIStore{
config: config,
db: db,
Expand Down
Loading
Loading