e2e: isolate external DB sync cleanup per suite (#1148)

Possible CI flake fix: track external DB sync cleanup per test suite to
avoid cross-suite config resets.\n\n- Lint: pass\n- Typecheck: pass

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->

## Summary by CodeRabbit

* **New Features**
* Added external database synchronization, enabling data replication
from the internal database to external PostgreSQL databases with
incremental batching and sequence tracking.
  * Introduced background cron job execution system for scheduled tasks.

* **Tests**
* Added comprehensive test suites for external database synchronization
scenarios including basic operations, advanced configurations,
high-volume data transfers, and race condition handling.

* **Chores**
* Enhanced CI/CD workflows to support external database sync operations.
* Updated Docker configuration and environment variables for improved
testing infrastructure.

* **Bug Fixes**
* Increased test timeout threshold for session expiration tests to
accommodate slower executions.

<!-- end of auto-generated comment: release notes by coderabbit.ai -->

Author: BilalG1

apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-advanced.test.ts

@@ -9,7 +9,7 @@ import {
   POSTGRES_USER,
   TEST_TIMEOUT,
   TestDbManager,
-  createProjectWithExternalDb,
+  createProjectWithExternalDb as createProjectWithExternalDbRaw,
   verifyNotInExternalDb,
   waitForCondition,
   waitForSyncedData,
@@ -21,6 +21,16 @@ const COMPLEX_SEQUENCE_TIMEOUT = TEST_TIMEOUT * 2 + 30_000;
 
 describe.sequential('External DB Sync - Advanced Tests', () => {
   let dbManager: TestDbManager;
+  const createProjectWithExternalDb = (
+    externalDatabases: any,
+    projectOptions?: { display_name?: string, description?: string }
+  ) => {
+    return createProjectWithExternalDbRaw(
+      externalDatabases,
+      projectOptions,
+      { projectTracker: dbManager.createdProjects }
+    );
+  };
 
   beforeAll(async () => {
     dbManager = new TestDbManager();

apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-basics.test.ts

@@ -4,7 +4,7 @@ import { User, niceBackendFetch } from '../../../backend-helpers';
 import {
   TEST_TIMEOUT,
   TestDbManager,
-  createProjectWithExternalDb,
+  createProjectWithExternalDb as createProjectWithExternalDbRaw,
   verifyInExternalDb,
   verifyNotInExternalDb,
   waitForCondition,
@@ -16,6 +16,16 @@ import {
 // Run tests sequentially to avoid concurrency issues with shared backend state
 describe.sequential('External DB Sync - Basic Tests', () => {
   let dbManager: TestDbManager;
+  const createProjectWithExternalDb = (
+    externalDatabases: any,
+    projectOptions?: { display_name?: string, description?: string }
+  ) => {
+    return createProjectWithExternalDbRaw(
+      externalDatabases,
+      projectOptions,
+      { projectTracker: dbManager.createdProjects }
+    );
+  };
 
   beforeAll(async () => {
     dbManager = new TestDbManager();

apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-high-volume.test.ts

@@ -8,14 +8,24 @@ import {
   POSTGRES_PASSWORD,
   POSTGRES_USER,
   TestDbManager,
-  createProjectWithExternalDb,
+  createProjectWithExternalDb as createProjectWithExternalDbRaw,
   waitForCondition,
   waitForTable,
 } from './external-db-sync-utils';
 
 // Run tests sequentially to avoid concurrency issues with shared backend state
 describe.sequential('External DB Sync - High Volume Tests', () => {
   let dbManager: TestDbManager;
+  const createProjectWithExternalDb = (
+    externalDatabases: any,
+    projectOptions?: { display_name?: string, description?: string }
+  ) => {
+    return createProjectWithExternalDbRaw(
+      externalDatabases,
+      projectOptions,
+      { projectTracker: dbManager.createdProjects }
+    );
+  };
 
   beforeAll(async () => {
     dbManager = new TestDbManager();

apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-race.test.ts

@@ -9,7 +9,7 @@ import {
   POSTGRES_USER,
   TEST_TIMEOUT,
   TestDbManager,
-  createProjectWithExternalDb,
+  createProjectWithExternalDb as createProjectWithExternalDbRaw,
   forceExternalDbSync,
   waitForCondition,
   waitForSyncedDeletion,
@@ -20,6 +20,16 @@ const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms));
 
 describe.sequential('External DB Sync - Race Condition Tests', () => {
   let dbManager: TestDbManager;
+  const createProjectWithExternalDb = (
+    externalDatabases: any,
+    projectOptions?: { display_name?: string, description?: string }
+  ) => {
+    return createProjectWithExternalDbRaw(
+      externalDatabases,
+      projectOptions,
+      { projectTracker: dbManager.createdProjects }
+    );
+  };
 
   beforeAll(async () => {
     dbManager = new TestDbManager();

apps/e2e/tests/backend/endpoints/api/v1/external-db-sync-utils.ts

@@ -34,11 +34,10 @@ const CLIENT_CONFIG: Partial<ClientConfig> = {
 };
 
 // Track all projects created with external DB configs for cleanup
-type ProjectContext = {
+export type ProjectContext = {
   projectId: string,
   superSecretAdminKey: string,
 };
-const createdProjects: ProjectContext[] = [];
 
 /**
  * Helper class to manage external test databases
@@ -47,6 +46,7 @@ export class TestDbManager {
   private setupClient: Client | null = null;
   private databases: Map<string, Client> = new Map();
   private databaseNames: Set<string> = new Set();
+  public readonly createdProjects: ProjectContext[] = [];
 
   async init() {
     this.setupClient = new Client({
@@ -81,7 +81,8 @@ export class TestDbManager {
 
   async cleanup() {
     // First, clean up all project configs to stop the sync cron from trying to connect
-    await cleanupAllProjectConfigs();
+    await cleanupProjectConfigs(this.createdProjects);
+    this.createdProjects.length = 0;
 
     // Close all tracked database clients
     const closePromises = Array.from(this.databases.values()).map(async (client) => {
@@ -321,7 +322,11 @@ export async function countUsersInExternalDb(client: Client): Promise<number> {
  * Helper to create a project and update its config with external DB settings.
  * Tracks the project for cleanup later.
  */
-export async function createProjectWithExternalDb(externalDatabases: any, projectOptions?: { display_name?: string, description?: string }) {
+export async function createProjectWithExternalDb(
+  externalDatabases: any,
+  projectOptions?: { display_name?: string, description?: string },
+  options?: { projectTracker?: ProjectContext[] }
+) {
   const project = await Project.createAndSwitch(projectOptions);
   const { projectKeys } = await InternalApiKey.createAndSetProjectKeys(project.adminAccessToken);
   if (!projectKeys.superSecretAdminKey) {
@@ -332,10 +337,12 @@ export async function createProjectWithExternalDb(externalDatabases: any, projec
   });
 
   // Track this project for cleanup
-  createdProjects.push({
-    projectId: project.projectId,
-    superSecretAdminKey: projectKeys.superSecretAdminKey,
-  });
+  if (options?.projectTracker) {
+    options.projectTracker.push({
+      projectId: project.projectId,
+      superSecretAdminKey: projectKeys.superSecretAdminKey,
+    });
+  }
 
   return project;
 }
@@ -356,8 +363,8 @@ export async function cleanupProjectExternalDb() {
  * Note: This function makes direct HTTP calls instead of using backendContext
  * because it runs in afterAll, which is outside the test context.
  */
-export async function cleanupAllProjectConfigs() {
-  for (const project of createdProjects) {
+export async function cleanupProjectConfigs(projects: ProjectContext[]) {
+  for (const project of projects) {
     try {
       // Make direct HTTP call to clear the external DB config
       await niceFetch(new URL('/api/latest/internal/config/override', STACK_BACKEND_BASE_URL), {
@@ -377,7 +384,4 @@ export async function cleanupAllProjectConfigs() {
       console.warn(`Failed to cleanup project ${project.projectId}:`, err);
     }
   }
-
-  // Clear the tracked projects
-  createdProjects.length = 0;
 }