feat(versioning): added api deployment versioning, change detection in workflow, added ci job to run migrations on merge to main (#220)

* added initial code for versioning

* ran migrations & improvements to versioning

* change sensitivity of state diff checker

* debounce the change detection function

* cleaned up file structure

* added ci job to run migrations when merging to main

* ran migrations

* cleanup unused files, remove unused dependencies

Author: waleedlatif1

.github/workflows/ci.yml

@@ -26,6 +26,12 @@ jobs:
         working-directory: ./sim
         run: npm ci
 
+      - name: Fix Rollup module issue
+        working-directory: ./sim
+        run: |
+          rm -rf node_modules package-lock.json
+          npm install
+        
       - name: Run tests with coverage
         working-directory: ./sim
         env:
@@ -41,4 +47,30 @@ jobs:
         with:
           directory: ./sim/coverage
           fail_ci_if_error: false
-          verbose: true 
+          verbose: true 
+
+  migrations:
+    name: Apply Database Migrations
+    runs-on: ubuntu-latest
+    if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+    needs: test
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: '20'
+          cache: 'npm'
+          cache-dependency-path: './sim/package-lock.json'
+          
+      - name: Install dependencies
+        working-directory: ./sim
+        run: npm ci
+        
+      - name: Apply migrations
+        working-directory: ./sim
+        env:
+          POSTGRES_URL: ${{ secrets.DATABASE_URL }}
+        run: npx drizzle-kit push 

sim/app/api/workflows/[id]/deploy/route.ts

@@ -32,6 +32,8 @@ export async function GET(request: NextRequest, { params }: { params: Promise<{
         isDeployed: workflow.isDeployed,
         deployedAt: workflow.deployedAt,
         userId: workflow.userId,
+        state: workflow.state,
+        deployedState: workflow.deployedState,
       })
       .from(workflow)
       .where(eq(workflow.id, id))
@@ -51,6 +53,7 @@ export async function GET(request: NextRequest, { params }: { params: Promise<{
         isDeployed: false,
         deployedAt: null,
         apiKey: null,
+        needsRedeployment: false,
       })
     }
 
@@ -63,11 +66,22 @@ export async function GET(request: NextRequest, { params }: { params: Promise<{
       .where(eq(apiKey.userId, workflowData.userId))
       .limit(1)
 
+    // Check if the workflow has meaningful changes that would require redeployment
+    let needsRedeployment = false
+    if (workflowData.deployedState) {
+      const { hasWorkflowChanged } = await import('@/lib/workflows/utils')
+      needsRedeployment = hasWorkflowChanged(
+        workflowData.state as any,
+        workflowData.deployedState as any
+      )
+    }
+
     logger.info(`[${requestId}] Successfully retrieved deployment info: ${id}`)
     return createSuccessResponse({
       apiKey: userApiKey.length > 0 ? userApiKey[0].key : null,
       isDeployed: workflowData.isDeployed,
       deployedAt: workflowData.deployedAt,
+      needsRedeployment,
     })
   } catch (error: any) {
     logger.error(`[${requestId}] Error fetching deployment info: ${id}`, error)
@@ -88,10 +102,11 @@ export async function POST(request: NextRequest, { params }: { params: Promise<{
       return createErrorResponse(validation.error.message, validation.error.status)
     }
 
-    // Get the workflow to find the user
+    // Get the workflow to find the user and current state
     const workflowData = await db
       .select({
         userId: workflow.userId,
+        state: workflow.state,
       })
       .from(workflow)
       .where(eq(workflow.id, id))
@@ -103,6 +118,7 @@ export async function POST(request: NextRequest, { params }: { params: Promise<{
     }
 
     const userId = workflowData[0].userId
+    const currentState = workflowData[0].state
     const deployedAt = new Date()
 
     // Check if the user already has an API key
@@ -132,12 +148,13 @@ export async function POST(request: NextRequest, { params }: { params: Promise<{
       userKey = userApiKey[0].key
     }
 
-    // Update the workflow deployment status
+    // Update the workflow deployment status and save current state as deployed state
     await db
       .update(workflow)
       .set({
         isDeployed: true,
         deployedAt,
+        deployedState: currentState,
       })
       .where(eq(workflow.id, id))
 
@@ -165,12 +182,13 @@ export async function DELETE(
       return createErrorResponse(validation.error.message, validation.error.status)
     }
 
-    // Update the workflow to remove deployment status
+    // Update the workflow to remove deployment status and deployed state
     await db
       .update(workflow)
       .set({
         isDeployed: false,
         deployedAt: null,
+        deployedState: null,
       })
       .where(eq(workflow.id, id))
 

sim/app/api/workflows/[id]/execute/route.ts

@@ -61,8 +61,18 @@ async function executeWorkflow(workflow: any, requestId: string, input?: any) {
     runningExecutions.add(workflowId)
     logger.info(`[${requestId}] Starting workflow execution: ${workflowId}`)
 
-    // Get the workflow state
-    const state = workflow.state as WorkflowState
+    // Use the deployed state if available, otherwise fall back to current state
+    const workflowState = workflow.deployedState || workflow.state
+
+    if (!workflow.deployedState) {
+      logger.warn(
+        `[${requestId}] No deployed state found for workflow: ${workflowId}, using current state`
+      )
+    } else {
+      logger.info(`[${requestId}] Using deployed state for workflow execution: ${workflowId}`)
+    }
+
+    const state = workflowState as WorkflowState
     const { blocks, edges, loops } = state
 
     // Use the same execution flow as in scheduled executions

sim/app/api/workflows/[id]/status/route.ts

@@ -1,5 +1,6 @@
 import { NextRequest } from 'next/server'
 import { createLogger } from '@/lib/logs/console-logger'
+import { hasWorkflowChanged } from '@/lib/workflows/utils'
 import { validateWorkflowAccess } from '../../middleware'
 import { createErrorResponse, createSuccessResponse } from '../../utils'
 
@@ -17,15 +18,26 @@ export async function GET(request: NextRequest, { params }: { params: Promise<{
       return createErrorResponse(validation.error.message, validation.error.status)
     }
 
+    // Check if the workflow has meaningful changes that would require redeployment
+    let needsRedeployment = false
+    if (validation.workflow.isDeployed && validation.workflow.deployedState) {
+      needsRedeployment = hasWorkflowChanged(
+        validation.workflow.state as any,
+        validation.workflow.deployedState as any
+      )
+    }
+
     logger.info(`[${requestId}] Retrieved status for workflow: ${id}`, {
       isDeployed: validation.workflow.isDeployed,
       isPublished: validation.workflow.isPublished,
+      needsRedeployment,
     })
 
     return createSuccessResponse({
       isDeployed: validation.workflow.isDeployed,
       deployedAt: validation.workflow.deployedAt,
       isPublished: validation.workflow.isPublished,
+      needsRedeployment,
     })
   } catch (error) {
     logger.error(`[${requestId}] Error getting status for workflow: ${(await params).id}`, error)