feat(knowledge): add Live sync option to KB connectors + fix embedding billing

apps/sim/app/api/knowledge/[id]/connectors/[connectorId]/route.ts

@@ -13,6 +13,7 @@ import { z } from 'zod'
 import { decryptApiKey } from '@/lib/api-key/crypto'
 import { AuditAction, AuditResourceType, recordAudit } from '@/lib/audit/log'
 import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid'
+import { hasLiveSyncAccess } from '@/lib/billing/core/subscription'
 import { generateRequestId } from '@/lib/core/utils/request'
 import { deleteDocumentStorageFiles } from '@/lib/knowledge/documents/service'
 import { cleanupUnusedTagDefinitions } from '@/lib/knowledge/tags/service'
@@ -116,6 +117,20 @@ export async function PATCH(request: NextRequest, { params }: RouteParams) {
       )
     }
 
+    if (
+      parsed.data.syncIntervalMinutes !== undefined &&
+      parsed.data.syncIntervalMinutes > 0 &&
+      parsed.data.syncIntervalMinutes < 60
+    ) {
+      const canUseLiveSync = await hasLiveSyncAccess(auth.userId)
+      if (!canUseLiveSync) {
+        return NextResponse.json(
+          { error: 'Live sync requires a Max or Enterprise plan' },
+          { status: 403 }
+        )
+      }
+    }
+
     if (parsed.data.sourceConfig !== undefined) {
       const existingRows = await db
         .select()

apps/sim/app/api/knowledge/[id]/connectors/route.ts

@@ -7,6 +7,7 @@ import { z } from 'zod'
 import { encryptApiKey } from '@/lib/api-key/crypto'
 import { AuditAction, AuditResourceType, recordAudit } from '@/lib/audit/log'
 import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid'
+import { hasLiveSyncAccess } from '@/lib/billing/core/subscription'
 import { generateRequestId } from '@/lib/core/utils/request'
 import { dispatchSync } from '@/lib/knowledge/connectors/sync-engine'
 import { allocateTagSlots } from '@/lib/knowledge/constants'
@@ -97,6 +98,16 @@ export async function POST(request: NextRequest, { params }: { params: Promise<{
 
     const { connectorType, credentialId, apiKey, sourceConfig, syncIntervalMinutes } = parsed.data
 
+    if (syncIntervalMinutes > 0 && syncIntervalMinutes < 60) {
+      const canUseLiveSync = await hasLiveSyncAccess(auth.userId)
+      if (!canUseLiveSync) {
+        return NextResponse.json(
+          { error: 'Live sync requires a Max or Enterprise plan' },
+          { status: 403 }
+        )
+      }
+    }
+
     const connectorConfig = CONNECTOR_REGISTRY[connectorType]
     if (!connectorConfig) {
       return NextResponse.json(

apps/sim/app/api/knowledge/search/utils.test.ts

@@ -5,6 +5,7 @@
  * @vitest-environment node
  */
 import { createEnvMock, databaseMock, loggerMock } from '@sim/testing'
+import { mockNextFetchResponse } from '@sim/testing/mocks'
 import { beforeEach, describe, expect, it, vi } from 'vitest'
 
 vi.mock('drizzle-orm')
@@ -14,16 +15,6 @@ vi.mock('@/lib/knowledge/documents/utils', () => ({
   retryWithExponentialBackoff: (fn: any) => fn(),
 }))
 
-vi.stubGlobal(
-  'fetch',
-  vi.fn().mockResolvedValue({
-    ok: true,
-    json: async () => ({
-      data: [{ embedding: [0.1, 0.2, 0.3] }],
-    }),
-  })
-)
-
 vi.mock('@/lib/core/config/env', () => createEnvMock())
 
 import {
@@ -178,17 +169,16 @@ describe('Knowledge Search Utils', () => {
         OPENAI_API_KEY: 'test-openai-key',
       })
 
-      const fetchSpy = vi.mocked(fetch)
-      fetchSpy.mockResolvedValueOnce({
-        ok: true,
-        json: async () => ({
+      mockNextFetchResponse({
+        json: {
           data: [{ embedding: [0.1, 0.2, 0.3] }],
-        }),
-      } as any)
+          usage: { prompt_tokens: 1, total_tokens: 1 },
+        },
+      })
 
       const result = await generateSearchEmbedding('test query')
 
-      expect(fetchSpy).toHaveBeenCalledWith(
+      expect(vi.mocked(fetch)).toHaveBeenCalledWith(
         'https://test.openai.azure.com/openai/deployments/text-embedding-ada-002/embeddings?api-version=2024-12-01-preview',
         expect.objectContaining({
           headers: expect.objectContaining({
@@ -209,17 +199,16 @@ describe('Knowledge Search Utils', () => {
         OPENAI_API_KEY: 'test-openai-key',
       })
 
-      const fetchSpy = vi.mocked(fetch)
-      fetchSpy.mockResolvedValueOnce({
-        ok: true,
-        json: async () => ({
+      mockNextFetchResponse({
+        json: {
           data: [{ embedding: [0.1, 0.2, 0.3] }],
-        }),
-      } as any)
+          usage: { prompt_tokens: 1, total_tokens: 1 },
+        },
+      })
 
       const result = await generateSearchEmbedding('test query')
 
-      expect(fetchSpy).toHaveBeenCalledWith(
+      expect(vi.mocked(fetch)).toHaveBeenCalledWith(
         'https://api.openai.com/v1/embeddings',
         expect.objectContaining({
           headers: expect.objectContaining({
@@ -243,17 +232,16 @@ describe('Knowledge Search Utils', () => {
         OPENAI_API_KEY: 'test-openai-key',
       })
 
-      const fetchSpy = vi.mocked(fetch)
-      fetchSpy.mockResolvedValueOnce({
-        ok: true,
-        json: async () => ({
+      mockNextFetchResponse({
+        json: {
           data: [{ embedding: [0.1, 0.2, 0.3] }],
-        }),
-      } as any)
+          usage: { prompt_tokens: 1, total_tokens: 1 },
+        },
+      })
 
       await generateSearchEmbedding('test query')
 
-      expect(fetchSpy).toHaveBeenCalledWith(
+      expect(vi.mocked(fetch)).toHaveBeenCalledWith(
         expect.stringContaining('api-version='),
         expect.any(Object)
       )
@@ -273,17 +261,16 @@ describe('Knowledge Search Utils', () => {
         OPENAI_API_KEY: 'test-openai-key',
       })
 
-      const fetchSpy = vi.mocked(fetch)
-      fetchSpy.mockResolvedValueOnce({
-        ok: true,
-        json: async () => ({
+      mockNextFetchResponse({
+        json: {
           data: [{ embedding: [0.1, 0.2, 0.3] }],
-        }),
-      } as any)
+          usage: { prompt_tokens: 1, total_tokens: 1 },
+        },
+      })
 
       await generateSearchEmbedding('test query', 'text-embedding-3-small')
 
-      expect(fetchSpy).toHaveBeenCalledWith(
+      expect(vi.mocked(fetch)).toHaveBeenCalledWith(
         'https://test.openai.azure.com/openai/deployments/custom-embedding-model/embeddings?api-version=2024-12-01-preview',
         expect.any(Object)
       )
@@ -311,13 +298,12 @@ describe('Knowledge Search Utils', () => {
         KB_OPENAI_MODEL_NAME: 'text-embedding-ada-002',
       })
 
-      const fetchSpy = vi.mocked(fetch)
-      fetchSpy.mockResolvedValueOnce({
+      mockNextFetchResponse({
         ok: false,
         status: 404,
         statusText: 'Not Found',
-        text: async () => 'Deployment not found',
-      } as any)
+        text: 'Deployment not found',
+      })
 
       await expect(generateSearchEmbedding('test query')).rejects.toThrow('Embedding API failed')
 
@@ -332,13 +318,12 @@ describe('Knowledge Search Utils', () => {
         OPENAI_API_KEY: 'test-openai-key',
       })
 
-      const fetchSpy = vi.mocked(fetch)
-      fetchSpy.mockResolvedValueOnce({
+      mockNextFetchResponse({
         ok: false,
         status: 429,
         statusText: 'Too Many Requests',
-        text: async () => 'Rate limit exceeded',
-      } as any)
+        text: 'Rate limit exceeded',
+      })
 
       await expect(generateSearchEmbedding('test query')).rejects.toThrow('Embedding API failed')
 
@@ -356,17 +341,16 @@ describe('Knowledge Search Utils', () => {
         KB_OPENAI_MODEL_NAME: 'text-embedding-ada-002',
       })
 
-      const fetchSpy = vi.mocked(fetch)
-      fetchSpy.mockResolvedValueOnce({
-        ok: true,
-        json: async () => ({
+      mockNextFetchResponse({
+        json: {
           data: [{ embedding: [0.1, 0.2, 0.3] }],
-        }),
-      } as any)
+          usage: { prompt_tokens: 1, total_tokens: 1 },
+        },
+      })
 
       await generateSearchEmbedding('test query')
 
-      expect(fetchSpy).toHaveBeenCalledWith(
+      expect(vi.mocked(fetch)).toHaveBeenCalledWith(
         expect.any(String),
         expect.objectContaining({
           body: JSON.stringify({
@@ -387,17 +371,16 @@ describe('Knowledge Search Utils', () => {
         OPENAI_API_KEY: 'test-openai-key',
       })
 
-      const fetchSpy = vi.mocked(fetch)
-      fetchSpy.mockResolvedValueOnce({
-        ok: true,
-        json: async () => ({
+      mockNextFetchResponse({
+        json: {
           data: [{ embedding: [0.1, 0.2, 0.3] }],
-        }),
-      } as any)
+          usage: { prompt_tokens: 1, total_tokens: 1 },
+        },
+      })
 
       await generateSearchEmbedding('test query', 'text-embedding-3-small')
 
-      expect(fetchSpy).toHaveBeenCalledWith(
+      expect(vi.mocked(fetch)).toHaveBeenCalledWith(
         expect.any(String),
         expect.objectContaining({
           body: JSON.stringify({

apps/sim/app/api/knowledge/utils.test.ts

@@ -77,6 +77,7 @@ vi.stubGlobal(
         { embedding: [0.1, 0.2], index: 0 },
         { embedding: [0.3, 0.4], index: 1 },
       ],
+      usage: { prompt_tokens: 2, total_tokens: 2 },
     }),
   })
 )
@@ -294,7 +295,7 @@ describe('Knowledge Utils', () => {
     it.concurrent('should return same length as input', async () => {
       const result = await generateEmbeddings(['a', 'b'])
 
-      expect(result.length).toBe(2)
+      expect(result.embeddings.length).toBe(2)
     })
 
     it('should use Azure OpenAI when Azure config is provided', async () => {
@@ -313,6 +314,7 @@ describe('Knowledge Utils', () => {
         ok: true,
         json: async () => ({
           data: [{ embedding: [0.1, 0.2], index: 0 }],
+          usage: { prompt_tokens: 1, total_tokens: 1 },
         }),
       } as any)
 
@@ -342,6 +344,7 @@ describe('Knowledge Utils', () => {
         ok: true,
         json: async () => ({
           data: [{ embedding: [0.1, 0.2], index: 0 }],
+          usage: { prompt_tokens: 1, total_tokens: 1 },
         }),
       } as any)
 

apps/sim/app/api/tools/cloudwatch/describe-alarms/route.ts

@@ -0,0 +1,96 @@
+import {
+  type AlarmType,
+  CloudWatchClient,
+  DescribeAlarmsCommand,
+  type StateValue,
+} from '@aws-sdk/client-cloudwatch'
+import { createLogger } from '@sim/logger'
+import { type NextRequest, NextResponse } from 'next/server'
+import { z } from 'zod'
+import { checkInternalAuth } from '@/lib/auth/hybrid'
+
+const logger = createLogger('CloudWatchDescribeAlarms')
+
+const DescribeAlarmsSchema = z.object({
+  region: z.string().min(1, 'AWS region is required'),
+  accessKeyId: z.string().min(1, 'AWS access key ID is required'),
+  secretAccessKey: z.string().min(1, 'AWS secret access key is required'),
+  alarmNamePrefix: z.string().optional(),
+  stateValue: z.preprocess(
+    (v) => (v === '' ? undefined : v),
+    z.enum(['OK', 'ALARM', 'INSUFFICIENT_DATA']).optional()
+  ),
+  alarmType: z.preprocess(
+    (v) => (v === '' ? undefined : v),
+    z.enum(['MetricAlarm', 'CompositeAlarm']).optional()
+  ),
+  limit: z.preprocess(
+    (v) => (v === '' || v === undefined || v === null ? undefined : v),
+    z.number({ coerce: true }).int().positive().optional()
+  ),
+})
+
+export async function POST(request: NextRequest) {
+  try {
+    const auth = await checkInternalAuth(request)
+    if (!auth.success || !auth.userId) {
+      return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 })
+    }
+
+    const body = await request.json()
+    const validatedData = DescribeAlarmsSchema.parse(body)
+
+    const client = new CloudWatchClient({
+      region: validatedData.region,
+      credentials: {
+        accessKeyId: validatedData.accessKeyId,
+        secretAccessKey: validatedData.secretAccessKey,
+      },
+    })
+
+    const command = new DescribeAlarmsCommand({
+      ...(validatedData.alarmNamePrefix && { AlarmNamePrefix: validatedData.alarmNamePrefix }),
+      ...(validatedData.stateValue && { StateValue: validatedData.stateValue as StateValue }),
+      ...(validatedData.alarmType && { AlarmTypes: [validatedData.alarmType as AlarmType] }),
+      ...(validatedData.limit !== undefined && { MaxRecords: validatedData.limit }),
+    })
+
+    const response = await client.send(command)
+
+    const metricAlarms = (response.MetricAlarms ?? []).map((a) => ({
+      alarmName: a.AlarmName ?? '',
+      alarmArn: a.AlarmArn ?? '',
+      stateValue: a.StateValue ?? 'UNKNOWN',
+      stateReason: a.StateReason ?? '',
+      metricName: a.MetricName,
+      namespace: a.Namespace,
+      comparisonOperator: a.ComparisonOperator,
+      threshold: a.Threshold,
+      evaluationPeriods: a.EvaluationPeriods,
+      stateUpdatedTimestamp: a.StateUpdatedTimestamp?.getTime(),
+    }))
+
+    const compositeAlarms = (response.CompositeAlarms ?? []).map((a) => ({
+      alarmName: a.AlarmName ?? '',
+      alarmArn: a.AlarmArn ?? '',
+      stateValue: a.StateValue ?? 'UNKNOWN',
+      stateReason: a.StateReason ?? '',
+      metricName: undefined,
+      namespace: undefined,
+      comparisonOperator: undefined,
+      threshold: undefined,
+      evaluationPeriods: undefined,
+      stateUpdatedTimestamp: a.StateUpdatedTimestamp?.getTime(),
+    }))
+
+    return NextResponse.json({
+      success: true,
+      output: { alarms: [...metricAlarms, ...compositeAlarms] },
+    })
+  } catch (error) {
+    const errorMessage =
+      error instanceof Error ? error.message : 'Failed to describe CloudWatch alarms'
+    logger.error('DescribeAlarms failed', { error: errorMessage })
+    return NextResponse.json({ error: errorMessage }, { status: 500 })
+  }
+}