further wip

Author: brendan-kellam

Dockerfile

@@ -43,12 +43,14 @@ COPY ./packages/schemas ./packages/schemas
 COPY ./packages/crypto ./packages/crypto
 COPY ./packages/error ./packages/error
 COPY ./packages/logger ./packages/logger
+COPY ./packages/shared ./packages/shared
 
 RUN yarn workspace @sourcebot/db install
 RUN yarn workspace @sourcebot/schemas install
 RUN yarn workspace @sourcebot/crypto install
 RUN yarn workspace @sourcebot/error install
 RUN yarn workspace @sourcebot/logger install
+RUN yarn workspace @sourcebot/shared install
 # ------------------------------------
 
 # ------ Build Web ------
@@ -92,6 +94,7 @@ COPY --from=shared-libs-builder /app/packages/schemas ./packages/schemas
 COPY --from=shared-libs-builder /app/packages/crypto ./packages/crypto
 COPY --from=shared-libs-builder /app/packages/error ./packages/error
 COPY --from=shared-libs-builder /app/packages/logger ./packages/logger
+COPY --from=shared-libs-builder /app/packages/shared ./packages/shared
 
 # Fixes arm64 timeouts
 RUN yarn workspace @sourcebot/web install
@@ -132,6 +135,7 @@ COPY --from=shared-libs-builder /app/packages/schemas ./packages/schemas
 COPY --from=shared-libs-builder /app/packages/crypto ./packages/crypto
 COPY --from=shared-libs-builder /app/packages/error ./packages/error
 COPY --from=shared-libs-builder /app/packages/logger ./packages/logger
+COPY --from=shared-libs-builder /app/packages/shared ./packages/shared
 RUN yarn workspace @sourcebot/backend install
 RUN yarn workspace @sourcebot/backend build
 
@@ -215,6 +219,7 @@ COPY --from=shared-libs-builder /app/packages/schemas ./packages/schemas
 COPY --from=shared-libs-builder /app/packages/crypto ./packages/crypto
 COPY --from=shared-libs-builder /app/packages/error ./packages/error
 COPY --from=shared-libs-builder /app/packages/logger ./packages/logger
+COPY --from=shared-libs-builder /app/packages/shared ./packages/shared
 
 # Configure dependencies
 RUN apk add --no-cache git ca-certificates bind-tools tini jansson wget supervisor uuidgen curl perl jq redis postgresql postgresql-contrib openssl util-linux unzip

LICENSE

@@ -2,7 +2,7 @@ Copyright (c) 2025 Taqla Inc.
 
 Portions of this software are licensed as follows:
 
-- All content that resides under the "ee/", "packages/web/src/ee/", and "packages/backend/src/ee/" directories of this repository, if these directories exist, is licensed under the license defined in "ee/LICENSE".
+- All content that resides under the "ee/", "packages/web/src/ee/", and "packages/shared/src/ee/" directories of this repository, if these directories exist, is licensed under the license defined in "ee/LICENSE".
 - All third party components incorporated into the Sourcebot Software are licensed under the original license provided by the owner of the applicable component.
 - Content outside of the above mentioned directories or restrictions above is available under the "MIT Expat" license as defined below.
 

packages/backend/package.json

@@ -33,9 +33,9 @@
         "@sourcebot/error": "workspace:*",
         "@sourcebot/logger": "workspace:*",
         "@sourcebot/schemas": "workspace:*",
+        "@sourcebot/shared": "workspace:*",
         "@t3-oss/env-core": "^0.12.0",
         "@types/express": "^5.0.0",
-        "ajv": "^8.17.1",
         "argparse": "^2.0.1",
         "bullmq": "^5.34.10",
         "cross-fetch": "^4.0.0",
@@ -50,7 +50,6 @@
         "posthog-node": "^4.2.1",
         "prom-client": "^15.1.3",
         "simple-git": "^3.27.0",
-        "strip-json-comments": "^5.0.1",
         "zod": "^3.24.3"
     }
 }

packages/backend/src/connectionManager.ts

@@ -9,8 +9,7 @@ import { BackendError, BackendException } from "@sourcebot/error";
 import { captureEvent } from "./posthog.js";
 import { env } from "./env.js";
 import * as Sentry from "@sentry/node";
-import { syncSearchContexts } from "./ee/syncSearchContexts.js";
-import { AppContext } from "./types.js";
+import { loadConfig, syncSearchContexts } from "@sourcebot/shared";
 
 interface IConnectionManager {
     scheduleConnectionSync: (connection: Connection) => Promise<void>;
@@ -40,7 +39,6 @@ export class ConnectionManager implements IConnectionManager {
         private db: PrismaClient,
         private settings: Settings,
         redis: Redis,
-        private ctx: AppContext,
     ) {
         this.queue = new Queue<JobPayload>(QUEUE_NAME, {
             connection: redis,
@@ -267,7 +265,7 @@ export class ConnectionManager implements IConnectionManager {
 
     private async onSyncJobCompleted(job: Job<JobPayload>, result: JobResult) {
         this.logger.info(`Connection sync job for connection ${job.data.connectionName} (id: ${job.data.connectionId}, jobId: ${job.id}) completed`);
-        const { connectionId } = job.data;
+        const { connectionId, orgId } = job.data;
 
         let syncStatusMetadata: Record<string, unknown> = (await this.db.connection.findUnique({
             where: { id: connectionId },
@@ -294,7 +292,18 @@ export class ConnectionManager implements IConnectionManager {
             }
         });
 
-        await syncSearchContexts(this.db, this.ctx.config?.contexts);
+        // After a connection has synced, we need to re-sync the org's search contexts as
+        // there may be new repos that match the search context's include/exclude patterns.
+        if (env.CONFIG_PATH) {
+            const config = await loadConfig(env.CONFIG_PATH);
+
+            await syncSearchContexts({
+                db: this.db,
+                orgId,
+                contexts: config.contexts,
+            });
+        }
+
 
         captureEvent('backend_connection_sync_job_completed', {
             connectionId: connectionId,

packages/backend/src/index.ts

@@ -9,43 +9,9 @@ import { main } from "./main.js"
 import { PrismaClient } from "@sourcebot/db";
 import { env } from "./env.js";
 import { createLogger } from "@sourcebot/logger";
-import { isRemotePath } from './utils.js';
-import { readFile } from 'fs/promises';
-import stripJsonComments from 'strip-json-comments';
-import { SourcebotConfig } from '@sourcebot/schemas/v3/index.type';
-import { indexSchema } from '@sourcebot/schemas/v3/index.schema';
-import { Ajv } from "ajv";
 
 const logger = createLogger('backend-entrypoint');
-const ajv = new Ajv({
-    validateFormats: false,
-});
-
-const loadConfig = async (configPath?: string) => {
-    if (!configPath) {
-        return undefined;
-    }
-
-    const configContent = await (async () => {
-        if (isRemotePath(configPath)) {
-            const response = await fetch(configPath);
-            if (!response.ok) {
-                throw new Error(`Failed to fetch config file ${configPath}: ${response.statusText}`);
-            }
-            return response.text();
-        } else {
-            return readFile(configPath, { encoding: 'utf-8' });
-        }
-    })();
 
-    const config = JSON.parse(stripJsonComments(configContent)) as SourcebotConfig;
-    const isValidConfig = ajv.validate(indexSchema, config);
-    if (!isValidConfig) {
-        throw new Error(`Config file '${configPath}' is invalid: ${ajv.errorsText(ajv.errors)}`);
-    }
-
-    return config;
-}
 
 // Register handler for normal exit
 process.on('exit', (code) => {
@@ -85,13 +51,10 @@ if (!existsSync(indexPath)) {
     await mkdir(indexPath, { recursive: true });
 }
 
-const config = await loadConfig(env.CONFIG_PATH);
-
 const context: AppContext = {
     indexPath,
     reposPath,
     cachePath: cacheDir,
-    config,
 }
 
 const prisma = new PrismaClient();

packages/backend/src/main.ts

@@ -1,16 +1,29 @@
 import { PrismaClient } from '@sourcebot/db';
 import { createLogger } from "@sourcebot/logger";
 import { AppContext } from "./types.js";
-import { DEFAULT_SETTINGS, SINGLE_TENANT_ORG_DOMAIN, SINGLE_TENANT_ORG_ID, SINGLE_TENANT_ORG_NAME } from './constants.js';
+import { DEFAULT_SETTINGS } from './constants.js';
 import { Redis } from 'ioredis';
 import { ConnectionManager } from './connectionManager.js';
 import { RepoManager } from './repoManager.js';
 import { env } from './env.js';
 import { PromClient } from './promClient.js';
-import { syncSearchContexts } from './ee/syncSearchContexts.js';
+import { loadConfig } from '@sourcebot/shared';
 
 const logger = createLogger('backend-main');
 
+const getSettings = async (configPath?: string) => {
+    if (!configPath) {
+        return DEFAULT_SETTINGS;
+    }
+
+    const config = await loadConfig(configPath);
+
+    return {
+        ...DEFAULT_SETTINGS,
+        ...config.settings,
+    }
+}
+
 export const main = async (db: PrismaClient, context: AppContext) => {
     const redis = new Redis(env.REDIS_URL, {
         maxRetriesPerRequest: null
@@ -23,28 +36,11 @@ export const main = async (db: PrismaClient, context: AppContext) => {
         process.exit(1);
     });
 
-    const settings = {
-        ...DEFAULT_SETTINGS,
-        ...context.config?.settings,
-    }
-
-    await db.org.upsert({
-        where: {
-            id: SINGLE_TENANT_ORG_ID,
-        },
-        update: {},
-        create: {
-            name: SINGLE_TENANT_ORG_NAME,
-            domain: SINGLE_TENANT_ORG_DOMAIN,
-            id: SINGLE_TENANT_ORG_ID
-        }
-    });
-
-    await syncSearchContexts(db, context.config?.contexts);
+    const settings = await getSettings(env.CONFIG_PATH);
 
     const promClient = new PromClient();
 
-    const connectionManager = new ConnectionManager(db, settings, redis, context);
+    const connectionManager = new ConnectionManager(db, settings, redis);
     connectionManager.registerPollingCallback();
 
     const repoManager = new RepoManager(db, settings, redis, promClient, context);

packages/backend/src/types.ts

@@ -1,4 +1,4 @@
-import { Settings as SettingsSchema, SourcebotConfig } from "@sourcebot/schemas/v3/index.type";
+import { Settings as SettingsSchema } from "@sourcebot/schemas/v3/index.type";
 import { z } from "zod";
 
 export type AppContext = {
@@ -13,8 +13,6 @@ export type AppContext = {
     indexPath: string;
 
     cachePath: string;
-
-    config?: SourcebotConfig;
 }
 
 export type Settings = Required<SettingsSchema>;

packages/backend/src/utils.ts

@@ -20,10 +20,6 @@ export const marshalBool = (value?: boolean) => {
     return !!value ? '1' : '0';
 }
 
-export const isRemotePath = (path: string) => {
-    return path.startsWith('https://') || path.startsWith('http://');
-}
-
 export const getTokenFromConfig = async (token: any, orgId: number, db: PrismaClient, logger?: Logger) => {
     try {
         return await getTokenFromConfigBase(token, orgId, db);

packages/shared/.gitignore

@@ -0,0 +1,2 @@
+dist/
+*.tsbuildinfo 

packages/shared/package.json

@@ -0,0 +1,31 @@
+{
+    "name": "@sourcebot/shared",
+    "version": "0.1.0",
+    "type": "module",
+    "private": true,
+    "scripts": {
+        "build": "tsc",
+        "build:watch": "tsc-watch --preserveWatchOutput",
+        "postinstall": "yarn build"
+    },
+    "dependencies": {
+        "@sourcebot/crypto": "workspace:*",
+        "@sourcebot/db": "workspace:*",
+        "@sourcebot/logger": "workspace:*",
+        "@sourcebot/schemas": "workspace:*",
+        "@t3-oss/env-core": "^0.12.0",
+        "ajv": "^8.17.1",
+        "server-only": "^0.0.1",
+        "strip-json-comments": "^5.0.1",
+        "zod": "^3.24.3"
+    },
+    "devDependencies": {
+        "@types/node": "^22.7.5",
+        "tsc-watch": "6.2.1",
+        "typescript": "^5.7.3"
+    },
+    "exports": {
+        ".": "./dist/index.server.js",
+        "./client": "./dist/index.client.js"
+    }
+}