Skip to content

Commit 4c52059

Browse files
brendan-kellambrendan-kellam
authored
Declarative connection configuration (#235)
1 parent 2b17fd6 commit 4c52059

File tree

44 files changed

+1628
-377
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

44 files changed

+1628
-377
lines changed

‎.env.development‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ SRC_TENANT_ENFORCEMENT_MODE=strict
1515
# You can generate a new secret with:
1616
# openssl rand -base64 33
1717
# @see: https://authjs.dev/getting-started/deployment#auth_secret
18-
AUTH_SECRET="secret"
18+
AUTH_SECRET="00000000000000000000000000000000000000000000"
1919
AUTH_URL="http://localhost:3000"
2020
# AUTH_CREDENTIALS_LOGIN_ENABLED=true
2121
# AUTH_GITHUB_CLIENT_ID=""
@@ -59,7 +59,7 @@ REDIS_URL="redis://localhost:6379"
5959

6060
# Generated using:
6161
# openssl rand -base64 24
62-
SOURCEBOT_ENCRYPTION_KEY="secret"
62+
SOURCEBOT_ENCRYPTION_KEY="00000000000000000000000000000000"
6363

6464
SOURCEBOT_LOG_LEVEL="debug" # valid values: info, debug, warn, error
6565
SOURCEBOT_TELEMETRY_DISABLED=true # Disables telemetry collection
@@ -79,6 +79,5 @@ SOURCEBOT_TELEMETRY_DISABLED=true # Disables telemetry collection
7979
# NEXT_PUBLIC_SOURCEBOT_VERSION=
8080

8181
# CONFIG_MAX_REPOS_NO_TOKEN=
82-
# SOURCEBOT_ROOT_DOMAIN=
8382
# NODE_ENV=
84-
# SOURCEBOT_TENANCY_MODE=mutli
83+
# SOURCEBOT_TENANCY_MODE=single

‎package.json‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,13 +7,13 @@
77
"build": "yarn workspaces run build",
88
"test": "yarn workspaces run test",
99

10-
"dev": "yarn dev:prisma:migrate && npm-run-all --print-label --parallel dev:zoekt dev:backend dev:web",
10+
"dev": "yarn dev:prisma:migrate:dev && npm-run-all --print-label --parallel dev:zoekt dev:backend dev:web",
1111
"with-env": "cross-env PATH=\"$PWD/bin:$PATH\" dotenv -e .env.development -c --",
1212
"dev:zoekt": "yarn with-env zoekt-webserver -index .sourcebot/index -rpc",
1313
"dev:backend": "yarn with-env yarn workspace @sourcebot/backend dev:watch",
1414
"dev:web": "yarn with-env yarn workspace @sourcebot/web dev",
1515

16-
"dev:prisma:migrate": "yarn with-env yarn workspace @sourcebot/db prisma:migrate:dev",
16+
"dev:prisma:migrate:dev": "yarn with-env yarn workspace @sourcebot/db prisma:migrate:dev",
1717
"dev:prisma:studio": "yarn with-env yarn workspace @sourcebot/db prisma:studio",
1818
"dev:prisma:migrate:reset": "yarn with-env yarn workspace @sourcebot/db prisma:migrate:reset"
1919
},

‎packages/backend/package.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434
"@sourcebot/schemas": "^0.1.0",
3535
"@t3-oss/env-core": "^0.12.0",
3636
"@types/express": "^5.0.0",
37+
"ajv": "^8.17.1",
3738
"argparse": "^2.0.1",
3839
"bullmq": "^5.34.10",
3940
"cross-fetch": "^4.0.0",

‎packages/backend/src/env.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -41,8 +41,8 @@ export const env = createEnv({
4141
LOGTAIL_TOKEN: z.string().optional(),
4242
LOGTAIL_HOST: z.string().url().optional(),
4343

44-
INDEX_CONCURRENCY_MULTIPLE: numberSchema.optional(),
45-
DATABASE_URL: z.string().url().default("postgresql://postgres:postgres@localhost:5432/postgres")
44+
DATABASE_URL: z.string().url().default("postgresql://postgres:postgres@localhost:5432/postgres"),
45+
CONFIG_PATH: z.string().optional(),
4646
},
4747
runtimeEnv: process.env,
4848
emptyStringAsUndefined: true,

‎packages/backend/src/gitea.ts‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,8 +12,7 @@ import { env } from './env.js';
1212
const logger = createLogger('Gitea');
1313

1414
export const getGiteaReposFromConfig = async (config: GiteaConnectionConfig, orgId: number, db: PrismaClient) => {
15-
const tokenResult = config.token ? await getTokenFromConfig(config.token, orgId, db) : undefined;
16-
const token = tokenResult?.token ?? env.FALLBACK_GITEA_TOKEN;
15+
const token = config.token ? await getTokenFromConfig(config.token, orgId, db, logger) : env.FALLBACK_GITEA_TOKEN;
1716

1817
const api = giteaApi(config.url ?? 'https://gitea.com', {
1918
token: token,

‎packages/backend/src/github.ts‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -40,12 +40,10 @@ const isHttpError = (error: unknown, status: number): boolean => {
4040
}
4141

4242
export const getGitHubReposFromConfig = async (config: GithubConnectionConfig, orgId: number, db: PrismaClient, signal: AbortSignal) => {
43-
const tokenResult = config.token ? await getTokenFromConfig(config.token, orgId, db) : undefined;
44-
const token = tokenResult?.token;
45-
const secretKey = tokenResult?.secretKey;
43+
const token = config.token ? await getTokenFromConfig(config.token, orgId, db, logger) : env.FALLBACK_GITHUB_TOKEN;
4644

4745
const octokit = new Octokit({
48-
auth: token ?? env.FALLBACK_GITHUB_TOKEN,
46+
auth: token,
4947
...(config.url ? {
5048
baseUrl: `${config.url}/api/v3`
5149
} : {}),
@@ -59,7 +57,9 @@ export const getGitHubReposFromConfig = async (config: GithubConnectionConfig, o
5957

6058
if (isHttpError(error, 401)) {
6159
const e = new BackendException(BackendError.CONNECTION_SYNC_INVALID_TOKEN, {
62-
secretKey,
60+
...(config.token && 'secret' in config.token ? {
61+
secretKey: config.token.secret,
62+
} : {}),
6363
});
6464
Sentry.captureException(e);
6565
throw e;

‎packages/backend/src/gitlab.ts‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,8 +12,7 @@ const logger = createLogger("GitLab");
1212
export const GITLAB_CLOUD_HOSTNAME = "gitlab.com";
1313

1414
export const getGitLabReposFromConfig = async (config: GitlabConnectionConfig, orgId: number, db: PrismaClient) => {
15-
const tokenResult = config.token ? await getTokenFromConfig(config.token, orgId, db) : undefined;
16-
const token = tokenResult?.token ?? env.FALLBACK_GITLAB_TOKEN;
15+
const token = config.token ? await getTokenFromConfig(config.token, orgId, db, logger) : env.FALLBACK_GITLAB_TOKEN;
1716

1817
const api = new Gitlab({
1918
...(token ? {

‎packages/backend/src/index.ts‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,6 @@ const parser = new ArgumentParser({
4242
});
4343

4444
type Arguments = {
45-
configPath: string;
4645
cacheDir: string;
4746
}
4847

@@ -67,7 +66,6 @@ const context: AppContext = {
6766
indexPath,
6867
reposPath,
6968
cachePath: cacheDir,
70-
configPath: args.configPath,
7169
}
7270

7371
const prisma = new PrismaClient();

‎packages/backend/src/main.ts‎

Lines changed: 39 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,46 @@ import { ConnectionManager } from './connectionManager.js';
77
import { RepoManager } from './repoManager.js';
88
import { env } from './env.js';
99
import { PromClient } from './promClient.js';
10+
import { isRemotePath } from './utils.js';
11+
import { readFile } from 'fs/promises';
12+
import stripJsonComments from 'strip-json-comments';
13+
import { SourcebotConfig } from '@sourcebot/schemas/v3/index.type';
14+
import { indexSchema } from '@sourcebot/schemas/v3/index.schema';
15+
import { Ajv } from "ajv";
1016

1117
const logger = createLogger('main');
18+
const ajv = new Ajv({
19+
validateFormats: false,
20+
});
21+
22+
const getSettings = async (configPath?: string) => {
23+
if (!configPath) {
24+
return DEFAULT_SETTINGS;
25+
}
26+
27+
const configContent = await (async () => {
28+
if (isRemotePath(configPath)) {
29+
const response = await fetch(configPath);
30+
if (!response.ok) {
31+
throw new Error(`Failed to fetch config file ${configPath}: ${response.statusText}`);
32+
}
33+
return response.text();
34+
} else {
35+
return readFile(configPath, { encoding: 'utf-8' });
36+
}
37+
})();
38+
39+
const config = JSON.parse(stripJsonComments(configContent)) as SourcebotConfig;
40+
const isValidConfig = ajv.validate(indexSchema, config);
41+
if (!isValidConfig) {
42+
throw new Error(`Config file '${configPath}' is invalid: ${ajv.errorsText(ajv.errors)}`);
43+
}
44+
45+
return {
46+
...DEFAULT_SETTINGS,
47+
...config.settings,
48+
}
49+
}
1250

1351
export const main = async (db: PrismaClient, context: AppContext) => {
1452
const redis = new Redis(env.REDIS_URL, {
@@ -22,10 +60,7 @@ export const main = async (db: PrismaClient, context: AppContext) => {
2260
process.exit(1);
2361
});
2462

25-
const settings = DEFAULT_SETTINGS;
26-
if (env.INDEX_CONCURRENCY_MULTIPLE) {
27-
settings.indexConcurrencyMultiple = env.INDEX_CONCURRENCY_MULTIPLE;
28-
}
63+
const settings = await getSettings(env.CONFIG_PATH);
2964

3065
const promClient = new PromClient();
3166

‎packages/backend/src/repoManager.ts‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ import { indexGitRepository } from "./zoekt.js";
1111
import os from 'os';
1212
import { PromClient } from './promClient.js';
1313
import * as Sentry from "@sentry/node";
14+
1415
interface IRepoManager {
1516
blockingPollLoop: () => void;
1617
dispose: () => void;
@@ -177,8 +178,7 @@ export class RepoManager implements IRepoManager {
177178

178179
const config = connection.config as unknown as GithubConnectionConfig | GitlabConnectionConfig | GiteaConnectionConfig;
179180
if (config.token) {
180-
const tokenResult = await getTokenFromConfig(config.token, connection.orgId, db);
181-
token = tokenResult?.token;
181+
token = await getTokenFromConfig(config.token, connection.orgId, db, this.logger);
182182
if (token) {
183183
break;
184184
}
@@ -207,7 +207,7 @@ export class RepoManager implements IRepoManager {
207207
this.logger.info(`Fetching ${repo.id}...`);
208208

209209
const { durationMs } = await measure(() => fetchRepository(repoPath, ({ method, stage, progress }) => {
210-
//this.logger.info(`git.${method} ${stage} stage ${progress}% complete for ${repo.id}`)
210+
this.logger.debug(`git.${method} ${stage} stage ${progress}% complete for ${repo.id}`)
211211
}));
212212
fetchDuration_s = durationMs / 1000;
213213

@@ -234,7 +234,7 @@ export class RepoManager implements IRepoManager {
234234
}
235235

236236
const { durationMs } = await measure(() => cloneRepository(cloneUrl.toString(), repoPath, metadata.gitConfig, ({ method, stage, progress }) => {
237-
//this.logger.info(`git.${method} ${stage} stage ${progress}% complete for ${repo.id}`)
237+
this.logger.debug(`git.${method} ${stage} stage ${progress}% complete for ${repo.id}`)
238238
}));
239239
cloneDuration_s = durationMs / 1000;
240240

@@ -243,7 +243,7 @@ export class RepoManager implements IRepoManager {
243243
}
244244

245245
this.logger.info(`Indexing ${repo.id}...`);
246-
const { durationMs } = await measure(() => indexGitRepository(repo, this.ctx));
246+
const { durationMs } = await measure(() => indexGitRepository(repo, this.settings, this.ctx));
247247
const indexDuration_s = durationMs / 1000;
248248
this.logger.info(`Indexed ${repo.id} in ${indexDuration_s}s`);
249249

0 commit comments

Comments
 (0)