Skip to content

Commit a167acc

Browse files
brendan-kellambrendan-kellam
authored
feat(worker,web): Improved connection management (#579)
1 parent 3ff88da commit a167acc

63 files changed

Lines changed: 2428 additions & 1192 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
2828
- Changed navbar indexing indicator to only report progress for first time indexing jobs. [#563](https://github.com/sourcebot-dev/sourcebot/pull/563)
2929
- Improved repo indexing job stability and robustness. [#563](https://github.com/sourcebot-dev/sourcebot/pull/563)
3030
- Improved repositories table. [#572](https://github.com/sourcebot-dev/sourcebot/pull/572)
31+
- Improved connections table. [#579](https://github.com/sourcebot-dev/sourcebot/pull/579)
3132

3233
### Removed
3334
- Removed spam "login page loaded" log. [#552](https://github.com/sourcebot-dev/sourcebot/pull/552)

‎docs/snippets/schemas/v3/app.schema.mdx‎

Lines changed: 61 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -3,11 +3,9 @@
33
{
44
"$schema": "http://json-schema.org/draft-07/schema#",
55
"title": "AppConfig",
6-
"oneOf": [
7-
{
8-
"$schema": "http://json-schema.org/draft-07/schema#",
6+
"definitions": {
7+
"GitHubAppConfig": {
98
"type": "object",
10-
"title": "GithubAppConfig",
119
"properties": {
1210
"type": {
1311
"const": "githubApp",
@@ -61,19 +59,70 @@
6159
},
6260
"required": [
6361
"type",
64-
"id"
62+
"id",
63+
"privateKey"
6564
],
66-
"oneOf": [
67-
{
68-
"required": [
69-
"privateKey"
65+
"additionalProperties": false
66+
}
67+
},
68+
"oneOf": [
69+
{
70+
"type": "object",
71+
"properties": {
72+
"type": {
73+
"const": "githubApp",
74+
"description": "GitHub App Configuration"
75+
},
76+
"deploymentHostname": {
77+
"type": "string",
78+
"format": "hostname",
79+
"default": "github.com",
80+
"description": "The hostname of the GitHub App deployment.",
81+
"examples": [
82+
"github.com",
83+
"github.example.com"
7084
]
7185
},
72-
{
73-
"required": [
74-
"privateKeyPath"
86+
"id": {
87+
"type": "string",
88+
"description": "The ID of the GitHub App."
89+
},
90+
"privateKey": {
91+
"description": "The private key of the GitHub App.",
92+
"anyOf": [
93+
{
94+
"type": "object",
95+
"properties": {
96+
"secret": {
97+
"type": "string",
98+
"description": "The name of the secret that contains the token."
99+
}
100+
},
101+
"required": [
102+
"secret"
103+
],
104+
"additionalProperties": false
105+
},
106+
{
107+
"type": "object",
108+
"properties": {
109+
"env": {
110+
"type": "string",
111+
"description": "The name of the environment variable that contains the token. Only supported in declarative connection configs."
112+
}
113+
},
114+
"required": [
115+
"env"
116+
],
117+
"additionalProperties": false
118+
}
75119
]
76120
}
121+
},
122+
"required": [
123+
"type",
124+
"id",
125+
"privateKey"
77126
],
78127
"additionalProperties": false
79128
}

‎docs/snippets/schemas/v3/index.schema.mdx‎

Lines changed: 62 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -4280,11 +4280,9 @@
42804280
"items": {
42814281
"$schema": "http://json-schema.org/draft-07/schema#",
42824282
"title": "AppConfig",
4283-
"oneOf": [
4284-
{
4285-
"$schema": "http://json-schema.org/draft-07/schema#",
4283+
"definitions": {
4284+
"GitHubAppConfig": {
42864285
"type": "object",
4287-
"title": "GithubAppConfig",
42884286
"properties": {
42894287
"type": {
42904288
"const": "githubApp",
@@ -4338,19 +4336,70 @@
43384336
},
43394337
"required": [
43404338
"type",
4341-
"id"
4339+
"id",
4340+
"privateKey"
43424341
],
4343-
"oneOf": [
4344-
{
4345-
"required": [
4346-
"privateKey"
4347-
]
4342+
"additionalProperties": false
4343+
}
4344+
},
4345+
"oneOf": [
4346+
{
4347+
"type": "object",
4348+
"properties": {
4349+
"type": {
4350+
"const": "githubApp",
4351+
"description": "GitHub App Configuration"
43484352
},
4349-
{
4350-
"required": [
4351-
"privateKeyPath"
4353+
"deploymentHostname": {
4354+
"type": "string",
4355+
"format": "hostname",
4356+
"default": "github.com",
4357+
"description": "The hostname of the GitHub App deployment.",
4358+
"examples": [
4359+
"github.com",
4360+
"github.example.com"
43524361
]
4362+
},
4363+
"id": {
4364+
"type": "string",
4365+
"description": "The ID of the GitHub App."
4366+
},
4367+
"privateKey": {
4368+
"anyOf": [
4369+
{
4370+
"type": "object",
4371+
"properties": {
4372+
"secret": {
4373+
"type": "string",
4374+
"description": "The name of the secret that contains the token."
4375+
}
4376+
},
4377+
"required": [
4378+
"secret"
4379+
],
4380+
"additionalProperties": false
4381+
},
4382+
{
4383+
"type": "object",
4384+
"properties": {
4385+
"env": {
4386+
"type": "string",
4387+
"description": "The name of the environment variable that contains the token. Only supported in declarative connection configs."
4388+
}
4389+
},
4390+
"required": [
4391+
"env"
4392+
],
4393+
"additionalProperties": false
4394+
}
4395+
],
4396+
"description": "The private key of the GitHub App."
43534397
}
4398+
},
4399+
"required": [
4400+
"type",
4401+
"id",
4402+
"privateKey"
43544403
],
43554404
"additionalProperties": false
43564405
}

‎packages/backend/package.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@
4040
"argparse": "^2.0.1",
4141
"azure-devops-node-api": "^15.1.1",
4242
"bullmq": "^5.34.10",
43+
"chokidar": "^4.0.3",
4344
"cross-fetch": "^4.0.0",
4445
"dotenv": "^16.4.5",
4546
"express": "^4.21.2",

‎packages/backend/src/azuredevops.ts‎

Lines changed: 36 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,14 @@
11
import { AzureDevOpsConnectionConfig } from "@sourcebot/schemas/v3/azuredevops.type";
22
import { createLogger } from "@sourcebot/logger";
3-
import { getTokenFromConfig, measure, fetchWithRetry } from "./utils.js";
3+
import { measure, fetchWithRetry } from "./utils.js";
44
import micromatch from "micromatch";
55
import { PrismaClient } from "@sourcebot/db";
66
import { BackendException, BackendError } from "@sourcebot/error";
77
import { processPromiseResults, throwIfAnyFailed } from "./connectionUtils.js";
88
import * as Sentry from "@sentry/node";
99
import * as azdev from "azure-devops-node-api";
1010
import { GitRepository } from "azure-devops-node-api/interfaces/GitInterfaces.js";
11+
import { getTokenFromConfig } from "@sourcebot/crypto";
1112

1213
const logger = createLogger('azuredevops');
1314
const AZUREDEVOPS_CLOUD_HOSTNAME = "dev.azure.com";
@@ -34,7 +35,7 @@ export const getAzureDevOpsReposFromConfig = async (
3435
const baseUrl = config.url || `https://${AZUREDEVOPS_CLOUD_HOSTNAME}`;
3536

3637
const token = config.token ?
37-
await getTokenFromConfig(config.token, orgId, db, logger) :
38+
await getTokenFromConfig(config.token, orgId, db) :
3839
undefined;
3940

4041
if (!token) {
@@ -47,47 +48,39 @@ export const getAzureDevOpsReposFromConfig = async (
4748

4849
const useTfsPath = config.useTfsPath || false;
4950
let allRepos: GitRepository[] = [];
50-
let notFound: {
51-
users: string[],
52-
orgs: string[],
53-
repos: string[],
54-
} = {
55-
users: [],
56-
orgs: [],
57-
repos: [],
58-
};
51+
let allWarnings: string[] = [];
5952

6053
if (config.orgs) {
61-
const { validRepos, notFoundOrgs } = await getReposForOrganizations(
54+
const { repos, warnings } = await getReposForOrganizations(
6255
config.orgs,
6356
baseUrl,
6457
token,
6558
useTfsPath
6659
);
67-
allRepos = allRepos.concat(validRepos);
68-
notFound.orgs = notFoundOrgs;
60+
allRepos = allRepos.concat(repos);
61+
allWarnings = allWarnings.concat(warnings);
6962
}
7063

7164
if (config.projects) {
72-
const { validRepos, notFoundProjects } = await getReposForProjects(
65+
const { repos, warnings } = await getReposForProjects(
7366
config.projects,
7467
baseUrl,
7568
token,
7669
useTfsPath
7770
);
78-
allRepos = allRepos.concat(validRepos);
79-
notFound.repos = notFound.repos.concat(notFoundProjects);
71+
allRepos = allRepos.concat(repos);
72+
allWarnings = allWarnings.concat(warnings);
8073
}
8174

8275
if (config.repos) {
83-
const { validRepos, notFoundRepos } = await getRepos(
76+
const { repos, warnings } = await getRepos(
8477
config.repos,
8578
baseUrl,
8679
token,
8780
useTfsPath
8881
);
89-
allRepos = allRepos.concat(validRepos);
90-
notFound.repos = notFound.repos.concat(notFoundRepos);
82+
allRepos = allRepos.concat(repos);
83+
allWarnings = allWarnings.concat(warnings);
9184
}
9285

9386
let repos = allRepos
@@ -103,8 +96,8 @@ export const getAzureDevOpsReposFromConfig = async (
10396
logger.debug(`Found ${repos.length} total repositories.`);
10497

10598
return {
106-
validRepos: repos,
107-
notFound,
99+
repos,
100+
warnings: allWarnings,
108101
};
109102
};
110103

@@ -221,22 +214,23 @@ async function getReposForOrganizations(
221214

222215
// Check if it's a 404-like error (organization not found)
223216
if (error && typeof error === 'object' && 'statusCode' in error && error.statusCode === 404) {
224-
logger.error(`Organization ${org} not found or no access`);
217+
const warning = `Organization ${org} not found or no access`;
218+
logger.warn(warning);
225219
return {
226-
type: 'notFound' as const,
227-
value: org
220+
type: 'warning' as const,
221+
warning
228222
};
229223
}
230224
throw error;
231225
}
232226
}));
233227

234228
throwIfAnyFailed(results);
235-
const { validItems: validRepos, notFoundItems: notFoundOrgs } = processPromiseResults<GitRepository>(results);
229+
const { validItems: repos, warnings } = processPromiseResults<GitRepository>(results);
236230

237231
return {
238-
validRepos,
239-
notFoundOrgs,
232+
repos,
233+
warnings,
240234
};
241235
}
242236

@@ -274,22 +268,23 @@ async function getReposForProjects(
274268
logger.error(`Failed to fetch repositories for project ${project}.`, error);
275269

276270
if (error && typeof error === 'object' && 'statusCode' in error && error.statusCode === 404) {
277-
logger.error(`Project ${project} not found or no access`);
271+
const warning = `Project ${project} not found or no access`;
272+
logger.warn(warning);
278273
return {
279-
type: 'notFound' as const,
280-
value: project
274+
type: 'warning' as const,
275+
warning
281276
};
282277
}
283278
throw error;
284279
}
285280
}));
286281

287282
throwIfAnyFailed(results);
288-
const { validItems: validRepos, notFoundItems: notFoundProjects } = processPromiseResults<GitRepository>(results);
283+
const { validItems: repos, warnings } = processPromiseResults<GitRepository>(results);
289284

290285
return {
291-
validRepos,
292-
notFoundProjects,
286+
repos,
287+
warnings,
293288
};
294289
}
295290

@@ -328,21 +323,22 @@ async function getRepos(
328323
logger.error(`Failed to fetch repository ${repo}.`, error);
329324

330325
if (error && typeof error === 'object' && 'statusCode' in error && error.statusCode === 404) {
331-
logger.error(`Repository ${repo} not found or no access`);
326+
const warning = `Repository ${repo} not found or no access`;
327+
logger.warn(warning);
332328
return {
333-
type: 'notFound' as const,
334-
value: repo
329+
type: 'warning' as const,
330+
warning
335331
};
336332
}
337333
throw error;
338334
}
339335
}));
340336

341337
throwIfAnyFailed(results);
342-
const { validItems: validRepos, notFoundItems: notFoundRepos } = processPromiseResults<GitRepository>(results);
338+
const { validItems: repos, warnings } = processPromiseResults<GitRepository>(results);
343339

344340
return {
345-
validRepos,
346-
notFoundRepos,
341+
repos,
342+
warnings,
347343
};
348344
}

0 commit comments

Comments
 (0)