Skip to content

Commit c2db06f

Browse files
msukkarimsukkari
authored
Merge branch 'main' into conditional_db_init
2 parents 220dcc3 + 244afea commit c2db06f

File tree

4 files changed

+83
-60
lines changed

4 files changed

+83
-60
lines changed

‎docs/self-hosting/configuration.mdx‎

Lines changed: 13 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -10,19 +10,19 @@ Sourcebot accepts a variety of environment variables to fine tune your deploymen
1010

1111
| Variable | Default | Description |
1212
| :------- | :------ | :---------- |
13-
| `SOURCEBOT_LOG_LEVEL` | `info` | The Sourcebot logging level. Valid values are `debug`, `info`, `warn`, `error`, in order of severity. |
14-
| `DATABASE_URL` | `postgresql://postgres@ localhost:5432/sourcebot` | Connection string of your Postgres database. By default, a Postgres database is automatically provisioned at startup within the container. |
15-
| `REDIS_URL` | `redis://localhost:6379` | Connection string of your Redis instance. By default, a Redis database is automatically provisioned at startup within the container. |
16-
| `SOURCEBOT_ENCRYPTION_KEY` | - | Used to encrypt connection secrets. Generated using `openssl rand -base64 24`. Automatically generated at startup if no value is provided. |
17-
| `AUTH_SECRET` | - | Used to validate login session cookies. Generated using `openssl rand -base64 33`. Automatically generated at startup if no value is provided. |
18-
| `AUTH_URL` | - | URL of your Sourcebot deployment, e.g., `https://example.com` or `http://localhost:3000`. Required when `SOURCEBOT_AUTH_ENABLED` is `true`. |
19-
| `SOURCEBOT_TENANCY_MODE` | `single` | The tenancy configuration for Sourcebot. Valid values are `single` or `multi`. See [this doc](/self-hosting/more/tenancy) for more info. |
20-
| `SOURCEBOT_AUTH_ENABLED` | `false` | Enables/disables authentication in Sourcebot. If set to `false`, `SOURCEBOT_TENANCY_MODE` must be `single`. See [this doc](/self-hosting/more/authentication) for more info. |
21-
| `SOURCEBOT_TELEMETRY_DISABLED` | `false` | Enables/disables telemetry collection in Sourcebot. See [this doc](/self-hosting/security/telemetry) for more info. |
22-
| `DATA_DIR` | `/data` | The directory within the container to store all persistent data. Typically, this directory will be volume mapped such that data is persisted across container restarts (e.g., `docker run -v $(pwd):/data`) |
23-
| `DATA_CACHE_DIR` | `$DATA_DIR/.sourcebot` | The root data directory in which all data written to disk by Sourcebot will be located. |
24-
| `DATABASE_DATA_DIR` | `$DATA_CACHE_DIR/db` | The data directory for the default Postgres database. |
25-
| `REDIS_DATA_DIR` | `$DATA_CACHE_DIR/redis` | The data directory for the default Redis instance. |
13+
| `SOURCEBOT_LOG_LEVEL` | `info` | <p>The Sourcebot logging level. Valid values are `debug`, `info`, `warn`, `error`, in order of severity.</p> |
14+
| `DATABASE_URL` | `postgresql://postgres@ localhost:5432/sourcebot` | <p>Connection string of your Postgres database. By default, a Postgres database is automatically provisioned at startup within the container.</p><p>If you'd like to use a non-default schema, you can provide it as a parameter in the database url </p> |
15+
| `REDIS_URL` | `redis://localhost:6379` | <p>Connection string of your Redis instance. By default, a Redis database is automatically provisioned at startup within the container.</p> |
16+
| `SOURCEBOT_ENCRYPTION_KEY` | - | <p>Used to encrypt connection secrets. Generated using `openssl rand -base64 24`. Automatically generated at startup if no value is provided.</p> |
17+
| `AUTH_SECRET` | - | <p>Used to validate login session cookies. Generated using `openssl rand -base64 33`. Automatically generated at startup if no value is provided.</p> |
18+
| `AUTH_URL` | - | <p>URL of your Sourcebot deployment, e.g., `https://example.com` or `http://localhost:3000`. Required when `SOURCEBOT_AUTH_ENABLED` is `true`.</p> |
19+
| `SOURCEBOT_TENANCY_MODE` | `single` | <p>The tenancy configuration for Sourcebot. Valid values are `single` or `multi`. See [this doc](/self-hosting/more/tenancy) for more info.</p> |
20+
| `SOURCEBOT_AUTH_ENABLED` | `false` | <p>Enables/disables authentication in Sourcebot. If set to `false`, `SOURCEBOT_TENANCY_MODE` must be `single`. See [this doc](/self-hosting/more/authentication) for more info.</p> |
21+
| `SOURCEBOT_TELEMETRY_DISABLED` | `false` | <p>Enables/disables telemetry collection in Sourcebot. See [this doc](/self-hosting/security/telemetry) for more info.</p> |
22+
| `DATA_DIR` | `/data` | <p>The directory within the container to store all persistent data. Typically, this directory will be volume mapped such that data is persisted across container restarts (e.g., `docker run -v $(pwd):/data`)</p> |
23+
| `DATA_CACHE_DIR` | `$DATA_DIR/.sourcebot` | <p>The root data directory in which all data written to disk by Sourcebot will be located.</p> |
24+
| `DATABASE_DATA_DIR` | `$DATA_CACHE_DIR/db` | <p>The data directory for the default Postgres database.</p> |
25+
| `REDIS_DATA_DIR` | `$DATA_CACHE_DIR/redis` | <p>The data directory for the default Redis instance.</p> |
2626

2727

2828
## Additional Features

‎packages/backend/src/connectionManager.ts‎

Lines changed: 36 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ const QUEUE_NAME = 'connectionSyncQueue';
2020

2121
type JobPayload = {
2222
connectionId: number,
23+
connectionName: string,
2324
orgId: number,
2425
config: ConnectionConfig,
2526
};
@@ -60,12 +61,13 @@ export class ConnectionManager implements IConnectionManager {
6061

6162
await this.queue.add('connectionSyncJob', {
6263
connectionId: connection.id,
64+
connectionName: connection.name,
6365
orgId: connection.orgId,
6466
config: connectionConfig,
6567
});
66-
this.logger.info(`Added job to queue for connection ${connection.id}`);
68+
this.logger.info(`Added job to queue for connection ${connection.name} (id: ${connection.id})`);
6769
}).catch((err: unknown) => {
68-
this.logger.error(`Failed to add job to queue for connection ${connection.id}: ${err}`);
70+
this.logger.error(`Failed to add job to queue for connection ${connection.name} (id: ${connection.id}): ${err}`);
6971
});
7072
}
7173

@@ -83,14 +85,18 @@ export class ConnectionManager implements IConnectionManager {
8385
// (or if the date isn't set for some reason).
8486
{
8587
AND: [
86-
{ OR: [
87-
{ syncStatus: ConnectionSyncStatus.SYNCED },
88-
{ syncStatus: ConnectionSyncStatus.SYNCED_WITH_WARNINGS },
89-
]},
90-
{ OR: [
91-
{ syncedAt: null },
92-
{ syncedAt: { lt: thresholdDate } },
93-
]}
88+
{
89+
OR: [
90+
{ syncStatus: ConnectionSyncStatus.SYNCED },
91+
{ syncStatus: ConnectionSyncStatus.SYNCED_WITH_WARNINGS },
92+
]
93+
},
94+
{
95+
OR: [
96+
{ syncedAt: null },
97+
{ syncedAt: { lt: thresholdDate } },
98+
]
99+
}
94100
]
95101
}
96102
]
@@ -103,7 +109,7 @@ export class ConnectionManager implements IConnectionManager {
103109
}
104110

105111
private async runSyncJob(job: Job<JobPayload>): Promise<JobResult> {
106-
const { config, orgId } = job.data;
112+
const { config, orgId, connectionName } = job.data;
107113
// @note: We aren't actually doing anything with this atm.
108114
const abortController = new AbortController();
109115

@@ -120,7 +126,7 @@ export class ConnectionManager implements IConnectionManager {
120126
Sentry.captureException(e);
121127
throw e;
122128
}
123-
129+
124130
// Reset the syncStatusMetadata to an empty object at the start of the sync job
125131
await this.db.connection.update({
126132
where: {
@@ -131,7 +137,7 @@ export class ConnectionManager implements IConnectionManager {
131137
syncStatusMetadata: {}
132138
}
133139
})
134-
140+
135141

136142
let result: {
137143
repoData: RepoData[],
@@ -167,7 +173,7 @@ export class ConnectionManager implements IConnectionManager {
167173
}
168174
})();
169175
} catch (err) {
170-
this.logger.error(`Failed to compile repo data for connection ${job.data.connectionId}: ${err}`);
176+
this.logger.error(`Failed to compile repo data for connection ${job.data.connectionId} (${connectionName}): ${err}`);
171177
Sentry.captureException(err);
172178

173179
if (err instanceof BackendException) {
@@ -191,7 +197,7 @@ export class ConnectionManager implements IConnectionManager {
191197
syncStatusMetadata: { notFound }
192198
}
193199
});
194-
200+
195201
// Filter out any duplicates by external_id and external_codeHostUrl.
196202
repoData = repoData.filter((repo, index, self) => {
197203
return index === self.findIndex(r =>
@@ -218,7 +224,7 @@ export class ConnectionManager implements IConnectionManager {
218224
}
219225
});
220226
const deleteDuration = performance.now() - deleteStart;
221-
this.logger.info(`Deleted all RepoToConnection records for connection ${job.data.connectionId} in ${deleteDuration}ms`);
227+
this.logger.info(`Deleted all RepoToConnection records for connection ${connectionName} (id: ${job.data.connectionId}) in ${deleteDuration}ms`);
222228

223229
const totalUpsertStart = performance.now();
224230
for (const repo of repoData) {
@@ -235,10 +241,10 @@ export class ConnectionManager implements IConnectionManager {
235241
create: repo,
236242
})
237243
const upsertDuration = performance.now() - upsertStart;
238-
this.logger.info(`Upserted repo ${repo.external_id} in ${upsertDuration}ms`);
244+
this.logger.info(`Upserted repo ${repo.displayName} (id: ${repo.external_id}) in ${upsertDuration}ms`);
239245
}
240246
const totalUpsertDuration = performance.now() - totalUpsertStart;
241-
this.logger.info(`Upserted ${repoData.length} repos in ${totalUpsertDuration}ms`);
247+
this.logger.info(`Upserted ${repoData.length} repos for connection ${connectionName} (id: ${job.data.connectionId}) in ${totalUpsertDuration}ms`);
242248
}, { timeout: env.CONNECTION_MANAGER_UPSERT_TIMEOUT_MS });
243249

244250
return {
@@ -248,18 +254,20 @@ export class ConnectionManager implements IConnectionManager {
248254

249255

250256
private async onSyncJobCompleted(job: Job<JobPayload>, result: JobResult) {
251-
this.logger.info(`Connection sync job ${job.id} completed`);
257+
this.logger.info(`Connection sync job for connection ${job.data.connectionName} (id: ${job.data.connectionId}, jobId: ${job.id}) completed`);
252258
const { connectionId } = job.data;
253259

254260
let syncStatusMetadata: Record<string, unknown> = (await this.db.connection.findUnique({
255261
where: { id: connectionId },
256262
select: { syncStatusMetadata: true }
257263
}))?.syncStatusMetadata as Record<string, unknown> ?? {};
258-
const { notFound } = syncStatusMetadata as { notFound: {
259-
users: string[],
260-
orgs: string[],
261-
repos: string[],
262-
}};
264+
const { notFound } = syncStatusMetadata as {
265+
notFound: {
266+
users: string[],
267+
orgs: string[],
268+
repos: string[],
269+
}
270+
};
263271

264272
await this.db.connection.update({
265273
where: {
@@ -268,8 +276,8 @@ export class ConnectionManager implements IConnectionManager {
268276
data: {
269277
syncStatus:
270278
notFound.users.length > 0 ||
271-
notFound.orgs.length > 0 ||
272-
notFound.repos.length > 0 ? ConnectionSyncStatus.SYNCED_WITH_WARNINGS : ConnectionSyncStatus.SYNCED,
279+
notFound.orgs.length > 0 ||
280+
notFound.repos.length > 0 ? ConnectionSyncStatus.SYNCED_WITH_WARNINGS : ConnectionSyncStatus.SYNCED,
273281
syncedAt: new Date()
274282
}
275283
})
@@ -281,7 +289,7 @@ export class ConnectionManager implements IConnectionManager {
281289
}
282290

283291
private async onSyncJobFailed(job: Job<JobPayload> | undefined, err: unknown) {
284-
this.logger.info(`Connection sync job failed with error: ${err}`);
292+
this.logger.info(`Connection sync job for connection ${job?.data.connectionName} (id: ${job?.data.connectionId}, jobId: ${job?.id}) failed with error: ${err}`);
285293
Sentry.captureException(err, {
286294
tags: {
287295
connectionid: job?.data.connectionId,
@@ -312,7 +320,7 @@ export class ConnectionManager implements IConnectionManager {
312320
}
313321
} else {
314322
syncStatusMetadata = {
315-
...syncStatusMetadata,
323+
...syncStatusMetadata,
316324
error: 'UNKNOWN',
317325
}
318326
}

‎packages/backend/src/repoCompileUtils.ts‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,12 +6,15 @@ import { getGerritReposFromConfig } from "./gerrit.js";
66
import { Prisma, PrismaClient } from '@sourcebot/db';
77
import { WithRequired } from "./types.js"
88
import { marshalBool } from "./utils.js";
9+
import { createLogger } from './logger.js';
910
import { GerritConnectionConfig, GiteaConnectionConfig, GitlabConnectionConfig } from '@sourcebot/schemas/v3/connection.type';
1011
import { RepoMetadata } from './types.js';
1112
import path from 'path';
1213

1314
export type RepoData = WithRequired<Prisma.RepoCreateInput, 'connections'>;
1415

16+
const logger = createLogger('RepoCompileUtils');
17+
1518
export const compileGithubConfig = async (
1619
config: GithubConnectionConfig,
1720
connectionId: number,
@@ -39,6 +42,8 @@ export const compileGithubConfig = async (
3942
const repoName = path.join(repoNameRoot, repoDisplayName);
4043
const cloneUrl = new URL(repo.clone_url!);
4144

45+
logger.debug(`Found github repo ${repoDisplayName} with webUrl: ${repo.html_url}`);
46+
4247
const record: RepoData = {
4348
external_id: repo.id.toString(),
4449
external_codeHostType: 'github',
@@ -110,6 +115,8 @@ export const compileGitlabConfig = async (
110115
const repoDisplayName = project.path_with_namespace;
111116
const repoName = path.join(repoNameRoot, repoDisplayName);
112117

118+
logger.debug(`Found gitlab repo ${repoDisplayName} with webUrl: ${projectUrl}`);
119+
113120
const record: RepoData = {
114121
external_id: project.id.toString(),
115122
external_codeHostType: 'gitlab',
@@ -177,6 +184,8 @@ export const compileGiteaConfig = async (
177184
const repoDisplayName = repo.full_name!;
178185
const repoName = path.join(repoNameRoot, repoDisplayName);
179186

187+
logger.debug(`Found gitea repo ${repoDisplayName} with webUrl: ${repo.html_url}`);
188+
180189
const record: RepoData = {
181190
external_id: repo.id!.toString(),
182191
external_codeHostType: 'gitea',
@@ -246,11 +255,15 @@ export const compileGerritConfig = async (
246255
const webLink = project.web_links[0];
247256
const webUrl = webLink.url;
248257

258+
logger.debug(`Found gerrit repo ${project.name} with webUrl: ${webUrl}`);
259+
249260
// Handle case where webUrl is just a gitiles path
250261
// https://github.com/GerritCodeReview/plugins_gitiles/blob/5ee7f57/src/main/java/com/googlesource/gerrit/plugins/gitiles/GitilesWeblinks.java#L50
251262
if (webUrl.startsWith('/plugins/gitiles/')) {
263+
logger.debug(`WebUrl is a gitiles path, joining with hostUrl: ${webUrl}`);
252264
return path.join(hostUrl, webUrl);
253265
} else {
266+
logger.debug(`WebUrl is not a gitiles path, returning as is: ${webUrl}`);
254267
return webUrl;
255268
}
256269
})();

0 commit comments

Comments
 (0)