Skip to content

Commit 5a670bf

Browse files
brendan-kellambrendan-kellam
committed
Add branch table to repos detail view
1 parent c16ef08 commit 5a670bf

12 files changed

Lines changed: 342 additions & 94 deletions

File tree

‎packages/backend/src/git.ts‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -278,6 +278,16 @@ export const getCommitHashForRefName = async ({
278278
refName: string,
279279
}) => {
280280
const git = createGitClientForPath(path);
281-
const rev = await git.revparse(refName);
282-
return rev;
281+
282+
try {
283+
// The `^{commit}` suffix is used to fully dereference the ref to a commit hash.
284+
const rev = await git.revparse(`${refName}^{commit}`);
285+
return rev;
286+
287+
// @note: Was hitting errors when the repository is empty,
288+
// so we're catching the error and returning undefined.
289+
} catch (error: unknown) {
290+
console.error(error);
291+
return undefined;
292+
}
283293
}

‎packages/backend/src/repoCompileUtils.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,12 +13,12 @@ import { marshalBool } from "./utils.js";
1313
import { createLogger } from '@sourcebot/logger';
1414
import { BitbucketConnectionConfig, GerritConnectionConfig, GiteaConnectionConfig, GitlabConnectionConfig, GenericGitHostConnectionConfig, AzureDevOpsConnectionConfig } from '@sourcebot/schemas/v3/connection.type';
1515
import { ProjectVisibility } from "azure-devops-node-api/interfaces/CoreInterfaces.js";
16-
import { RepoMetadata } from './types.js';
1716
import path from 'path';
1817
import { glob } from 'glob';
1918
import { getOriginUrl, isPathAValidGitRepoRoot, isUrlAValidGitRepo } from './git.js';
2019
import assert from 'assert';
2120
import GitUrlParse from 'git-url-parse';
21+
import { RepoMetadata } from '@sourcebot/shared';
2222

2323
export type RepoData = WithRequired<Prisma.RepoCreateInput, 'connections'>;
2424

‎packages/backend/src/repoIndexManager.ts‎

Lines changed: 69 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,18 @@
11
import * as Sentry from '@sentry/node';
22
import { PrismaClient, Repo, RepoIndexingJobStatus, RepoIndexingJobType } from "@sourcebot/db";
33
import { createLogger, Logger } from "@sourcebot/logger";
4+
import { repoMetadataSchema, RepoIndexingJobMetadata, repoIndexingJobMetadataSchema, RepoMetadata } from '@sourcebot/shared';
45
import { existsSync } from 'fs';
56
import { readdir, rm } from 'fs/promises';
67
import { Job, Queue, ReservedJob, Worker } from "groupmq";
78
import { Redis } from 'ioredis';
9+
import micromatch from 'micromatch';
810
import { INDEX_CACHE_DIR } from './constants.js';
911
import { env } from './env.js';
10-
import { cloneRepository, fetchRepository, getCommitHashForRefName, isPathAValidGitRepoRoot, unsetGitConfig, upsertGitConfig } from './git.js';
12+
import { cloneRepository, fetchRepository, getBranches, getCommitHashForRefName, getTags, isPathAValidGitRepoRoot, unsetGitConfig, upsertGitConfig } from './git.js';
13+
import { captureEvent } from './posthog.js';
1114
import { PromClient } from './promClient.js';
12-
import { repoMetadataSchema, RepoWithConnections, Settings } from "./types.js";
15+
import { RepoWithConnections, Settings } from "./types.js";
1316
import { getAuthCredentialsForRepo, getRepoPath, getShardPrefix, groupmqLifecycleExceptionWrapper, measure } from './utils.js';
1417
import { indexGitRepository } from './zoekt.js';
1518

@@ -61,7 +64,7 @@ export class RepoIndexManager {
6164
concurrency: this.settings.maxRepoIndexingJobConcurrency,
6265
...(env.DEBUG_ENABLE_GROUPMQ_LOGGING === 'true' ? {
6366
logger: true,
64-
}: {}),
67+
} : {}),
6568
});
6669

6770
this.worker.on('completed', this.onJobCompleted.bind(this));
@@ -263,7 +266,16 @@ export class RepoIndexManager {
263266

264267
try {
265268
if (jobType === RepoIndexingJobType.INDEX) {
266-
await this.indexRepository(repo, logger, abortController.signal);
269+
const revisions = await this.indexRepository(repo, logger, abortController.signal);
270+
271+
await this.db.repoIndexingJob.update({
272+
where: { id },
273+
data: {
274+
metadata: {
275+
indexedRevisions: revisions,
276+
} satisfies RepoIndexingJobMetadata,
277+
},
278+
});
267279
} else if (jobType === RepoIndexingJobType.CLEANUP) {
268280
await this.cleanupRepository(repo, logger);
269281
}
@@ -285,7 +297,7 @@ export class RepoIndexManager {
285297
// If the repo path exists but it is not a valid git repository root, this indicates
286298
// that the repository is in a bad state. To fix, we remove the directory and perform
287299
// a fresh clone.
288-
if (existsSync(repoPath) && !(await isPathAValidGitRepoRoot( { path: repoPath } ))) {
300+
if (existsSync(repoPath) && !(await isPathAValidGitRepoRoot({ path: repoPath }))) {
289301
const isValidGitRepo = await isPathAValidGitRepoRoot({
290302
path: repoPath,
291303
signal,
@@ -354,10 +366,54 @@ export class RepoIndexManager {
354366
});
355367
}
356368

369+
let revisions = [
370+
'HEAD'
371+
];
372+
373+
if (metadata.branches) {
374+
const branchGlobs = metadata.branches
375+
const allBranches = await getBranches(repoPath);
376+
const matchingBranches =
377+
allBranches
378+
.filter((branch) => micromatch.isMatch(branch, branchGlobs))
379+
.map((branch) => `refs/heads/${branch}`);
380+
381+
revisions = [
382+
...revisions,
383+
...matchingBranches
384+
];
385+
}
386+
387+
if (metadata.tags) {
388+
const tagGlobs = metadata.tags;
389+
const allTags = await getTags(repoPath);
390+
const matchingTags =
391+
allTags
392+
.filter((tag) => micromatch.isMatch(tag, tagGlobs))
393+
.map((tag) => `refs/tags/${tag}`);
394+
395+
revisions = [
396+
...revisions,
397+
...matchingTags
398+
];
399+
}
400+
401+
// zoekt has a limit of 64 branches/tags to index.
402+
if (revisions.length > 64) {
403+
logger.warn(`Too many revisions (${revisions.length}) for repo ${repo.id}, truncating to 64`);
404+
captureEvent('backend_revisions_truncated', {
405+
repoId: repo.id,
406+
revisionCount: revisions.length,
407+
});
408+
revisions = revisions.slice(0, 64);
409+
}
410+
357411
logger.info(`Indexing ${repo.name} (id: ${repo.id})...`);
358-
const { durationMs } = await measure(() => indexGitRepository(repo, this.settings, signal));
412+
const { durationMs } = await measure(() => indexGitRepository(repo, this.settings, revisions, signal));
359413
const indexDuration_s = durationMs / 1000;
360414
logger.info(`Indexed ${repo.name} (id: ${repo.id}) in ${indexDuration_s}s`);
415+
416+
return revisions;
361417
}
362418

363419
private async cleanupRepository(repo: Repo, logger: Logger) {
@@ -398,12 +454,18 @@ export class RepoIndexManager {
398454
path: repoPath,
399455
refName: 'HEAD',
400456
});
401-
457+
458+
const jobMetadata = repoIndexingJobMetadataSchema.parse(jobData.metadata);
459+
402460
const repo = await this.db.repo.update({
403461
where: { id: jobData.repoId },
404462
data: {
405463
indexedAt: new Date(),
406464
indexedCommitHash: commitHash,
465+
metadata: {
466+
...(jobData.repo.metadata as RepoMetadata),
467+
indexedRevisions: jobMetadata.indexedRevisions,
468+
} satisfies RepoMetadata,
407469
}
408470
});
409471

‎packages/backend/src/types.ts‎

Lines changed: 0 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -1,36 +1,8 @@
11
import { Connection, Repo, RepoToConnection } from "@sourcebot/db";
22
import { Settings as SettingsSchema } from "@sourcebot/schemas/v3/index.type";
3-
import { z } from "zod";
43

54
export type Settings = Required<SettingsSchema>;
65

7-
// Structure of the `metadata` field in the `Repo` table.
8-
//
9-
// @WARNING: If you modify this schema, please make sure it is backwards
10-
// compatible with any prior versions of the schema!!
11-
// @NOTE: If you move this schema, please update the comment in schema.prisma
12-
// to point to the new location.
13-
export const repoMetadataSchema = z.object({
14-
/**
15-
* A set of key-value pairs that will be used as git config
16-
* variables when cloning the repo.
17-
* @see: https://git-scm.com/docs/git-clone#Documentation/git-clone.txt-code--configcodecodeltkeygtltvaluegtcode
18-
*/
19-
gitConfig: z.record(z.string(), z.string()).optional(),
20-
21-
/**
22-
* A list of branches to index. Glob patterns are supported.
23-
*/
24-
branches: z.array(z.string()).optional(),
25-
26-
/**
27-
* A list of tags to index. Glob patterns are supported.
28-
*/
29-
tags: z.array(z.string()).optional(),
30-
});
31-
32-
export type RepoMetadata = z.infer<typeof repoMetadataSchema>;
33-
346
// @see : https://stackoverflow.com/a/61132308
357
export type DeepPartial<T> = T extends object ? {
368
[P in keyof T]?: DeepPartial<T[P]>;

‎packages/backend/src/zoekt.ts‎

Lines changed: 4 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -1,62 +1,16 @@
11
import { Repo } from "@sourcebot/db";
22
import { createLogger } from "@sourcebot/logger";
33
import { exec } from "child_process";
4-
import micromatch from "micromatch";
54
import { INDEX_CACHE_DIR } from "./constants.js";
6-
import { getBranches, getTags } from "./git.js";
7-
import { captureEvent } from "./posthog.js";
8-
import { repoMetadataSchema, Settings } from "./types.js";
5+
import { Settings } from "./types.js";
96
import { getRepoPath, getShardPrefix } from "./utils.js";
107

118
const logger = createLogger('zoekt');
129

13-
export const indexGitRepository = async (repo: Repo, settings: Settings, signal?: AbortSignal) => {
14-
let revisions = [
15-
'HEAD'
16-
];
17-
10+
export const indexGitRepository = async (repo: Repo, settings: Settings, revisions: string[], signal?: AbortSignal) => {
1811
const { path: repoPath } = getRepoPath(repo);
1912
const shardPrefix = getShardPrefix(repo.orgId, repo.id);
20-
const metadata = repoMetadataSchema.parse(repo.metadata);
21-
22-
if (metadata.branches) {
23-
const branchGlobs = metadata.branches
24-
const allBranches = await getBranches(repoPath);
25-
const matchingBranches =
26-
allBranches
27-
.filter((branch) => micromatch.isMatch(branch, branchGlobs))
28-
.map((branch) => `refs/heads/${branch}`);
29-
30-
revisions = [
31-
...revisions,
32-
...matchingBranches
33-
];
34-
}
35-
36-
if (metadata.tags) {
37-
const tagGlobs = metadata.tags;
38-
const allTags = await getTags(repoPath);
39-
const matchingTags =
40-
allTags
41-
.filter((tag) => micromatch.isMatch(tag, tagGlobs))
42-
.map((tag) => `refs/tags/${tag}`);
4313

44-
revisions = [
45-
...revisions,
46-
...matchingTags
47-
];
48-
}
49-
50-
// zoekt has a limit of 64 branches/tags to index.
51-
if (revisions.length > 64) {
52-
logger.warn(`Too many revisions (${revisions.length}) for repo ${repo.id}, truncating to 64`);
53-
captureEvent('backend_revisions_truncated', {
54-
repoId: repo.id,
55-
revisionCount: revisions.length,
56-
});
57-
revisions = revisions.slice(0, 64);
58-
}
59-
6014
const command = [
6115
'zoekt-git-index',
6216
'-allow_missing_branches',
@@ -76,7 +30,7 @@ export const indexGitRepository = async (repo: Repo, settings: Settings, signal?
7630
reject(error);
7731
return;
7832
}
79-
33+
8034
if (stdout) {
8135
stdout.split('\n').filter(line => line.trim()).forEach(line => {
8236
logger.info(line);
@@ -89,7 +43,7 @@ export const indexGitRepository = async (repo: Repo, settings: Settings, signal?
8943
logger.info(line);
9044
});
9145
}
92-
46+
9347
resolve({
9448
stdout,
9549
stderr

‎packages/db/prisma/migrations/20251024214005_add_metadata_field_to_repo_job_table/migration.sql‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
-- AlterTable
2+
ALTER TABLE "RepoIndexingJob" ADD COLUMN "metadata" JSONB;

‎packages/db/prisma/schema.prisma‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ model Repo {
3838
isFork Boolean
3939
isArchived Boolean
4040
isPublic Boolean @default(false)
41-
metadata Json /// For schema see repoMetadataSchema in packages/backend/src/types.ts
41+
metadata Json /// For schema see repoMetadataSchema in packages/shared/src/types.ts
4242
cloneUrl String
4343
webUrl String?
4444
connections RepoToConnection[]
@@ -84,6 +84,7 @@ model RepoIndexingJob {
8484
createdAt DateTime @default(now())
8585
updatedAt DateTime @updatedAt
8686
completedAt DateTime?
87+
metadata Json? /// For schema see repoIndexingJobMetadataSchema in packages/shared/src/types.ts
8788
8889
errorMessage String?
8990

‎packages/shared/src/index.server.ts‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,14 @@ export type {
99
Plan,
1010
Entitlement,
1111
} from "./entitlements.js";
12+
export type {
13+
RepoMetadata,
14+
RepoIndexingJobMetadata,
15+
} from "./types.js";
16+
export {
17+
repoMetadataSchema,
18+
repoIndexingJobMetadataSchema,
19+
} from "./types.js";
1220
export {
1321
base64Decode,
1422
loadConfig,

‎packages/shared/src/types.ts‎

Lines changed: 43 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,45 @@
11
import { Settings as SettingsSchema } from "@sourcebot/schemas/v3/index.type";
2+
import { z } from "zod";
23

3-
export type ConfigSettings = Required<SettingsSchema>;
4+
export type ConfigSettings = Required<SettingsSchema>;
5+
6+
// Structure of the `metadata` field in the `Repo` table.
7+
//
8+
// @WARNING: If you modify this schema, please make sure it is backwards
9+
// compatible with any prior versions of the schema!!
10+
// @NOTE: If you move this schema, please update the comment in schema.prisma
11+
// to point to the new location.
12+
export const repoMetadataSchema = z.object({
13+
/**
14+
* A set of key-value pairs that will be used as git config
15+
* variables when cloning the repo.
16+
* @see: https://git-scm.com/docs/git-clone#Documentation/git-clone.txt-code--configcodecodeltkeygtltvaluegtcode
17+
*/
18+
gitConfig: z.record(z.string(), z.string()).optional(),
19+
20+
/**
21+
* A list of branches to index. Glob patterns are supported.
22+
*/
23+
branches: z.array(z.string()).optional(),
24+
25+
/**
26+
* A list of tags to index. Glob patterns are supported.
27+
*/
28+
tags: z.array(z.string()).optional(),
29+
30+
/**
31+
* A list of revisions that were indexed for the repo.
32+
*/
33+
indexedRevisions: z.array(z.string()).optional(),
34+
});
35+
36+
export type RepoMetadata = z.infer<typeof repoMetadataSchema>;
37+
38+
export const repoIndexingJobMetadataSchema = z.object({
39+
/**
40+
* A list of revisions that were indexed for the repo.
41+
*/
42+
indexedRevisions: z.array(z.string()).optional(),
43+
});
44+
45+
export type RepoIndexingJobMetadata = z.infer<typeof repoIndexingJobMetadataSchema>;

0 commit comments

Comments
 (0)