Skip to content

Commit 204fc82

Browse files
brendan-kellambrendan-kellam
authored
experiment(ask_gh): Support on demand indexing of GitHub repositories (#785)
1 parent 6f22ffa commit 204fc82

File tree

16 files changed

+481
-63
lines changed

16 files changed

+481
-63
lines changed

‎packages/backend/src/api.ts‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,9 @@ import z from 'zod';
77
import { ConnectionManager } from './connectionManager.js';
88
import { PromClient } from './promClient.js';
99
import { RepoIndexManager } from './repoIndexManager.js';
10+
import { createGitHubRepoRecord } from './repoCompileUtils.js';
11+
import { Octokit } from '@octokit/rest';
12+
import { SINGLE_TENANT_ORG_ID } from './constants.js';
1013

1114
const logger = createLogger('api');
1215
const PORT = 3060;
@@ -33,6 +36,7 @@ export class Api {
3336

3437
app.post('/api/sync-connection', this.syncConnection.bind(this));
3538
app.post('/api/index-repo', this.indexRepo.bind(this));
39+
app.post(`/api/experimental/add-github-repo`, this.experimental_addGithubRepo.bind(this));
3640

3741
this.server = app.listen(PORT, () => {
3842
logger.info(`API server is running on port ${PORT}`);
@@ -92,6 +96,47 @@ export class Api {
9296
res.status(200).json({ jobId });
9397
}
9498

99+
private async experimental_addGithubRepo(req: Request, res: Response) {
100+
const schema = z.object({
101+
owner: z.string(),
102+
repo: z.string(),
103+
}).strict();
104+
105+
const parsed = schema.safeParse(req.body);
106+
if (!parsed.success) {
107+
res.status(400).json({ error: parsed.error.message });
108+
return;
109+
}
110+
111+
const octokit = new Octokit();
112+
const response = await octokit.rest.repos.get({
113+
owner: parsed.data.owner,
114+
repo: parsed.data.repo,
115+
});
116+
117+
const record = createGitHubRepoRecord({
118+
repo: response.data,
119+
hostUrl: 'https://github.com',
120+
isAutoCleanupDisabled: true,
121+
});
122+
123+
const repo = await this.prisma.repo.upsert({
124+
where: {
125+
external_id_external_codeHostUrl_orgId: {
126+
external_id: record.external_id,
127+
external_codeHostUrl: record.external_codeHostUrl,
128+
orgId: SINGLE_TENANT_ORG_ID,
129+
}
130+
},
131+
update: record,
132+
create: record,
133+
});
134+
135+
const [jobId ] = await this.repoIndexManager.createJobs([repo], RepoIndexingJobType.INDEX);
136+
137+
res.status(200).json({ jobId, repoId: repo.id });
138+
}
139+
95140
public async dispose() {
96141
return new Promise<void>((resolve, reject) => {
97142
this.server.close((err) => {

‎packages/backend/src/repoCompileUtils.ts‎

Lines changed: 73 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
import { GithubConnectionConfig } from '@sourcebot/schemas/v3/github.type';
2-
import { getGitHubReposFromConfig } from "./github.js";
2+
import { getGitHubReposFromConfig, OctokitRepository } from "./github.js";
33
import { getGitLabReposFromConfig } from "./gitlab.js";
44
import { getGiteaReposFromConfig } from "./gitea.js";
55
import { getGerritReposFromConfig } from "./gerrit.js";
@@ -62,60 +62,23 @@ export const compileGithubConfig = async (
6262
const warnings = gitHubReposResult.warnings;
6363

6464
const hostUrl = config.url ?? 'https://github.com';
65-
const repoNameRoot = new URL(hostUrl)
66-
.toString()
67-
.replace(/^https?:\/\//, '');
6865

6966
const repos = gitHubRepos.map((repo) => {
70-
const repoDisplayName = repo.full_name;
71-
const repoName = path.join(repoNameRoot, repoDisplayName);
72-
const cloneUrl = new URL(repo.clone_url!);
73-
const isPublic = repo.private === false;
74-
75-
logger.debug(`Found github repo ${repoDisplayName} with webUrl: ${repo.html_url}`);
67+
const record = createGitHubRepoRecord({
68+
repo,
69+
hostUrl,
70+
branches: config.revisions?.branches ?? undefined,
71+
tags: config.revisions?.tags ?? undefined,
72+
})
7673

77-
const record: RepoData = {
78-
external_id: repo.id.toString(),
79-
external_codeHostType: 'github',
80-
external_codeHostUrl: hostUrl,
81-
cloneUrl: cloneUrl.toString(),
82-
webUrl: repo.html_url,
83-
name: repoName,
84-
displayName: repoDisplayName,
85-
imageUrl: repo.owner.avatar_url,
86-
isFork: repo.fork,
87-
isArchived: !!repo.archived,
88-
isPublic: isPublic,
89-
org: {
90-
connect: {
91-
id: SINGLE_TENANT_ORG_ID,
92-
},
93-
},
74+
return {
75+
...record,
9476
connections: {
9577
create: {
9678
connectionId: connectionId,
9779
}
9880
},
99-
metadata: {
100-
gitConfig: {
101-
'zoekt.web-url-type': 'github',
102-
'zoekt.web-url': repo.html_url,
103-
'zoekt.name': repoName,
104-
'zoekt.github-stars': (repo.stargazers_count ?? 0).toString(),
105-
'zoekt.github-watchers': (repo.watchers_count ?? 0).toString(),
106-
'zoekt.github-subscribers': (repo.subscribers_count ?? 0).toString(),
107-
'zoekt.github-forks': (repo.forks_count ?? 0).toString(),
108-
'zoekt.archived': marshalBool(repo.archived),
109-
'zoekt.fork': marshalBool(repo.fork),
110-
'zoekt.public': marshalBool(isPublic),
111-
'zoekt.display-name': repoDisplayName,
112-
},
113-
branches: config.revisions?.branches ?? undefined,
114-
tags: config.revisions?.tags ?? undefined,
115-
} satisfies RepoMetadata,
11681
};
117-
118-
return record;
11982
})
12083

12184
return {
@@ -124,6 +87,70 @@ export const compileGithubConfig = async (
12487
};
12588
}
12689

90+
export const createGitHubRepoRecord = ({
91+
repo,
92+
hostUrl,
93+
branches,
94+
tags,
95+
isAutoCleanupDisabled,
96+
}: {
97+
repo: OctokitRepository,
98+
hostUrl: string,
99+
branches?: string[],
100+
tags?: string[],
101+
isAutoCleanupDisabled?: boolean,
102+
}) => {
103+
const repoNameRoot = new URL(hostUrl)
104+
.toString()
105+
.replace(/^https?:\/\//, '');
106+
107+
const repoDisplayName = repo.full_name;
108+
const repoName = path.join(repoNameRoot, repoDisplayName);
109+
const cloneUrl = new URL(repo.clone_url!);
110+
const isPublic = repo.private === false;
111+
112+
logger.debug(`Found github repo ${repoDisplayName} with webUrl: ${repo.html_url}`);
113+
114+
const record: Prisma.RepoCreateInput = {
115+
external_id: repo.id.toString(),
116+
external_codeHostType: 'github',
117+
external_codeHostUrl: hostUrl,
118+
cloneUrl: cloneUrl.toString(),
119+
webUrl: repo.html_url,
120+
name: repoName,
121+
displayName: repoDisplayName,
122+
imageUrl: repo.owner.avatar_url,
123+
isFork: repo.fork,
124+
isArchived: !!repo.archived,
125+
isPublic: isPublic,
126+
isAutoCleanupDisabled,
127+
org: {
128+
connect: {
129+
id: SINGLE_TENANT_ORG_ID,
130+
},
131+
},
132+
metadata: {
133+
gitConfig: {
134+
'zoekt.web-url-type': 'github',
135+
'zoekt.web-url': repo.html_url,
136+
'zoekt.name': repoName,
137+
'zoekt.github-stars': (repo.stargazers_count ?? 0).toString(),
138+
'zoekt.github-watchers': (repo.watchers_count ?? 0).toString(),
139+
'zoekt.github-subscribers': (repo.subscribers_count ?? 0).toString(),
140+
'zoekt.github-forks': (repo.forks_count ?? 0).toString(),
141+
'zoekt.archived': marshalBool(repo.archived),
142+
'zoekt.fork': marshalBool(repo.fork),
143+
'zoekt.public': marshalBool(isPublic),
144+
'zoekt.display-name': repoDisplayName,
145+
},
146+
branches,
147+
tags,
148+
} satisfies RepoMetadata,
149+
};
150+
151+
return record;
152+
}
153+
127154
export const compileGitlabConfig = async (
128155
config: GitlabConnectionConfig,
129156
connectionId: number): Promise<CompileResult> => {

‎packages/backend/src/repoIndexManager.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -160,6 +160,7 @@ export class RepoIndexManager {
160160
connections: {
161161
none: {}
162162
},
163+
isAutoCleanupDisabled: false,
163164
OR: [
164165
{ indexedAt: null },
165166
{ indexedAt: { lt: gcGracePeriodMs } },

‎packages/db/prisma/migrations/20260121012745_add_is_auto_cleanup_disabled_flag_to_repo_table/migration.sql‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
-- AlterTable
2+
ALTER TABLE "Repo" ADD COLUMN "isAutoCleanupDisabled" BOOLEAN NOT NULL DEFAULT false;

‎packages/db/prisma/schema.prisma‎

Lines changed: 14 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -45,19 +45,20 @@ enum CodeHostType {
4545
}
4646

4747
model Repo {
48-
id Int @id @default(autoincrement())
49-
name String /// Full repo name, including the vcs hostname (ex. github.com/sourcebot-dev/sourcebot)
50-
displayName String? /// Display name of the repo for UI (ex. sourcebot-dev/sourcebot)
51-
createdAt DateTime @default(now())
52-
updatedAt DateTime @updatedAt
53-
isFork Boolean
54-
isArchived Boolean
55-
isPublic Boolean @default(false)
56-
metadata Json /// For schema see repoMetadataSchema in packages/shared/src/types.ts
57-
cloneUrl String
58-
webUrl String?
59-
connections RepoToConnection[]
60-
imageUrl String?
48+
id Int @id @default(autoincrement())
49+
name String /// Full repo name, including the vcs hostname (ex. github.com/sourcebot-dev/sourcebot)
50+
displayName String? /// Display name of the repo for UI (ex. sourcebot-dev/sourcebot)
51+
createdAt DateTime @default(now())
52+
updatedAt DateTime @updatedAt
53+
isFork Boolean
54+
isArchived Boolean
55+
isPublic Boolean @default(false)
56+
isAutoCleanupDisabled Boolean @default(false) /// If true, automatic cleanup of this repo when it becomes orphaned will be disabled.
57+
metadata Json /// For schema see repoMetadataSchema in packages/shared/src/types.ts
58+
cloneUrl String
59+
webUrl String?
60+
connections RepoToConnection[]
61+
imageUrl String?
6162
6263
permittedAccounts AccountToRepoPermission[]
6364
permissionSyncJobs RepoPermissionSyncJob[]

‎packages/shared/src/env.server.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -196,6 +196,7 @@ export const env = createEnv({
196196
// @NOTE: Take care to update actions.ts when changing the name of this.
197197
EXPERIMENT_SELF_SERVE_REPO_INDEXING_GITHUB_TOKEN: z.string().optional(),
198198
EXPERIMENT_EE_PERMISSION_SYNC_ENABLED: booleanSchema.default('false'),
199+
EXPERIMENT_ASK_GH_ENABLED: booleanSchema.default('false'),
199200

200201
SOURCEBOT_ENCRYPTION_KEY: z.string(),
201202
SOURCEBOT_INSTALL_ID: z.string().default("unknown"),

‎packages/web/src/app/[domain]/askgh/[owner]/[repo]/api.ts‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
import 'server-only';
2+
3+
import { sew } from '@/actions';
4+
import { notFound, ServiceError } from '@/lib/serviceError';
5+
import { withOptionalAuthV2 } from '@/withAuthV2';
6+
import { RepoInfo } from './types';
7+
8+
export const getRepoInfo = async (repoId: number): Promise<RepoInfo | ServiceError> => sew(() =>
9+
withOptionalAuthV2(async ({ prisma }) => {
10+
const repo = await prisma.repo.findUnique({
11+
where: { id: repoId },
12+
include: {
13+
jobs: {
14+
orderBy: {
15+
createdAt: 'desc',
16+
},
17+
take: 1,
18+
},
19+
},
20+
});
21+
22+
if (!repo) {
23+
return notFound();
24+
}
25+
26+
return {
27+
id: repo.id,
28+
name: repo.name,
29+
displayName: repo.displayName,
30+
imageUrl: repo.imageUrl,
31+
isIndexed: repo.indexedAt !== null,
32+
};
33+
})
34+
)

0 commit comments

Comments
 (0)