Skip to content

Commit fd9973a

Browse files
brendan-kellambrendan-kellam
committed
wip on ondemand indexing
1 parent 6f22ffa commit fd9973a

5 files changed

Lines changed: 137 additions & 82 deletions

File tree

‎packages/backend/src/api.ts‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,8 @@ import z from 'zod';
77
import { ConnectionManager } from './connectionManager.js';
88
import { PromClient } from './promClient.js';
99
import { RepoIndexManager } from './repoIndexManager.js';
10+
import { createGitHubRepoRecord } from './repoCompileUtils.js';
11+
import { Octokit } from '@octokit/rest';
1012

1113
const logger = createLogger('api');
1214
const PORT = 3060;
@@ -33,6 +35,7 @@ export class Api {
3335

3436
app.post('/api/sync-connection', this.syncConnection.bind(this));
3537
app.post('/api/index-repo', this.indexRepo.bind(this));
38+
app.post(`/api/experimental/add-github-repo`, this.addGithubRepo.bind(this));
3639

3740
this.server = app.listen(PORT, () => {
3841
logger.info(`API server is running on port ${PORT}`);
@@ -92,6 +95,38 @@ export class Api {
9295
res.status(200).json({ jobId });
9396
}
9497

98+
private async addGithubRepo(req: Request, res: Response) {
99+
const schema = z.object({
100+
owner: z.string(),
101+
repo: z.string(),
102+
}).strict();
103+
104+
const parsed = schema.safeParse(req.body);
105+
if (!parsed.success) {
106+
res.status(400).json({ error: parsed.error.message });
107+
return;
108+
}
109+
110+
const octokit = new Octokit();
111+
const response = await octokit.rest.repos.get({
112+
owner: parsed.data.owner,
113+
repo: parsed.data.repo,
114+
});
115+
116+
const record = createGitHubRepoRecord({
117+
repo: response.data,
118+
hostUrl: 'https://github.com',
119+
});
120+
121+
const repo = await this.prisma.repo.create({
122+
data: record,
123+
});
124+
125+
const [jobId ] = await this.repoIndexManager.createJobs([repo], RepoIndexingJobType.INDEX);
126+
127+
res.status(200).json({ jobId });
128+
}
129+
95130
public async dispose() {
96131
return new Promise<void>((resolve, reject) => {
97132
this.server.close((err) => {

‎packages/backend/src/repoCompileUtils.ts‎

Lines changed: 70 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
import { GithubConnectionConfig } from '@sourcebot/schemas/v3/github.type';
2-
import { getGitHubReposFromConfig } from "./github.js";
2+
import { getGitHubReposFromConfig, OctokitRepository } from "./github.js";
33
import { getGitLabReposFromConfig } from "./gitlab.js";
44
import { getGiteaReposFromConfig } from "./gitea.js";
55
import { getGerritReposFromConfig } from "./gerrit.js";
@@ -62,60 +62,23 @@ export const compileGithubConfig = async (
6262
const warnings = gitHubReposResult.warnings;
6363

6464
const hostUrl = config.url ?? 'https://github.com';
65-
const repoNameRoot = new URL(hostUrl)
66-
.toString()
67-
.replace(/^https?:\/\//, '');
6865

6966
const repos = gitHubRepos.map((repo) => {
70-
const repoDisplayName = repo.full_name;
71-
const repoName = path.join(repoNameRoot, repoDisplayName);
72-
const cloneUrl = new URL(repo.clone_url!);
73-
const isPublic = repo.private === false;
74-
75-
logger.debug(`Found github repo ${repoDisplayName} with webUrl: ${repo.html_url}`);
67+
const record = createGitHubRepoRecord({
68+
repo,
69+
hostUrl,
70+
branches: config.revisions?.branches ?? undefined,
71+
tags: config.revisions?.tags ?? undefined,
72+
})
7673

77-
const record: RepoData = {
78-
external_id: repo.id.toString(),
79-
external_codeHostType: 'github',
80-
external_codeHostUrl: hostUrl,
81-
cloneUrl: cloneUrl.toString(),
82-
webUrl: repo.html_url,
83-
name: repoName,
84-
displayName: repoDisplayName,
85-
imageUrl: repo.owner.avatar_url,
86-
isFork: repo.fork,
87-
isArchived: !!repo.archived,
88-
isPublic: isPublic,
89-
org: {
90-
connect: {
91-
id: SINGLE_TENANT_ORG_ID,
92-
},
93-
},
74+
return {
75+
...record,
9476
connections: {
9577
create: {
9678
connectionId: connectionId,
9779
}
9880
},
99-
metadata: {
100-
gitConfig: {
101-
'zoekt.web-url-type': 'github',
102-
'zoekt.web-url': repo.html_url,
103-
'zoekt.name': repoName,
104-
'zoekt.github-stars': (repo.stargazers_count ?? 0).toString(),
105-
'zoekt.github-watchers': (repo.watchers_count ?? 0).toString(),
106-
'zoekt.github-subscribers': (repo.subscribers_count ?? 0).toString(),
107-
'zoekt.github-forks': (repo.forks_count ?? 0).toString(),
108-
'zoekt.archived': marshalBool(repo.archived),
109-
'zoekt.fork': marshalBool(repo.fork),
110-
'zoekt.public': marshalBool(isPublic),
111-
'zoekt.display-name': repoDisplayName,
112-
},
113-
branches: config.revisions?.branches ?? undefined,
114-
tags: config.revisions?.tags ?? undefined,
115-
} satisfies RepoMetadata,
11681
};
117-
118-
return record;
11982
})
12083

12184
return {
@@ -124,6 +87,67 @@ export const compileGithubConfig = async (
12487
};
12588
}
12689

90+
export const createGitHubRepoRecord = ({
91+
repo,
92+
hostUrl,
93+
branches,
94+
tags,
95+
}: {
96+
repo: OctokitRepository,
97+
hostUrl: string,
98+
branches?: string[],
99+
tags?: string[],
100+
}) => {
101+
const repoNameRoot = new URL(hostUrl)
102+
.toString()
103+
.replace(/^https?:\/\//, '');
104+
105+
const repoDisplayName = repo.full_name;
106+
const repoName = path.join(repoNameRoot, repoDisplayName);
107+
const cloneUrl = new URL(repo.clone_url!);
108+
const isPublic = repo.private === false;
109+
110+
logger.debug(`Found github repo ${repoDisplayName} with webUrl: ${repo.html_url}`);
111+
112+
const record: Prisma.RepoCreateInput = {
113+
external_id: repo.id.toString(),
114+
external_codeHostType: 'github',
115+
external_codeHostUrl: hostUrl,
116+
cloneUrl: cloneUrl.toString(),
117+
webUrl: repo.html_url,
118+
name: repoName,
119+
displayName: repoDisplayName,
120+
imageUrl: repo.owner.avatar_url,
121+
isFork: repo.fork,
122+
isArchived: !!repo.archived,
123+
isPublic: isPublic,
124+
org: {
125+
connect: {
126+
id: SINGLE_TENANT_ORG_ID,
127+
},
128+
},
129+
metadata: {
130+
gitConfig: {
131+
'zoekt.web-url-type': 'github',
132+
'zoekt.web-url': repo.html_url,
133+
'zoekt.name': repoName,
134+
'zoekt.github-stars': (repo.stargazers_count ?? 0).toString(),
135+
'zoekt.github-watchers': (repo.watchers_count ?? 0).toString(),
136+
'zoekt.github-subscribers': (repo.subscribers_count ?? 0).toString(),
137+
'zoekt.github-forks': (repo.forks_count ?? 0).toString(),
138+
'zoekt.archived': marshalBool(repo.archived),
139+
'zoekt.fork': marshalBool(repo.fork),
140+
'zoekt.public': marshalBool(isPublic),
141+
'zoekt.display-name': repoDisplayName,
142+
},
143+
branches,
144+
tags,
145+
} satisfies RepoMetadata,
146+
};
147+
148+
return record;
149+
}
150+
127151
export const compileGitlabConfig = async (
128152
config: GitlabConnectionConfig,
129153
connectionId: number): Promise<CompileResult> => {

‎packages/web/src/app/askgh/[owner]/[repo]/page.tsx‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
import { addGithubRepo } from "@/features/workerApi/actions";
2+
3+
interface PageProps {
4+
params: Promise<{ owner: string; repo: string }>;
5+
}
6+
7+
export default async function GitHubRepoPage(props: PageProps) {
8+
const params = await props.params;
9+
const { owner, repo } = params;
10+
11+
const response = await addGithubRepo(owner, repo);
12+
13+
return <p>{JSON.stringify(response, null, 2)}</p>;
14+
}
15+
16+

‎packages/web/src/features/workerApi/actions.ts‎

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22

33
import { sew } from "@/actions";
44
import { unexpectedError } from "@/lib/serviceError";
5-
import { withAuthV2, withMinimumOrgRole } from "@/withAuthV2";
5+
import { withAuthV2, withMinimumOrgRole, withOptionalAuthV2 } from "@/withAuthV2";
66
import { OrgRole } from "@sourcebot/db";
77
import z from "zod";
88

@@ -57,3 +57,18 @@ export const indexRepo = async (repoId: number) => sew(() =>
5757
})
5858
)
5959
);
60+
61+
export const addGithubRepo = async (owner: string, repo: string) => sew(() =>
62+
withOptionalAuthV2(async () => {
63+
const response = await fetch(`${WORKER_API_URL}/api/experimental/add-github-repo`, {
64+
method: 'POST',
65+
body: JSON.stringify({ owner, repo }),
66+
headers: {
67+
'Content-Type': 'application/json',
68+
},
69+
});
70+
71+
const data = await response.json();
72+
return data;
73+
})
74+
);

‎packages/web/src/middleware.ts‎

Lines changed: 0 additions & 35 deletions
This file was deleted.

0 commit comments

Comments
 (0)