Skip to content

Commit c6f6ecc

Browse files
chenxin-yanchenxin-yan
authored
feat: add admin role and dashboard with app configs table (#36)
1 parent 79237b7 commit c6f6ecc

24 files changed

Lines changed: 1156 additions & 114 deletions

File tree

‎apps/scraper/package.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
"devDependencies": {
2828
"@biomejs/biome": "2.2.5",
2929
"@libsql/client": "^0.15.15",
30+
"convex": "^1.27.5",
3031
"drizzle-kit": "^0.31.5",
3132
"wrangler": "^4.42.1"
3233
}

‎apps/scraper/src/index.ts‎

Lines changed: 88 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ import type {
44
} from "@dev-team-fall-25/server/convex/http";
55
import { eq } from "drizzle-orm";
66
import { Hono } from "hono";
7-
import type z from "zod";
7+
import * as z from "zod/mini";
88
import getDB from "./drizzle";
99
import { errorLogs, jobs } from "./drizzle/schema";
1010
import { ConvexApi } from "./lib/convex";
@@ -20,40 +20,102 @@ app.get("/", async (c) => {
2020
return c.json({ status: "ok" });
2121
});
2222

23+
const ZCacheData = z.object({
24+
isMajorsEnabled: z.transform((val) => val === "true"),
25+
isCoursesEnabled: z.transform((val) => val === "true"),
26+
});
27+
2328
export default {
2429
fetch: app.fetch,
2530

2631
async scheduled(_event: ScheduledEvent, env: CloudflareBindings) {
27-
// NOTE: the worker will not execute anything for now until the flag for toggle scrapers are set up
28-
return;
29-
// biome-ignore lint/correctness/noUnreachable: WIP
3032
const db = getDB(env);
33+
const convex = new ConvexApi({
34+
baseUrl: env.CONVEX_SITE_URL,
35+
apiKey: env.CONVEX_API_KEY,
36+
});
3137

32-
// FIXME: need to handle when programsUr or coursesUrl is empty array
33-
const programsUrl = new URL("/programs", env.SCRAPING_BASE_URL).toString();
34-
const coursesUrl = new URL("/courses", env.SCRAPING_BASE_URL).toString();
35-
36-
const [[programsJob], [coursesJob]] = await Promise.all([
37-
db
38-
.insert(jobs)
39-
.values({
40-
url: programsUrl,
41-
jobType: "discover-programs",
42-
})
43-
.returning(),
44-
db
45-
.insert(jobs)
46-
.values({
47-
url: coursesUrl,
48-
jobType: "discover-courses",
49-
})
50-
.returning(),
51-
]);
38+
const cache = caches.default;
39+
const cacheKey = `${env.CONVEX_SITE_URL}/app-configs`;
40+
41+
let isMajorsEnabled = false;
42+
let isCoursesEnabled = false;
43+
44+
// Check to see if app configs are cached
45+
const cached = await cache.match(cacheKey);
46+
if (cached) {
47+
const { data, success } = ZCacheData.safeParse(await cached.json());
48+
49+
if (!success) {
50+
throw new JobError("Failed to parse cache data", "validation");
51+
}
52+
53+
isMajorsEnabled = data.isMajorsEnabled;
54+
isCoursesEnabled = data.isCoursesEnabled;
55+
} else {
56+
const [isScrapingMajors, isScrapingCourses] = await Promise.all([
57+
convex.getAppConfig({ key: "is_scraping_majors" }),
58+
convex.getAppConfig({ key: "is_scraping_courses" }),
59+
]);
60+
61+
isMajorsEnabled = isScrapingMajors === "true";
62+
isCoursesEnabled = isScrapingCourses === "true";
63+
64+
await cache.put(
65+
cacheKey,
66+
new Response(
67+
JSON.stringify({
68+
isScrapingMajors,
69+
isScrapingCourses,
70+
}),
71+
{
72+
headers: { "Cache-Control": "max-age=3600" },
73+
},
74+
),
75+
);
76+
}
77+
78+
const jobsToCreate: Array<{
79+
url: string;
80+
jobType: "discover-programs" | "discover-courses";
81+
}> = [];
82+
const flagsToDisable: string[] = [];
83+
84+
// add major discovery job to the queue
85+
if (isMajorsEnabled) {
86+
const programsUrl = new URL(
87+
"/programs",
88+
env.SCRAPING_BASE_URL,
89+
).toString();
90+
jobsToCreate.push({ url: programsUrl, jobType: "discover-programs" });
91+
flagsToDisable.push("is_scraping_majors");
92+
}
93+
94+
// add course discovery job to the queue
95+
if (isCoursesEnabled) {
96+
const coursesUrl = new URL("/courses", env.SCRAPING_BASE_URL).toString();
97+
jobsToCreate.push({ url: coursesUrl, jobType: "discover-courses" });
98+
flagsToDisable.push("is_scraping_courses");
99+
}
100+
101+
if (jobsToCreate.length === 0) {
102+
console.log("No scraping jobs enabled, skipping");
103+
return;
104+
}
105+
106+
const createdJobs = await db.insert(jobs).values(jobsToCreate).returning();
52107

53108
await Promise.all([
54-
env.SCRAPING_QUEUE.send({ jobId: programsJob.id }),
55-
env.SCRAPING_QUEUE.send({ jobId: coursesJob.id }),
109+
...createdJobs.map((job) => env.SCRAPING_QUEUE.send({ jobId: job.id })),
110+
...flagsToDisable.map((flag) =>
111+
convex.setAppConfig({ key: flag, value: "false" }),
112+
),
113+
cache.delete(cacheKey),
56114
]);
115+
116+
console.log(
117+
`Created ${createdJobs.length} jobs [${createdJobs.map((j) => j.jobType).join(", ")}], disabled flags: ${flagsToDisable.join(", ")}`,
118+
);
57119
},
58120

59121
async queue(

‎apps/scraper/src/lib/convex.ts‎

Lines changed: 50 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,14 @@
1+
import type { internal } from "@dev-team-fall-25/server/convex/_generated/api";
12
import {
3+
ZGetAppConfig,
4+
type ZSetAppConfig,
25
ZUpsertCourse,
36
ZUpsertCourseOffering,
47
ZUpsertPrerequisites,
58
ZUpsertProgram,
69
ZUpsertRequirements,
710
} from "@dev-team-fall-25/server/convex/http";
11+
import type { FunctionReturnType } from "convex/server";
812
import * as z from "zod/mini";
913
import { JobError } from "./queue";
1014

@@ -20,11 +24,11 @@ export class ConvexApi {
2024
this.config = config;
2125
}
2226

23-
private async request<T extends z.ZodMiniType>(
27+
private async request<R>(
2428
path: string,
25-
schema: T,
26-
data: z.infer<T>,
27-
): Promise<{ success: boolean; id?: string }> {
29+
schema: z.ZodMiniType,
30+
data: unknown,
31+
): Promise<{ data: R }> {
2832
const { data: validated, success, error } = schema.safeParse(data);
2933

3034
if (!success) {
@@ -54,47 +58,61 @@ export class ConvexApi {
5458
}
5559

5660
async upsertCourse(data: z.infer<typeof ZUpsertCourse>) {
57-
const result = await this.request(
58-
"/api/courses/upsert",
59-
ZUpsertCourse,
60-
data,
61-
);
62-
return result.id;
61+
const res = await this.request<
62+
FunctionReturnType<typeof internal.courses.upsertCourseInternal>
63+
>("/api/courses/upsert", ZUpsertCourse, data);
64+
return res.data;
6365
}
6466

6567
async upsertProgram(data: z.infer<typeof ZUpsertProgram>) {
66-
const result = await this.request(
67-
"/api/programs/upsert",
68-
ZUpsertProgram,
69-
data,
70-
);
71-
return result.id;
68+
const res = await this.request<
69+
FunctionReturnType<typeof internal.programs.upsertProgramInternal>
70+
>("/api/programs/upsert", ZUpsertProgram, data);
71+
return res.data;
7272
}
7373

7474
async upsertRequirements(data: z.infer<typeof ZUpsertRequirements>) {
75-
const result = await this.request(
76-
"/api/requirements/upsert",
77-
ZUpsertRequirements,
78-
data,
79-
);
80-
return result.id;
75+
const res = await this.request<
76+
FunctionReturnType<
77+
typeof internal.requirements.createRequirementsInternal
78+
>
79+
>("/api/requirements/upsert", ZUpsertRequirements, data);
80+
return res.data;
8181
}
8282

8383
async upsertPrerequisites(data: z.infer<typeof ZUpsertPrerequisites>) {
84-
const result = await this.request(
85-
"/api/prerequisites/upsert",
86-
ZUpsertPrerequisites,
87-
data,
88-
);
89-
return result.id;
84+
const res = await this.request<
85+
FunctionReturnType<
86+
typeof internal.prerequisites.createPrerequisitesInternal
87+
>
88+
>("/api/prerequisites/upsert", ZUpsertPrerequisites, data);
89+
return res.data;
9090
}
9191

9292
async upsertCourseOffering(data: z.infer<typeof ZUpsertCourseOffering>) {
93-
const result = await this.request(
94-
"/api/courseOfferings/upsert",
95-
ZUpsertCourseOffering,
93+
const res = await this.request<
94+
FunctionReturnType<
95+
typeof internal.courseOfferings.upsertCourseOfferingInternal
96+
>
97+
>("/api/courseOfferings/upsert", ZUpsertCourseOffering, data);
98+
return res.data;
99+
}
100+
101+
async getAppConfig(data: z.infer<typeof ZGetAppConfig>) {
102+
const res = await this.request<
103+
FunctionReturnType<typeof internal.appConfigs.getAppConfigInternal>
104+
>("api/appConfigs/get", ZGetAppConfig, data);
105+
return res.data;
106+
}
107+
108+
async setAppConfig(data: z.infer<typeof ZSetAppConfig>) {
109+
const res = await this.request<
110+
FunctionReturnType<typeof internal.appConfigs.setAppConfigInternal>
111+
>(
112+
"api/appConfigs/set",
113+
z.object({ key: z.string(), value: z.string() }),
96114
data,
97115
);
98-
return result.id;
116+
return res.data;
99117
}
100118
}

‎apps/web/package.json‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,10 +17,13 @@
1717
"@radix-ui/react-avatar": "^1.1.10",
1818
"@radix-ui/react-dialog": "^1.1.15",
1919
"@radix-ui/react-dropdown-menu": "^2.1.16",
20+
"@radix-ui/react-label": "^2.1.7",
2021
"@radix-ui/react-separator": "^1.1.7",
2122
"@radix-ui/react-slot": "^1.2.3",
2223
"@radix-ui/react-tooltip": "^1.2.8",
2324
"@t3-oss/env-nextjs": "^0.13.8",
25+
"@tanstack/react-form": "^1.23.7",
26+
"@tanstack/react-table": "^8.21.3",
2427
"class-variance-authority": "^0.7.1",
2528
"clsx": "^2.1.1",
2629
"convex": "^1.27.4",

0 commit comments

Comments
 (0)