Skip to content

Commit d6fc002

Browse files
committed
Queue remote reply scraping
Persisting remote posts now records replies collections as background scrape jobs instead of fetching and walking them synchronously. The job state is stored in PostgreSQL with per-origin tracking so worker nodes can process it later. Fixes #445 Assisted-by: Codex:gpt-5.5
1 parent 83b62e2 commit d6fc002

7 files changed

Lines changed: 4691 additions & 17 deletions

File tree

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
CREATE TYPE "public"."remote_reply_scrape_job_status" AS ENUM('pending', 'processing', 'completed', 'failed');--> statement-breakpoint
2+
CREATE TABLE "remote_reply_scrape_jobs" (
3+
"id" uuid PRIMARY KEY NOT NULL,
4+
"post_id" uuid NOT NULL,
5+
"post_iri" text NOT NULL,
6+
"replies_iri" text NOT NULL,
7+
"base_url" text NOT NULL,
8+
"origin_host" text NOT NULL,
9+
"depth" integer DEFAULT 0 NOT NULL,
10+
"status" "remote_reply_scrape_job_status" DEFAULT 'pending' NOT NULL,
11+
"attempts" integer DEFAULT 0 NOT NULL,
12+
"fetched_items" integer DEFAULT 0 NOT NULL,
13+
"next_attempt_at" timestamp with time zone DEFAULT CURRENT_TIMESTAMP NOT NULL,
14+
"error_message" text,
15+
"created" timestamp with time zone DEFAULT CURRENT_TIMESTAMP NOT NULL,
16+
"started_at" timestamp with time zone,
17+
"completed_at" timestamp with time zone,
18+
"updated" timestamp with time zone DEFAULT CURRENT_TIMESTAMP NOT NULL,
19+
CONSTRAINT "remote_reply_scrape_jobs_replies_iri_unique" UNIQUE("replies_iri")
20+
);
21+
--> statement-breakpoint
22+
CREATE TABLE "remote_reply_scrape_origins" (
23+
"origin_host" text PRIMARY KEY NOT NULL,
24+
"next_request_at" timestamp with time zone DEFAULT CURRENT_TIMESTAMP NOT NULL,
25+
"last_request_at" timestamp with time zone,
26+
"processing_job_id" uuid,
27+
"processing_started_at" timestamp with time zone,
28+
"updated" timestamp with time zone DEFAULT CURRENT_TIMESTAMP NOT NULL
29+
);
30+
--> statement-breakpoint
31+
ALTER TABLE "remote_reply_scrape_jobs" ADD CONSTRAINT "remote_reply_scrape_jobs_post_id_posts_id_fk" FOREIGN KEY ("post_id") REFERENCES "public"."posts"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
32+
CREATE INDEX "remote_reply_scrape_jobs_status_next_attempt_at_index" ON "remote_reply_scrape_jobs" USING btree ("status","next_attempt_at");--> statement-breakpoint
33+
CREATE INDEX "remote_reply_scrape_jobs_origin_host_status_next_attempt_at_index" ON "remote_reply_scrape_jobs" USING btree ("origin_host","status","next_attempt_at");--> statement-breakpoint
34+
CREATE INDEX "remote_reply_scrape_origins_next_request_at_index" ON "remote_reply_scrape_origins" USING btree ("next_request_at");--> statement-breakpoint
35+
CREATE INDEX "remote_reply_scrape_origins_processing_job_id_index" ON "remote_reply_scrape_origins" USING btree ("processing_job_id");

0 commit comments

Comments
 (0)