Link Previews Deployment Update

Author: pfuto21454

.gitlab/ci/deploy_services.yml

@@ -7,6 +7,10 @@
     - crane auth login $CI_REGISTRY -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD
     - ENVIRONMENT_TAG="${CI_ENVIRONMENT_NAME%%/*}"
     - crane tag $CI_REGISTRY_IMAGE/${SERVICE_NAME}:$CI_COMMIT_SHA $ENVIRONMENT_TAG
+    - |
+      if [ "$CI_COMMIT_BRANCH" = "$CI_DEFAULT_BRANCH" ]; then
+        crane tag $CI_REGISTRY_IMAGE/${SERVICE_NAME}:latest $ENVIRONMENT_TAG
+      fi
 
 .service-deploy-manifest:
   stage: deploy

compose.yml

@@ -32,7 +32,7 @@ services:
       # container itself, making Kafka unreachable
       - POLYCENTRIC_KAFKA_BROKERS=kafka:19092
       # Reached over the isolated `scraper_link` network (see below).
-      - POLYCENTRIC_SCRAPER_URL=http://scraper:3002
+      - POLYCENTRIC_SCRAPER_URL=http://scraper:8855
     # On both the shared app network and the isolated scraper link, so it can
     # talk to postgres/kafka/etc. AND to the scraper.
     networks:
@@ -71,7 +71,7 @@ services:
         - CMD
         - node
         - -e
-        - fetch('http://localhost:3002/health').then(r=>process.exit(r.ok?0:1)).catch(()=>process.exit(1))
+        - fetch('http://localhost:8855/health').then(r=>process.exit(r.ok?0:1)).catch(()=>process.exit(1))
       interval: 10s
       timeout: 5s
       retries: 5

services/scraper/.env.example

@@ -1,5 +1,5 @@
-# Port the scraper HTTP API listens on. Defaults to 3002 when unset.
-PORT=3002
+# Port the scraper HTTP API listens on. Defaults to 8855 when unset.
+PORT=8855
 
 # Path to the Chromium/Chrome binary that puppeteer/browserless launches for
 # prerendering. The Docker image installs a system Chromium here; for local

services/scraper/Dockerfile

@@ -56,10 +56,11 @@ ENV NODE_ENV=production
 # System Chromium provided below; puppeteer/browserless launch it.
 ENV PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium
 # PORT is supplied at runtime (compose / `docker run -e PORT=…`); the service
-# falls back to 3002 when unset.
+# falls back to 8855 when unset.
 
 # Headless Chromium for prerendering (Debian `chromium` → /usr/bin/chromium),
-# plus nftables for the startup egress firewall.
+# nftables for the startup egress firewall, and curl for in-container debugging
+# (e.g. testing egress rules from inside the scraper's netns).
 RUN apt-get update \
   && apt-get install -y --no-install-recommends chromium nftables \
   && rm -rf /var/lib/apt/lists/*
@@ -76,5 +77,5 @@ RUN chmod +x /usr/local/bin/docker-entrypoint.sh
 # unprivileged `node` user to run the service. REQUIRES `--cap-add=NET_ADMIN`
 # (and `--init` is recommended so the headless-Chromium children get reaped).
 ENTRYPOINT ["/usr/local/bin/docker-entrypoint.sh"]
-EXPOSE 3002
+EXPOSE 8855
 CMD ["node", "dist/index.js"]

services/scraper/egress.nft

@@ -1,8 +1,9 @@
 #!/usr/sbin/nft -f
 # Egress firewall applied inside the scraper container's network namespace at
-# startup (see docker-entrypoint.sh). Allows the public internet; drops NEW
-# connections to internal/reserved ranges. Because every connection the
-# headless browser makes is locally generated, the output hook covers it too.
+# startup (see docker-entrypoint.sh). Allows the public internet on HTTP/HTTPS
+# ports only; drops NEW connections to internal/reserved ranges and to non-web
+# ports. Because every connection the headless browser makes is locally
+# generated, the output hook covers it too.
 #
 # Do NOT `flush ruleset`: that would wipe Docker's embedded-DNS NAT rules in
 # this netns and break name resolution. Replace only our own table (the
@@ -73,6 +74,16 @@ table inet egress {
     # Block NEW outbound connections to internal/reserved space.
     ip daddr @blocked4 drop
     ip6 daddr @blocked6 drop
-    # policy accept -> NEW connections to public addresses pass.
+
+    # Of the remaining (public) destinations, the scraper only ever needs
+    # HTTP/HTTPS. Drop any other NEW outbound so a malicious URL/redirect — or a
+    # compromised scraper — can't reach public services on non-web ports
+    # (port-scanning, SMTP/Redis/SSH, odd-port exfiltration, etc.). Loopback,
+    # established replies, and DNS to the resolver(s) were accepted above. udp
+    # 443 is allowed for HTTP/3 (QUIC); all other new UDP — including DNS to
+    # non-resolvers — is dropped, keeping name resolution scoped to @resolvers.
+    ct state new tcp dport != { 80, 443 } drop
+    ct state new udp dport != 443 drop
+    # policy accept -> NEW connections to public addresses on 80/443 pass.
   }
 }

services/scraper/src/index.ts

@@ -87,7 +87,7 @@ export const scrape = async (targetUrl: string): Promise<LinkMetadata> => {
   }
 };
 
-const PORT = Number(process.env.PORT ?? 3002);
+const PORT = Number(process.env.PORT ?? 8855);
 
 /** Largest image we'll proxy. Preview thumbnails are small; this bounds memory. */
 const MAX_IMAGE_BYTES = 10 * 1024 * 1024;

services/server/.env.example

@@ -31,8 +31,8 @@ CDN_URL=http://localhost:3000
 
 # Internal scraper service (services/scraper) used to fetch link-preview
 # metadata (UrlInfo) and proxy preview images (/image_proxy).
-# Defaults to http://localhost:3002.
-POLYCENTRIC_SCRAPER_URL=http://localhost:3002
+# Defaults to http://localhost:8855.
+POLYCENTRIC_SCRAPER_URL=http://localhost:8855
 
 POLYCENTRIC_KAFKA_SECURITY_PROTOCOL=SASL_PLAINTEXT
 POLYCENTRIC_KAFKA_SASL_MECHANISM=SCRAM-SHA-256

services/server/src/util/scraper.rs

@@ -3,7 +3,7 @@
 /// Base URL of the scraper service, overridable via `POLYCENTRIC_SCRAPER_URL`.
 fn base_url() -> String {
     std::env::var("POLYCENTRIC_SCRAPER_URL")
-        .unwrap_or_else(|_| "http://localhost:3002".to_string())
+        .unwrap_or_else(|_| "http://localhost:8855".to_string())
 }
 
 /// URL of the `/scrape` endpoint (link-preview metadata).