Skip to content

Commit fac1538

Browse files
isTravisisTravis
authored
feat: Mirror config (#9)
1 parent 0860309 commit fac1538

21 files changed

Lines changed: 4967 additions & 28 deletions

‎.env.enc‎

Lines changed: 18 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,22 +1,24 @@
1-
#ENC[AES256_GCM,data:5YN6f+XIQasY,iv:70tE+1xTurmic6fcjiuDCVovEbYVhDbp2lvbONhNW5A=,tag:LkAy0bLDgDhNgwrGdqmp4Q==,type:comment]
2-
DATABASE_URL=ENC[AES256_GCM,data:/28bqrOFc9GcQH4Dv4spSwNgD71BzEOzm8rKN7nlwY9qQckHL76YOvBHNToU0sZqC9v5DT4=,iv:F4oswhOvlREkllYVooBIELUjM4WECckSwdRDxcu6E8I=,tag:V7+Hxt0Veij5+52PdyvSAQ==,type:str]
3-
SESSION_SECRET=ENC[AES256_GCM,data:MirMFVzowHq3jnaGrNPe82EKONv5XOppyhg3BI7R3h4=,iv:ehkoryGJcEL2c830LD9hZt/Uf6U3brbZlaJPNyJLiVE=,tag:oP/na2LsYuQhc4vGDc4ACw==,type:str]
4-
APP_PORT=ENC[AES256_GCM,data:o8oBKw==,iv:9lXQy+YaVb3um42QaHKKH2EXA6p9VzgX4cF4yZq17qE=,tag:QK+7vAQw4Q7qHk113pM6Qw==,type:str]
5-
API_PORT=ENC[AES256_GCM,data:QNqN5Q==,iv:3ChZ+9fWEUUFab4f8MFanSJcrRLEMO1yaCy+tCajWH8=,tag:xyA9zZepwJFt1pWvml67Eg==,type:str]
6-
#ENC[AES256_GCM,data:MrMg,iv:hdJ+cS1GtNPfpcsDoAI1TdjRW4tQTDhxhD59VFKlC3w=,tag:00P1/zjgi4mUztzvc8WXRw==,type:comment]
7-
S3_BUCKET=ENC[AES256_GCM,data:x6ZvFtFlBn4oGjSjBCbJ,iv:Fs+ehEL+6ZJH0LFz8RL2Vlr3PTSNTRhLOrqTCFMJchY=,tag:8V95MpTnaj6GbTn3YEYBsw==,type:str]
8-
S3_ENDPOINT=ENC[AES256_GCM,data:cyqC2I4kUYrdGw/9OTv88LQptYbKWzDrgc3Bbz7ag1GmzdWBPS/CAf7mIkDm/X2qWyecEg3MvNvYjCM8KKXDEfE=,iv:MKalw/HDyJ5au6Q7bia6pr7aUc3UQK+QuyuVXfctTI8=,tag:ieN8kXP02mQK/D+aqTS/Ng==,type:str]
9-
S3_ACCESS_KEY=ENC[AES256_GCM,data:L0XFIxbE/yu45PhMh5pIfc2PDun6t+hppD2mb2DQVno=,iv:fCDNYLqpW1gblgQjRSWxNtDpH3Wvk/bF4AZ+SX/qSCg=,tag:J4DA77pMJbAKUOphfyGtww==,type:str]
10-
S3_SECRET_KEY=ENC[AES256_GCM,data:T+uXZRXKFhwkkeEAzGtfNaeQbPKQrQmLgo4ekWMSR84rcMh3T8mul/Fmr6zQai1bvYRDR/RqXjVB8+grN33NtA==,iv:h0AG9TD66pZP5xGJmavBuH03nHbVz+t75ePqUP/M9O4=,tag:6iCaK84XTQ4sD9o0DjuvKg==,type:str]
11-
sops_age__list_0__map_enc=-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBuUGFGMFFlT04wMmVSNHpm\ncUJTREVQeUsxM25hT3VOQWRTY3hMa2Q0U0VNCm9NeFMwVWRtdDVXT3l0dnRaUUNT\nTXZMZXBTWEozMCszMjQwQXp2SXYzTGMKLS0tICtWbWdZNUFEZVNXQlhlWnRkdWx0\nQjU5R0k4cVZ0akJiOTFhdnYwSVVFZHMKDbZsZkRamBqEZoKh1GqeTEETECW0FNwW\ne5jgJjK0t/uLa2GASecSGI0ONRM4n7J50jsBdtmb6lqaAgY3mDQWbA==\n-----END AGE ENCRYPTED FILE-----\n
1+
#ENC[AES256_GCM,data:OTvS9wE/1Y8D,iv:TH6FV7DEWuEFrHRUbPxWDn2wQH91kRdR7GiiIIbXZXM=,tag:mFIxqN+HaHJFBXmV9HwhmQ==,type:comment]
2+
DATABASE_URL=ENC[AES256_GCM,data:peVeoz3PtbZ/FDYyiYUYeGd12nv+CehcxL+PZPnCDTH0m2jC8hUoK7ORyX24gget6nmGYRI=,iv:dm3yToRlLYxawPyqukiAqQ5pL7SZzMUeC/UqLpc8yfU=,tag:w+O5b+8mpL0mUL1+QgEEjw==,type:str]
3+
SESSION_SECRET=ENC[AES256_GCM,data:UszhKBt+K7a9Rvm0RCb32XIPewEpavJgITzdvihqbM4=,iv:yJTgIzaZKSmYR2H8y5ZbtqGNj5WrG+MIYvLVqhQbCGI=,tag:9Vg0EbMoj7mkq3FEQ31/5g==,type:str]
4+
APP_PORT=ENC[AES256_GCM,data:Vb96ZA==,iv:y/sUKEViCx0lX6kAzrpiD8yICK+bOVA+5xF6yzdNneQ=,tag:bygtiSSg3NCc4tlULR3Mlg==,type:str]
5+
API_PORT=ENC[AES256_GCM,data:cimBEw==,iv:niHYZoy2P/an06znHc6M3P0sySyBdZRhnzanirhMbyE=,tag:7HIoxyhPNwuTyywt4cZMJA==,type:str]
6+
#ENC[AES256_GCM,data:nP8u,iv:kuXejWrfQYWksTiLT2YQ07WfsrRU6/UimFBCjgPG8iU=,tag:bdyvKXmUUrgjKKnQVcks5w==,type:comment]
7+
S3_BUCKET=ENC[AES256_GCM,data:MzahjK+0uTqAw/WhA8Se,iv:xLvpcYYV/1frYiTnwrOjHo3b8TCjgpgQ5QRLg1lhAc0=,tag:+DfZ2lWTi6S1lcJVJ+XjfQ==,type:str]
8+
S3_ENDPOINT=ENC[AES256_GCM,data:ybBuRodzg5hI5fa/iWmBkzCVimT+JHinLT3/6cZ2AIEWPWRcWeQcug4A+MDoWasYsWt9w2lRSHfxS355Pr3l4wA=,iv:9n/By1VXaIPODj/R5KqZgNEz1RUirJh+cHvvLci8344=,tag:aVx8J2V7+PMOvkqhYm79FQ==,type:str]
9+
S3_ACCESS_KEY=ENC[AES256_GCM,data:rOdXiMeBLu/3K8il8ayx9Rur3IlbrjQ4FnRAKKIMPgU=,iv:S5WiEHRCf3dBn2aVLCglnfc62qkLwyRoM+6QzoRWTTw=,tag:AaVNv6uSuKMbS2xdYpK35Q==,type:str]
10+
S3_SECRET_KEY=ENC[AES256_GCM,data:Feenm4pAlv8eglOUaU/1aPIf81WzdNzoqprB87jxzEfLJPc1BxKab7AqMaJOqQqOpQ7hNrK+I2dr01gFv6x/SA==,iv:kNTslAmOSNH3wSKtnE2qg1pIoASKUnZh6EW/kdrTxiQ=,tag:VdpCsHZ75yW0kDVaNxhMnw==,type:str]
11+
#ENC[AES256_GCM,data:z6C0FEY22W4af3lK1mx1BE3j,iv:BGNFYqhy/ySTaL9y9Bth4WqaaqUOTjDr6PigI2+zljc=,tag:C1+HDy9Vfopz3cwJn+NDFg==,type:comment]
12+
UNDERLAY_UPSTREAM_API_KEY=ENC[AES256_GCM,data:21NDl9yy4rvWiZOLBh6U5HrgzsAq5B2firmYYUxr9XCSemM=,iv:C4HYm3IF6cZEo5ZicNvCIcXo5qARHWKkZ4vwVTkn0Jk=,tag:GVB3atYFyk1p4Yr9j7u1Kg==,type:str]
13+
sops_age__list_0__map_enc=-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBsenFsVmhwbU1rc3ZJWTAw\nZTl5L0xKYmJaMVVyTEZFb2pmdm5jVlV4bUZvCldtVlg5czFrdndPZmxIYVBCb3FC\nM21SY2JYTkxoQ214dGRlNitWcGhOcHcKLS0tIDJtcmVRSEs4amlraURZVjlnS3dr\nQnBaZHVTeElrTFNINGo4aVNRSmxPYVEKsiCgHDdHEyg1dK4+liXapmPIov3Twl3p\nL3JuNxYs7D2KUdvsxYjpl0gd9LKHo8BnR4x4ltOIJZS84NQEDyyE9A==\n-----END AGE ENCRYPTED FILE-----\n
1214
sops_age__list_0__map_recipient=age1wravpjmed26772xfjhawmnsnc4933htapg6y5xseqml0jdv8z9hqemzhcr
13-
sops_age__list_1__map_enc=-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBSYnhoQ3hOL1V3TWVERmFy\nQ1hFNjYrK1RiS0R0T3RSbDJqRzYrcENLUTFvCnRhSmdIMCtvbkEyWlBLbkxGaFFS\nR3RiY2FnMzVRaVpXaHRYT0dITmZIMUkKLS0tIGRpRDBXSGlVRUdlYVNMeW90NGdS\nMWxuT01hRG4zWnFoOTFkYjlPNFVIRFEKc47+5w5UkcZn/cDQhoW1EqCqJ81fSU9s\nhmP2+FMkXl2iC8DvMXxvmv54lr0yVK4DH6g9NxpVR4ai/dWdEjvl9w==\n-----END AGE ENCRYPTED FILE-----\n
15+
sops_age__list_1__map_enc=-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBON0xxSDVqQ1hJU2NxWTRY\na1pFcmdPWE1EQ29oZDNiMXNjMzVHYmxGaVU0CjAvNTMrTzhGSndBWGVKeHNJM0hR\nckhWN0V0ZkgxcEFmNmNoYWI1aDZ1L0kKLS0tIHIrdjBBU25VWEh6SG5kZVNNTFdQ\nRDVuYjIwTUdyRHJRQU1wVExGVFJOU00KhSPc1Jd8ujgNKbddWNl5sOi4h+1t5/zw\n3Bd9k3ZZd+saU/PfOrxh9QUbR4RXS85sRGOPH54tIYhH8QoffthtJg==\n-----END AGE ENCRYPTED FILE-----\n
1416
sops_age__list_1__map_recipient=age1ysddqggsx3h8zkv7xn3z26sjak5pqms6pyqhnky9ukrvpk7es5jsayz8w7
15-
sops_age__list_2__map_enc=-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBKbUxwNE1PRkhMNnFVM1dH\neERReFZWL0toSUg2ekRoNUFZQ1ZFL1YvVlU0CjRPN2NlNWpPakp4UWp6NURvZm1D\nblJmOFRka0I0NEorU2tNMFh2aWYzMFEKLS0tIDZhbmFoTERXNzgzVW82b094OVAr\nQmtTdTV1K0hkZXpIbkxRck1TVjZVTm8Kl2hj5e7mv0wRWLZaYCoqW/CZwncB9+vO\niUcostwDMBHR7Q9Iln+OcZnmhIQtyawZ5OU/ARynG9ps/yF++J+RGg==\n-----END AGE ENCRYPTED FILE-----\n
17+
sops_age__list_2__map_enc=-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBuNitEVzFDNVBjZ054TWts\nNEpNQXpaS3BvWjRPY2ZTNG1BSVNwMmo5OWc4CjhQWTBtYkVQa3R6c3hsenNJTTRS\nYUgzbmtxaEtDODVqYWxaL1ZTVDNBK3MKLS0tIDR6VjNqNklRb2grN2hBdG9BbENn\nYUhFMXlkb3RsRjFmMWRXOCttN0h6WHcKqwZGyeJiTbMWS5V56EBidSOOba/eGGMM\nD1LUwZmdUKivD3gOfK1STO6M7mEvJMUdzUebJR7IyC6Ibt0vyV5Afg==\n-----END AGE ENCRYPTED FILE-----\n
1618
sops_age__list_2__map_recipient=age1pgxk292zq30wafwg03gge7hu5dlu3h7yfldp2y8kqekfaljjky7s752uwy
17-
sops_age__list_3__map_enc=-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBETDZhR3E3NUNPR3ZOTzBE\nY1kxRlRWY3dEeDM1TmJOOW45Ump1TWVUaUFnCkkrNEErYmRINEQ3c2UvQjFTcnJv\nOHJZbk1oUGpRWm5hM2wweGc4V1lSU2sKLS0tIEJCWTVRMzhhM0ZpYWl6aVliN1dk\nMDYzN01GUGtBRWI5NFRQdWJncUl2SGMKH2tGAaN0aTAz575Q39S4uorUjD/BO8Oa\n/aHc/ytU9Jpj2tdMkPx/dD39Tv0ZO/z+rPZijv3Oirzk/FmDeV8/2g==\n-----END AGE ENCRYPTED FILE-----\n
19+
sops_age__list_3__map_enc=-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBUK3ZWeVRHS0VscmdpYXJt\naG5WaG13Si9JdnZaS1IxQWdvWWY3QzJSVHhnCmxpdW9WQmRBWHVPNFljeVg2c0Ry\nVCtoRkxsdXM5RGFxUUloYXhFU1g2cEUKLS0tIExCOHlXUzRKYndSYVJxVmZzcUhp\nRWJab3ZWRVlNME9sckIycUxwQ0JvRVUKr0F/Zd4TY448DpFc2+oqHybprN43ydKS\n3WmdCCHYY8GhmxgIlTSN7LCaJ2eTtOQaEX12Fhj5+JY5vmisW/cygg==\n-----END AGE ENCRYPTED FILE-----\n
1820
sops_age__list_3__map_recipient=age1qn0x93jhqjpqwvx5tgxnrwq5e3vuzur9whrkdnrvapd58esm45rqfkuxqh
19-
sops_lastmodified=2026-05-01T02:33:52Z
20-
sops_mac=ENC[AES256_GCM,data:5fsji5DmNyZFgmyV23bBj+OxmlFo0aBIH4w5Vz7gFVnYOv3p+TuResFll2aXzFPWLt02/spwhXeqwkDx/korefkL0zzKIuASRtS05UtAln098hL96zdWblP5TVzSORA2sDHvyz488EpV3BPywLvDPHRrKQu9TmEzlEBlmsWvHxo=,iv:RXQuKcZ4ybhhXPbaZUIDvGzwjZxjLTXarj7Zm84fepQ=,tag:/fPOM7H5B8bH7oGdfg/Sag==,type:str]
21+
sops_lastmodified=2026-05-02T02:32:16Z
22+
sops_mac=ENC[AES256_GCM,data:setdDRn3+KnoAJf43iTmzKRK4AG9Mgwp/jFY09FQQshVDUAB5STtH63nKB1GAOGHoDqUK53w+Fb1plFCMuHwn1/AWGisQ3pdkg9I13OhqhRAdsvsW9hI9Bx5C953K8t6GQdpvdO/kh3qDfh0RDpTQ9scv13Mmf8n7BqBj/gW9Fg=,iv:O+94d6Sy+d9pRiUp0f4njS6Eb8CouWHDqACppkAU2Gw=,tag:IWLf41HYQqMps9bjxR2wJg==,type:str]
2123
sops_unencrypted_suffix=_unencrypted
2224
sops_version=3.11.0

‎.github/workflows/deploy-mirror.yml‎

Lines changed: 13 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -87,14 +87,9 @@ jobs:
8787
GHCR_USER: ${{ secrets.GHCR_USER }}
8888
GHCR_TOKEN: ${{ secrets.GHCR_TOKEN }}
8989
IMAGE_TAG: mirror-${{ github.sha }}
90-
S3_ENDPOINT: ${{ secrets.S3_ENDPOINT }}
91-
S3_REGION: ${{ secrets.S3_REGION }}
92-
S3_ACCESS_KEY: ${{ secrets.S3_ACCESS_KEY }}
93-
S3_SECRET_KEY: ${{ secrets.S3_SECRET_KEY }}
94-
S3_BUCKET: ${{ secrets.S3_BUCKET }}
9590
run: |
9691
ssh "${SSH_USER}@${SSH_HOST}" \
97-
"env GHCR_USER='${GHCR_USER}' GHCR_TOKEN='${GHCR_TOKEN}' IMAGE_TAG='${IMAGE_TAG}' S3_ENDPOINT='${S3_ENDPOINT}' S3_REGION='${S3_REGION}' S3_ACCESS_KEY='${S3_ACCESS_KEY}' S3_SECRET_KEY='${S3_SECRET_KEY}' S3_BUCKET='${S3_BUCKET}' bash -s -- '${REPO}' '${BRANCH}'" <<'EOS'
92+
"env GHCR_USER='${GHCR_USER}' GHCR_TOKEN='${GHCR_TOKEN}' IMAGE_TAG='${IMAGE_TAG}' bash -s -- '${REPO}' '${BRANCH}'" <<'EOS'
9893
set -euo pipefail
9994
10095
REPO="${1:?missing repo}"
@@ -123,6 +118,13 @@ jobs:
123118
git checkout "${BRANCH}"
124119
git pull origin "${BRANCH}"
125120
121+
# Decrypt .env.enc into this directory (self-contained, no dependency on prod)
122+
umask 077
123+
sops -d --input-type dotenv --output-type dotenv .env.enc > .env
124+
set -a
125+
source <(grep -v '^#' .env | grep -v '^$')
126+
set +a
127+
126128
# Init swarm if not already active
127129
if ! sudo docker info --format '{{.Swarm.LocalNodeState}}' | grep -qx active; then
128130
sudo docker swarm init
@@ -132,13 +134,14 @@ jobs:
132134
133135
sudo docker pull "ghcr.io/${REPO}:${IMAGE_TAG}"
134136
135-
# Deploy using docker-compose.mirror.yml — Postgres is self-contained,
136-
# only S3 credentials need to be passed through for file storage.
137+
# Deploy using docker-compose.mirror.yml — Postgres is self-contained.
138+
# S3 creds and API key are sourced from prod .env above.
137139
sudo env \
138140
IMAGE="ghcr.io/${REPO}" IMAGE_TAG="$IMAGE_TAG" \
139-
S3_ENDPOINT="$S3_ENDPOINT" S3_REGION="$S3_REGION" \
140-
S3_ACCESS_KEY="$S3_ACCESS_KEY" S3_SECRET_KEY="$S3_SECRET_KEY" \
141+
S3_ENDPOINT="${S3_ENDPOINT:-}" S3_REGION="${S3_REGION:-auto}" \
142+
S3_ACCESS_KEY="${S3_ACCESS_KEY:-}" S3_SECRET_KEY="${S3_SECRET_KEY:-}" \
141143
S3_BUCKET="${S3_BUCKET:-underlay}" \
144+
UNDERLAY_UPSTREAM_API_KEY="${UNDERLAY_UPSTREAM_API_KEY:-}" \
142145
SESSION_SECRET="$(openssl rand -hex 32)" \
143146
docker stack deploy -c docker-compose.mirror.yml \
144147
--with-registry-auth --resolve-image always --prune "${STACK_NAME}"

‎docker-compose.mirror.yml‎

Lines changed: 144 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,144 @@
1+
# Mirror stack — deploy with: docker stack deploy -c docker-compose.mirror.yml underlay-mirror
2+
# Self-contained: spins up its own Postgres, no .env file needed.
3+
# Triggered manually via .github/workflows/deploy-mirror.yml
4+
5+
services:
6+
app:
7+
image: ${IMAGE:-ghcr.io/knowledgefutures/underlay}:${IMAGE_TAG:-latest}
8+
environment:
9+
NODE_ENV: production
10+
NODE_OPTIONS: "--max-old-space-size=448"
11+
# Mirror-specific
12+
UNDERLAY_MODE: mirror
13+
UNDERLAY_UPSTREAM: https://www.underlay.org
14+
UNDERLAY_NODE_NAME: IUA Mirror
15+
UNDERLAY_SYNC_SCHEDULE: "0 0 * * 0"
16+
# Database (internal to this stack)
17+
DATABASE_URL: postgresql://mirror:mirror@postgres:5432/mirror
18+
# S3 — shared bucket with prod (content-addressed = free dedup)
19+
S3_ENDPOINT: ${S3_ENDPOINT:-}
20+
S3_REGION: ${S3_REGION:-auto}
21+
S3_ACCESS_KEY: ${S3_ACCESS_KEY:-}
22+
S3_SECRET_KEY: ${S3_SECRET_KEY:-}
23+
S3_BUCKET: ${S3_BUCKET:-underlay}
24+
# Upstream API key (for authenticated sync — higher rate limit)
25+
UNDERLAY_UPSTREAM_API_KEY: ${UNDERLAY_UPSTREAM_API_KEY:-}
26+
# Session (for admin login)
27+
SESSION_SECRET: ${SESSION_SECRET:-mirror-secret-change-me}
28+
ports:
29+
- "4323:4321"
30+
- "3002:3000"
31+
networks:
32+
- mirrornet
33+
depends_on:
34+
- postgres
35+
deploy:
36+
replicas: 1
37+
resources:
38+
limits:
39+
memory: 512m
40+
cpus: "1.0"
41+
reservations:
42+
memory: 256m
43+
cpus: "0.25"
44+
update_config:
45+
parallelism: 1
46+
delay: 10s
47+
order: start-first
48+
failure_action: rollback
49+
restart_policy:
50+
condition: on-failure
51+
healthcheck:
52+
test: ["CMD-SHELL", "node -e \"const h=require('http'),check=(port,path)=>new Promise((res,rej)=>{h.get('http://127.0.0.1:'+port+path,r=>r.statusCode<400?res():rej()).on('error',rej)});Promise.all([check(3000,'/api/health'),check(4321,'/')]).then(()=>process.exit(0)).catch(()=>process.exit(1));\""]
53+
interval: 30s
54+
timeout: 5s
55+
retries: 3
56+
start_period: 30s
57+
logging:
58+
driver: json-file
59+
options:
60+
max-size: "10m"
61+
max-file: "3"
62+
63+
cron:
64+
image: ${IMAGE:-ghcr.io/knowledgefutures/underlay}:${IMAGE_TAG:-latest}
65+
environment:
66+
NODE_ENV: production
67+
NODE_OPTIONS: "--max-old-space-size=128"
68+
UNDERLAY_MODE: mirror
69+
UNDERLAY_UPSTREAM: https://www.underlay.org
70+
UNDERLAY_NODE_NAME: UK Mirror
71+
UNDERLAY_SYNC_SCHEDULE: "0 0 * * 0"
72+
DATABASE_URL: postgresql://mirror:mirror@postgres:5432/mirror
73+
S3_ENDPOINT: ${S3_ENDPOINT:-}
74+
S3_REGION: ${S3_REGION:-auto}
75+
S3_ACCESS_KEY: ${S3_ACCESS_KEY:-}
76+
S3_SECRET_KEY: ${S3_SECRET_KEY:-}
77+
S3_BUCKET: ${S3_BUCKET:-underlay}
78+
UNDERLAY_UPSTREAM_API_KEY: ${UNDERLAY_UPSTREAM_API_KEY:-}
79+
command: ["node", "--import", "tsx/esm", "tools/cron.ts"]
80+
networks:
81+
- mirrornet
82+
depends_on:
83+
- postgres
84+
deploy:
85+
replicas: 1
86+
resources:
87+
limits:
88+
memory: 192m
89+
cpus: "0.5"
90+
reservations:
91+
memory: 64m
92+
cpus: "0.1"
93+
restart_policy:
94+
condition: any
95+
logging:
96+
driver: json-file
97+
options:
98+
max-size: "5m"
99+
max-file: "3"
100+
101+
postgres:
102+
image: postgres:16-alpine
103+
environment:
104+
POSTGRES_USER: mirror
105+
POSTGRES_PASSWORD: mirror
106+
POSTGRES_DB: mirror
107+
command: >
108+
-c shared_buffers=256MB
109+
-c effective_cache_size=512MB
110+
-c work_mem=16MB
111+
-c maintenance_work_mem=64MB
112+
-c max_connections=50
113+
volumes:
114+
- mirror-pgdata:/var/lib/postgresql/data
115+
networks:
116+
- mirrornet
117+
deploy:
118+
replicas: 1
119+
resources:
120+
limits:
121+
memory: 512m
122+
cpus: "0.5"
123+
reservations:
124+
memory: 128m
125+
cpus: "0.1"
126+
restart_policy:
127+
condition: any
128+
healthcheck:
129+
test: ["CMD-SHELL", "pg_isready -U mirror"]
130+
interval: 10s
131+
timeout: 5s
132+
retries: 5
133+
logging:
134+
driver: json-file
135+
options:
136+
max-size: "5m"
137+
max-file: "3"
138+
139+
networks:
140+
mirrornet:
141+
driver: overlay
142+
143+
volumes:
144+
mirror-pgdata:

‎package.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
"tool:backup": "tsx tools/backupDb.ts",
1818
"tool:restore": "tsx tools/restore.ts",
1919
"tool:pruneBackups": "tsx tools/pruneBackups.ts",
20+
"tool:seed-mirror": "tsx tools/seedMirror.ts",
2021
"secrets:encrypt": "sops -e --input-type dotenv --output-type dotenv --output .env.enc .env",
2122
"secrets:encrypt:dev": "sops -e --input-type dotenv --output-type dotenv --output .env.dev.enc .env.dev",
2223
"secrets:decrypt": "sops -d --input-type dotenv --output-type dotenv --output .env .env.enc",

‎src/api/routes/admin.ts‎

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
import type { FastifyInstance } from "fastify";
2+
import { getMirrorConfig } from "../../lib/mirror-config.js";
3+
import {
4+
runMirrorSync,
5+
testUpstreamConnection,
6+
getMirrorStatus,
7+
getSyncHistory,
8+
syncEvents,
9+
stopSync,
10+
cleanupStaleRuns,
11+
isSyncRunning,
12+
getActiveRunId,
13+
getActiveRunLogs,
14+
type SyncProgressEvent,
15+
} from "../../lib/mirror-sync.js";
16+
17+
export async function adminRoutes(app: FastifyInstance) {
18+
// All admin routes require mirror mode to be enabled
19+
app.addHook("onRequest", async (_request, reply) => {
20+
const config = getMirrorConfig();
21+
if (!config.enabled) {
22+
return reply.status(404).send({ error: "Not found", statusCode: 404 });
23+
}
24+
});
25+
26+
// Get mirror status
27+
app.get("/admin/mirror/status", async () => {
28+
const status = await getMirrorStatus();
29+
return status;
30+
});
31+
32+
// Test upstream connection
33+
app.post("/admin/mirror/test", async () => {
34+
const config = getMirrorConfig();
35+
const result = await testUpstreamConnection(config.upstream);
36+
return result;
37+
});
38+
39+
// Trigger a sync manually (fire-and-forget, client uses SSE for progress)
40+
app.post("/admin/mirror/sync", async () => {
41+
if (isSyncRunning()) {
42+
return { started: false, error: "A sync is already running" };
43+
}
44+
// Start sync in background — don't await
45+
runMirrorSync("manual").catch((err) => {
46+
console.error("[mirror-sync] Unhandled sync error:", err);
47+
});
48+
return { started: true };
49+
});
50+
51+
// Stop a running sync (also cleans up stale DB rows from crashed processes)
52+
app.post("/admin/mirror/sync/stop", async () => {
53+
const stopped = stopSync();
54+
if (!stopped) {
55+
// No active sync in this process — clean up stale DB rows
56+
const cleaned = await cleanupStaleRuns();
57+
return { stopped: false, cleaned };
58+
}
59+
return { stopped: true };
60+
});
61+
62+
// SSE endpoint for live sync progress (replays buffered logs on connect)
63+
app.get("/admin/mirror/sync/progress", async (request, reply) => {
64+
reply.raw.writeHead(200, {
65+
"Content-Type": "text/event-stream",
66+
"Cache-Control": "no-cache",
67+
Connection: "keep-alive",
68+
});
69+
70+
// Replay buffered logs so reconnects/refreshes don't lose history
71+
const buffered = getActiveRunLogs();
72+
if (buffered.length > 0) {
73+
for (const msg of buffered) {
74+
const replayEvent: SyncProgressEvent = { type: "collection", message: msg, progress: { collectionsTotal: 0, collectionsProcessed: 0, versionsPulled: 0, filesDownloaded: 0, filesSkipped: 0, errors: 0 } };
75+
reply.raw.write(`data: ${JSON.stringify(replayEvent)}\n\n`);
76+
}
77+
}
78+
79+
// If no sync is running, close immediately
80+
if (!isSyncRunning()) {
81+
reply.raw.end();
82+
return;
83+
}
84+
85+
function onProgress(event: SyncProgressEvent) {
86+
reply.raw.write(`data: ${JSON.stringify(event)}\n\n`);
87+
if (event.type === "done") {
88+
setTimeout(() => reply.raw.end(), 100);
89+
}
90+
}
91+
92+
syncEvents.on("progress", onProgress);
93+
94+
request.raw.on("close", () => {
95+
syncEvents.off("progress", onProgress);
96+
});
97+
});
98+
99+
// Get current sync running state (for page refresh reconnection)
100+
app.get("/admin/mirror/sync/active", async () => {
101+
return {
102+
running: isSyncRunning(),
103+
runId: getActiveRunId(),
104+
logs: getActiveRunLogs(),
105+
};
106+
});
107+
108+
// Sync history
109+
app.get("/admin/mirror/history", async (request) => {
110+
const limit = Math.min(
111+
Number((request.query as any)?.limit) || 20,
112+
100,
113+
);
114+
return getSyncHistory(limit);
115+
});
116+
}

‎src/api/server.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ import { versionRoutes } from "./routes/versions.js";
1212
import { uploadRoutes } from "./routes/uploads.js";
1313
import { fileRoutes } from "./routes/files.js";
1414
import { schemaRoutes } from "./routes/schemas.js";
15+
import { adminRoutes } from "./routes/admin.js";
1516
import { queryRoutes } from "./routes/query.js";
1617

1718
export async function buildApp() {
@@ -75,6 +76,7 @@ export async function buildApp() {
7576
await app.register(uploadRoutes, { prefix: "/api" });
7677
await app.register(fileRoutes, { prefix: "/api" });
7778
await app.register(schemaRoutes, { prefix: "/api" });
79+
await app.register(adminRoutes, { prefix: "/api" });
7880
await app.register(queryRoutes, { prefix: "/api" });
7981

8082
return app;

0 commit comments

Comments
 (0)