Skip to content

Commit 15ee7df

Browse files
CalvinAllenCalvinAllen
authored
feat(infra): add binary mirroring workflows and tool (#166)
1 parent 31801df commit 15ee7df

11 files changed

Lines changed: 1173 additions & 5 deletions

File tree

‎.github/workflows/generate-manifests-from-r2.yml‎

Lines changed: 148 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,148 @@
1+
name: Generate Manifests from R2
2+
3+
on:
4+
# Trigger after mirror workflows complete
5+
workflow_run:
6+
workflows: ["Mirror Sync", "Mirror All Binaries"]
7+
types:
8+
- completed
9+
# Manual trigger
10+
workflow_dispatch:
11+
inputs:
12+
runtime:
13+
description: 'Runtime to generate (node, python, ruby, or all)'
14+
required: true
15+
default: 'all'
16+
type: choice
17+
options:
18+
- all
19+
- node
20+
- python
21+
- ruby
22+
dry_run:
23+
description: 'Dry run (report only, no file changes)'
24+
required: false
25+
default: false
26+
type: boolean
27+
28+
jobs:
29+
generate:
30+
name: Generate Manifests
31+
runs-on: ubuntu-latest
32+
if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }}
33+
34+
steps:
35+
- name: Checkout
36+
uses: actions/checkout@v4
37+
38+
- name: Setup Go
39+
uses: actions/setup-go@v5
40+
with:
41+
go-version-file: 'go.mod'
42+
43+
- name: Build manifest generator
44+
run: |
45+
cd scripts/generate-manifests-from-r2
46+
go build -o generate-manifests .
47+
48+
- name: Generate manifests (dry run)
49+
if: ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run }}
50+
env:
51+
R2_ENDPOINT: https://${{ secrets.CLOUDFLARE_ACCOUNT_ID }}.r2.cloudflarestorage.com
52+
R2_BUCKET: ${{ secrets.CLOUDFLARE_R2_BUILDS_BUCKET }}
53+
R2_ACCESS_KEY: ${{ secrets.CLOUDFLARE_R2_ACCESS_KEY_ID }}
54+
R2_SECRET_KEY: ${{ secrets.CLOUDFLARE_R2_SECRET_ACCESS_KEY }}
55+
run: |
56+
RUNTIME="${{ inputs.runtime || 'all' }}"
57+
./scripts/generate-manifests-from-r2/generate-manifests \
58+
--runtime="$RUNTIME" \
59+
--output-dir=src/internal/manifest/data \
60+
--base-url="https://builds.dtvem.io" \
61+
--r2-endpoint="$R2_ENDPOINT" \
62+
--r2-bucket="$R2_BUCKET" \
63+
--r2-access-key="$R2_ACCESS_KEY" \
64+
--r2-secret-key="$R2_SECRET_KEY" \
65+
--dry-run
66+
67+
- name: Generate manifests
68+
if: ${{ github.event_name != 'workflow_dispatch' || !inputs.dry_run }}
69+
env:
70+
R2_ENDPOINT: https://${{ secrets.CLOUDFLARE_ACCOUNT_ID }}.r2.cloudflarestorage.com
71+
R2_BUCKET: ${{ secrets.CLOUDFLARE_R2_BUILDS_BUCKET }}
72+
R2_ACCESS_KEY: ${{ secrets.CLOUDFLARE_R2_ACCESS_KEY_ID }}
73+
R2_SECRET_KEY: ${{ secrets.CLOUDFLARE_R2_SECRET_ACCESS_KEY }}
74+
run: |
75+
RUNTIME="${{ inputs.runtime || 'all' }}"
76+
./scripts/generate-manifests-from-r2/generate-manifests \
77+
--runtime="$RUNTIME" \
78+
--output-dir=src/internal/manifest/data \
79+
--base-url="https://builds.dtvem.io" \
80+
--r2-endpoint="$R2_ENDPOINT" \
81+
--r2-bucket="$R2_BUCKET" \
82+
--r2-access-key="$R2_ACCESS_KEY" \
83+
--r2-secret-key="$R2_SECRET_KEY"
84+
85+
- name: Deploy manifests to R2
86+
if: ${{ github.event_name != 'workflow_dispatch' || !inputs.dry_run }}
87+
env:
88+
R2_ENDPOINT: https://${{ secrets.CLOUDFLARE_ACCOUNT_ID }}.r2.cloudflarestorage.com
89+
R2_BUCKET: ${{ secrets.CLOUDFLARE_R2_MANIFESTS_BUCKET }}
90+
run: |
91+
aws configure set aws_access_key_id ${{ secrets.CLOUDFLARE_R2_ACCESS_KEY_ID }}
92+
aws configure set aws_secret_access_key ${{ secrets.CLOUDFLARE_R2_SECRET_ACCESS_KEY }}
93+
aws configure set default.region auto
94+
95+
echo "Deploying manifests to R2..."
96+
for file in src/internal/manifest/data/*.json; do
97+
filename=$(basename "$file")
98+
echo "Uploading $filename..."
99+
aws s3 cp "$file" "s3://${R2_BUCKET}/${filename}" \
100+
--endpoint-url "${R2_ENDPOINT}" \
101+
--content-type "application/json" \
102+
--cache-control "public, max-age=300"
103+
done
104+
echo "Manifests deployed to R2!"
105+
106+
- name: Check for changes
107+
id: check-changes
108+
if: ${{ github.event_name != 'workflow_dispatch' || !inputs.dry_run }}
109+
run: |
110+
git diff --quiet src/internal/manifest/data/ || echo "changed=true" >> $GITHUB_OUTPUT
111+
112+
- name: Create Pull Request
113+
if: ${{ steps.check-changes.outputs.changed == 'true' }}
114+
uses: peter-evans/create-pull-request@v7
115+
with:
116+
token: ${{ secrets.GITHUB_TOKEN }}
117+
commit-message: 'chore(manifest): regenerate manifests from R2'
118+
title: 'chore(manifest): regenerate manifests from R2'
119+
body: |
120+
Regenerated manifests from binaries hosted on `builds.dtvem.io`.
121+
122+
This PR was created by the [Generate Manifests from R2 workflow](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}).
123+
124+
## Changes
125+
- URLs now point to `builds.dtvem.io` (our hosted binaries)
126+
- Includes `sha256_source` field indicating checksum origin ("upstream" or "dtvem")
127+
128+
Please review the changes before merging.
129+
branch: chore/regenerate-manifests-from-r2
130+
delete-branch: true
131+
labels: |
132+
automated
133+
manifest
134+
135+
- name: Generate summary
136+
if: always()
137+
run: |
138+
echo "## Manifest Generation" >> $GITHUB_STEP_SUMMARY
139+
echo "" >> $GITHUB_STEP_SUMMARY
140+
if [ "${{ inputs.dry_run }}" = "true" ]; then
141+
echo "**Mode:** Dry run (no changes made)" >> $GITHUB_STEP_SUMMARY
142+
elif [ "${{ steps.check-changes.outputs.changed }}" = "true" ]; then
143+
echo "**Result:** Changes detected" >> $GITHUB_STEP_SUMMARY
144+
echo "- Manifests deployed to R2 (live immediately)" >> $GITHUB_STEP_SUMMARY
145+
echo "- PR created for embedded manifests" >> $GITHUB_STEP_SUMMARY
146+
else
147+
echo "**Result:** No changes detected" >> $GITHUB_STEP_SUMMARY
148+
fi

‎.github/workflows/mirror-all.yml‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
name: Mirror All Binaries
2+
3+
on:
4+
workflow_dispatch:
5+
inputs:
6+
runtime:
7+
description: 'Runtime to mirror (node, python, ruby, or all)'
8+
required: true
9+
default: 'all'
10+
type: choice
11+
options:
12+
- all
13+
- node
14+
- python
15+
- ruby
16+
dry_run:
17+
description: 'Dry run (report only, no uploads)'
18+
required: false
19+
default: false
20+
type: boolean
21+
22+
jobs:
23+
mirror:
24+
name: Mirror ${{ matrix.runtime }}
25+
runs-on: ubuntu-latest
26+
timeout-minutes: 360 # 6 hours max
27+
strategy:
28+
fail-fast: false
29+
matrix:
30+
runtime: ${{ inputs.runtime == 'all' && fromJson('["node", "python", "ruby"]') || fromJson(format('["{0}"]', inputs.runtime)) }}
31+
32+
steps:
33+
- name: Checkout
34+
uses: actions/checkout@v4
35+
36+
- name: Setup Go
37+
uses: actions/setup-go@v5
38+
with:
39+
go-version-file: 'go.mod'
40+
41+
- name: Build mirror tool
42+
run: |
43+
cd scripts/mirror-binaries
44+
go build -o mirror-binaries .
45+
46+
- name: Mirror binaries (dry run)
47+
if: inputs.dry_run
48+
run: |
49+
./scripts/mirror-binaries/mirror-binaries \
50+
--runtime=${{ matrix.runtime }} \
51+
--manifest-dir=src/internal/manifest/data \
52+
--dry-run
53+
54+
- name: Mirror binaries
55+
if: ${{ !inputs.dry_run }}
56+
env:
57+
R2_ENDPOINT: https://${{ secrets.CLOUDFLARE_ACCOUNT_ID }}.r2.cloudflarestorage.com
58+
R2_BUCKET: ${{ secrets.CLOUDFLARE_R2_BUILDS_BUCKET }}
59+
R2_ACCESS_KEY: ${{ secrets.CLOUDFLARE_R2_ACCESS_KEY_ID }}
60+
R2_SECRET_KEY: ${{ secrets.CLOUDFLARE_R2_SECRET_ACCESS_KEY }}
61+
run: |
62+
./scripts/mirror-binaries/mirror-binaries \
63+
--runtime=${{ matrix.runtime }} \
64+
--manifest-dir=src/internal/manifest/data \
65+
--r2-endpoint="$R2_ENDPOINT" \
66+
--r2-bucket="$R2_BUCKET" \
67+
--r2-access-key="$R2_ACCESS_KEY" \
68+
--r2-secret-key="$R2_SECRET_KEY" \
69+
--workers=20
70+
71+
- name: Generate summary
72+
if: always()
73+
run: |
74+
echo "## Mirror Results for ${{ matrix.runtime }}" >> $GITHUB_STEP_SUMMARY
75+
echo "" >> $GITHUB_STEP_SUMMARY
76+
if [ "${{ inputs.dry_run }}" = "true" ]; then
77+
echo "**Mode:** Dry run (no uploads)" >> $GITHUB_STEP_SUMMARY
78+
else
79+
echo "**Mode:** Live upload to R2" >> $GITHUB_STEP_SUMMARY
80+
fi

‎.github/workflows/mirror-sync.yml‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
name: Mirror Sync
2+
3+
on:
4+
# Run daily to catch new upstream versions
5+
schedule:
6+
- cron: '0 4 * * *' # Every day at 4 AM UTC
7+
# Manual trigger
8+
workflow_dispatch:
9+
inputs:
10+
runtime:
11+
description: 'Runtime to sync (node, python, ruby, or all)'
12+
required: true
13+
default: 'all'
14+
type: choice
15+
options:
16+
- all
17+
- node
18+
- python
19+
- ruby
20+
21+
jobs:
22+
sync:
23+
name: Sync ${{ matrix.runtime }}
24+
runs-on: ubuntu-latest
25+
timeout-minutes: 180 # 3 hours max for sync
26+
strategy:
27+
fail-fast: false
28+
matrix:
29+
runtime: ${{ (github.event_name == 'workflow_dispatch' && inputs.runtime != 'all') && fromJson(format('["{0}"]', inputs.runtime)) || fromJson('["node", "python", "ruby"]') }}
30+
31+
steps:
32+
- name: Checkout
33+
uses: actions/checkout@v4
34+
35+
- name: Setup Go
36+
uses: actions/setup-go@v5
37+
with:
38+
go-version-file: 'go.mod'
39+
40+
- name: Build mirror tool
41+
run: |
42+
cd scripts/mirror-binaries
43+
go build -o mirror-binaries .
44+
45+
- name: Sync new binaries to R2
46+
env:
47+
R2_ENDPOINT: https://${{ secrets.CLOUDFLARE_ACCOUNT_ID }}.r2.cloudflarestorage.com
48+
R2_BUCKET: ${{ secrets.CLOUDFLARE_R2_BUILDS_BUCKET }}
49+
R2_ACCESS_KEY: ${{ secrets.CLOUDFLARE_R2_ACCESS_KEY_ID }}
50+
R2_SECRET_KEY: ${{ secrets.CLOUDFLARE_R2_SECRET_ACCESS_KEY }}
51+
run: |
52+
./scripts/mirror-binaries/mirror-binaries \
53+
--runtime=${{ matrix.runtime }} \
54+
--manifest-dir=src/internal/manifest/data \
55+
--r2-endpoint="$R2_ENDPOINT" \
56+
--r2-bucket="$R2_BUCKET" \
57+
--r2-access-key="$R2_ACCESS_KEY" \
58+
--r2-secret-key="$R2_SECRET_KEY" \
59+
--sync-only \
60+
--workers=10
61+
62+
- name: Generate summary
63+
if: always()
64+
run: |
65+
echo "## Sync Results for ${{ matrix.runtime }}" >> $GITHUB_STEP_SUMMARY
66+
echo "" >> $GITHUB_STEP_SUMMARY
67+
echo "Synced new binaries from upstream that were not already in R2." >> $GITHUB_STEP_SUMMARY
68+
echo "" >> $GITHUB_STEP_SUMMARY
69+
echo "To generate updated manifests, run the 'Generate Manifests from R2' workflow." >> $GITHUB_STEP_SUMMARY

‎schemas/manifest.schema.json‎

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@
4747
"download": {
4848
"type": "object",
4949
"description": "Download information for a pre-built binary",
50-
"required": ["url", "sha256"],
50+
"required": ["url"],
5151
"additionalProperties": false,
5252
"properties": {
5353
"url": {
@@ -59,6 +59,11 @@
5959
"type": "string",
6060
"pattern": "^[a-fA-F0-9]{64}$",
6161
"description": "SHA256 checksum (64 hex characters)"
62+
},
63+
"sha256_source": {
64+
"type": "string",
65+
"enum": ["upstream", "dtvem"],
66+
"description": "Origin of the SHA256 checksum: 'upstream' if from the original provider, 'dtvem' if generated by us during mirroring"
6267
}
6368
}
6469
}
@@ -69,13 +74,15 @@
6974
"versions": {
7075
"3.13.1": {
7176
"windows-amd64": {
72-
"url": "https://www.python.org/ftp/python/3.13.1/python-3.13.1-embed-amd64.zip",
73-
"sha256": "a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2"
77+
"url": "https://builds.dtvem.io/python/3.13.1/windows-amd64.zip",
78+
"sha256": "a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2",
79+
"sha256_source": "upstream"
7480
},
7581
"darwin-arm64": null,
7682
"linux-amd64": {
77-
"url": "https://github.com/astral-sh/python-build-standalone/releases/download/20251209/cpython-3.13.1.tar.gz",
78-
"sha256": "b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3"
83+
"url": "https://builds.dtvem.io/python/3.13.1/linux-amd64.tar.gz",
84+
"sha256": "b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3",
85+
"sha256_source": "dtvem"
7986
}
8087
}
8188
}

‎scripts/generate-manifests-from-r2/go.mod‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
module github.com/dtvem/dtvem/scripts/generate-manifests-from-r2
2+
3+
go 1.23.0
4+
5+
require (
6+
github.com/aws/aws-sdk-go-v2 v1.32.6
7+
github.com/aws/aws-sdk-go-v2/config v1.28.6
8+
github.com/aws/aws-sdk-go-v2/credentials v1.17.47
9+
github.com/aws/aws-sdk-go-v2/service/s3 v1.71.0
10+
)
11+
12+
require (
13+
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7 // indirect
14+
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.21 // indirect
15+
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.25 // indirect
16+
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.25 // indirect
17+
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect
18+
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.25 // indirect
19+
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1 // indirect
20+
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.6 // indirect
21+
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.6 // indirect
22+
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.6 // indirect
23+
github.com/aws/aws-sdk-go-v2/service/sso v1.24.7 // indirect
24+
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.6 // indirect
25+
github.com/aws/aws-sdk-go-v2/service/sts v1.33.2 // indirect
26+
github.com/aws/smithy-go v1.22.1 // indirect
27+
)

0 commit comments

Comments
 (0)