Skip to content

Commit 84a135f

Browse files
committed
feat(infra): add manifest generator and checksum source tracking
- Add generate-manifests-from-r2 tool to scan R2 bucket and generate manifests - Update mirror tool to write .meta.json files with checksum source - Add sha256_source field to manifest schema ("upstream" or "dtvem") - Add SHA256Source field to Go manifest.Download struct - Update mirror-sync workflow to weekly schedule (not manifest-triggered) - Add generate-manifests-from-r2.yml workflow
1 parent 0847d85 commit 84a135f

8 files changed

Lines changed: 615 additions & 55 deletions

File tree

Lines changed: 125 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,125 @@
1+
name: Generate Manifests from R2
2+
3+
on:
4+
# Trigger after mirror sync completes
5+
workflow_run:
6+
workflows: ["Mirror Sync"]
7+
types:
8+
- completed
9+
# Manual trigger
10+
workflow_dispatch:
11+
inputs:
12+
runtime:
13+
description: 'Runtime to generate (node, python, ruby, or all)'
14+
required: true
15+
default: 'all'
16+
type: choice
17+
options:
18+
- all
19+
- node
20+
- python
21+
- ruby
22+
dry_run:
23+
description: 'Dry run (report only, no file changes)'
24+
required: false
25+
default: false
26+
type: boolean
27+
28+
jobs:
29+
generate:
30+
name: Generate Manifests
31+
runs-on: ubuntu-latest
32+
if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }}
33+
34+
steps:
35+
- name: Checkout
36+
uses: actions/checkout@v4
37+
38+
- name: Setup Go
39+
uses: actions/setup-go@v5
40+
with:
41+
go-version-file: 'go.mod'
42+
43+
- name: Build manifest generator
44+
run: |
45+
cd scripts/generate-manifests-from-r2
46+
go build -o generate-manifests .
47+
48+
- name: Generate manifests (dry run)
49+
if: ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run }}
50+
env:
51+
R2_ENDPOINT: https://${{ secrets.CLOUDFLARE_ACCOUNT_ID }}.r2.cloudflarestorage.com
52+
R2_BUCKET: ${{ secrets.CLOUDFLARE_R2_BUILDS_BUCKET }}
53+
R2_ACCESS_KEY: ${{ secrets.CLOUDFLARE_R2_ACCESS_KEY_ID }}
54+
R2_SECRET_KEY: ${{ secrets.CLOUDFLARE_R2_SECRET_ACCESS_KEY }}
55+
run: |
56+
RUNTIME="${{ inputs.runtime || 'all' }}"
57+
./scripts/generate-manifests-from-r2/generate-manifests \
58+
--runtime="$RUNTIME" \
59+
--output-dir=src/internal/manifest/data \
60+
--base-url="https://builds.dtvem.io" \
61+
--r2-endpoint="$R2_ENDPOINT" \
62+
--r2-bucket="$R2_BUCKET" \
63+
--r2-access-key="$R2_ACCESS_KEY" \
64+
--r2-secret-key="$R2_SECRET_KEY" \
65+
--dry-run
66+
67+
- name: Generate manifests
68+
if: ${{ github.event_name != 'workflow_dispatch' || !inputs.dry_run }}
69+
env:
70+
R2_ENDPOINT: https://${{ secrets.CLOUDFLARE_ACCOUNT_ID }}.r2.cloudflarestorage.com
71+
R2_BUCKET: ${{ secrets.CLOUDFLARE_R2_BUILDS_BUCKET }}
72+
R2_ACCESS_KEY: ${{ secrets.CLOUDFLARE_R2_ACCESS_KEY_ID }}
73+
R2_SECRET_KEY: ${{ secrets.CLOUDFLARE_R2_SECRET_ACCESS_KEY }}
74+
run: |
75+
RUNTIME="${{ inputs.runtime || 'all' }}"
76+
./scripts/generate-manifests-from-r2/generate-manifests \
77+
--runtime="$RUNTIME" \
78+
--output-dir=src/internal/manifest/data \
79+
--base-url="https://builds.dtvem.io" \
80+
--r2-endpoint="$R2_ENDPOINT" \
81+
--r2-bucket="$R2_BUCKET" \
82+
--r2-access-key="$R2_ACCESS_KEY" \
83+
--r2-secret-key="$R2_SECRET_KEY"
84+
85+
- name: Check for changes
86+
id: check-changes
87+
if: ${{ github.event_name != 'workflow_dispatch' || !inputs.dry_run }}
88+
run: |
89+
git diff --quiet src/internal/manifest/data/ || echo "changed=true" >> $GITHUB_OUTPUT
90+
91+
- name: Create Pull Request
92+
if: ${{ steps.check-changes.outputs.changed == 'true' }}
93+
uses: peter-evans/create-pull-request@v7
94+
with:
95+
token: ${{ secrets.GITHUB_TOKEN }}
96+
commit-message: 'chore(manifest): regenerate manifests from R2'
97+
title: 'chore(manifest): regenerate manifests from R2'
98+
body: |
99+
Regenerated manifests from binaries hosted on `builds.dtvem.io`.
100+
101+
This PR was created by the [Generate Manifests from R2 workflow](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}).
102+
103+
## Changes
104+
- URLs now point to `builds.dtvem.io` (our hosted binaries)
105+
- Includes `sha256_source` field indicating checksum origin ("upstream" or "dtvem")
106+
107+
Please review the changes before merging.
108+
branch: chore/regenerate-manifests-from-r2
109+
delete-branch: true
110+
labels: |
111+
automated
112+
manifest
113+
114+
- name: Generate summary
115+
if: always()
116+
run: |
117+
echo "## Manifest Generation" >> $GITHUB_STEP_SUMMARY
118+
echo "" >> $GITHUB_STEP_SUMMARY
119+
if [ "${{ inputs.dry_run }}" = "true" ]; then
120+
echo "**Mode:** Dry run (no changes made)" >> $GITHUB_STEP_SUMMARY
121+
elif [ "${{ steps.check-changes.outputs.changed }}" = "true" ]; then
122+
echo "**Result:** Changes detected, PR created" >> $GITHUB_STEP_SUMMARY
123+
else
124+
echo "**Result:** No changes detected" >> $GITHUB_STEP_SUMMARY
125+
fi

.github/workflows/mirror-sync.yml

Lines changed: 9 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,9 @@
11
name: Mirror Sync
22

33
on:
4-
# Run after manifest updates are deployed
5-
workflow_run:
6-
workflows: ["Deploy Manifests"]
7-
types:
8-
- completed
4+
# Run weekly to catch new upstream versions
5+
schedule:
6+
- cron: '0 4 * * 0' # Every Sunday at 4 AM UTC
97
# Manual trigger
108
workflow_dispatch:
119
inputs:
@@ -24,12 +22,11 @@ jobs:
2422
sync:
2523
name: Sync ${{ matrix.runtime }}
2624
runs-on: ubuntu-latest
27-
if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }}
28-
timeout-minutes: 120 # 2 hours max for sync
25+
timeout-minutes: 180 # 3 hours max for sync
2926
strategy:
3027
fail-fast: false
3128
matrix:
32-
runtime: ${{ (github.event_name == 'workflow_dispatch' && inputs.runtime == 'all') && fromJson('["node", "python", "ruby"]') || (github.event_name == 'workflow_dispatch' && fromJson(format('["{0}"]', inputs.runtime))) || fromJson('["node", "python", "ruby"]') }}
29+
runtime: ${{ (github.event_name == 'workflow_dispatch' && inputs.runtime != 'all') && fromJson(format('["{0}"]', inputs.runtime)) || fromJson('["node", "python", "ruby"]') }}
3330

3431
steps:
3532
- name: Checkout
@@ -45,7 +42,7 @@ jobs:
4542
cd scripts/mirror-binaries
4643
go build -o mirror-binaries .
4744
48-
- name: Sync new binaries
45+
- name: Sync new binaries to R2
4946
env:
5047
R2_ENDPOINT: https://${{ secrets.CLOUDFLARE_ACCOUNT_ID }}.r2.cloudflarestorage.com
5148
R2_BUCKET: ${{ secrets.CLOUDFLARE_R2_BUILDS_BUCKET }}
@@ -67,4 +64,6 @@ jobs:
6764
run: |
6865
echo "## Sync Results for ${{ matrix.runtime }}" >> $GITHUB_STEP_SUMMARY
6966
echo "" >> $GITHUB_STEP_SUMMARY
70-
echo "Synced new binaries not already present in R2." >> $GITHUB_STEP_SUMMARY
67+
echo "Synced new binaries from upstream that were not already in R2." >> $GITHUB_STEP_SUMMARY
68+
echo "" >> $GITHUB_STEP_SUMMARY
69+
echo "To generate updated manifests, run the 'Generate Manifests from R2' workflow." >> $GITHUB_STEP_SUMMARY

schemas/manifest.schema.json

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@
4747
"download": {
4848
"type": "object",
4949
"description": "Download information for a pre-built binary",
50-
"required": ["url", "sha256"],
50+
"required": ["url"],
5151
"additionalProperties": false,
5252
"properties": {
5353
"url": {
@@ -59,6 +59,11 @@
5959
"type": "string",
6060
"pattern": "^[a-fA-F0-9]{64}$",
6161
"description": "SHA256 checksum (64 hex characters)"
62+
},
63+
"sha256_source": {
64+
"type": "string",
65+
"enum": ["upstream", "dtvem"],
66+
"description": "Origin of the SHA256 checksum: 'upstream' if from the original provider, 'dtvem' if generated by us during mirroring"
6267
}
6368
}
6469
}
@@ -69,13 +74,15 @@
6974
"versions": {
7075
"3.13.1": {
7176
"windows-amd64": {
72-
"url": "https://www.python.org/ftp/python/3.13.1/python-3.13.1-embed-amd64.zip",
73-
"sha256": "a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2"
77+
"url": "https://builds.dtvem.io/python/3.13.1/windows-amd64.zip",
78+
"sha256": "a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2",
79+
"sha256_source": "upstream"
7480
},
7581
"darwin-arm64": null,
7682
"linux-amd64": {
77-
"url": "https://github.com/astral-sh/python-build-standalone/releases/download/20251209/cpython-3.13.1.tar.gz",
78-
"sha256": "b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3"
83+
"url": "https://builds.dtvem.io/python/3.13.1/linux-amd64.tar.gz",
84+
"sha256": "b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3",
85+
"sha256_source": "dtvem"
7986
}
8087
}
8188
}
Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
module github.com/dtvem/dtvem/scripts/generate-manifests-from-r2
2+
3+
go 1.23.0
4+
5+
require (
6+
github.com/aws/aws-sdk-go-v2 v1.32.6
7+
github.com/aws/aws-sdk-go-v2/config v1.28.6
8+
github.com/aws/aws-sdk-go-v2/credentials v1.17.47
9+
github.com/aws/aws-sdk-go-v2/service/s3 v1.71.0
10+
)
11+
12+
require (
13+
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7 // indirect
14+
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.21 // indirect
15+
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.25 // indirect
16+
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.25 // indirect
17+
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect
18+
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.25 // indirect
19+
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1 // indirect
20+
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.6 // indirect
21+
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.6 // indirect
22+
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.6 // indirect
23+
github.com/aws/aws-sdk-go-v2/service/sso v1.24.7 // indirect
24+
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.6 // indirect
25+
github.com/aws/aws-sdk-go-v2/service/sts v1.33.2 // indirect
26+
github.com/aws/smithy-go v1.22.1 // indirect
27+
)
Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
github.com/aws/aws-sdk-go-v2 v1.32.6 h1:7BokKRgRPuGmKkFMhEg/jSul+tB9VvXhcViILtfG8b4=
2+
github.com/aws/aws-sdk-go-v2 v1.32.6/go.mod h1:P5WJBrYqqbWVaOxgH0X/FYYD47/nooaPOZPlQdmiN2U=
3+
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7 h1:lL7IfaFzngfx0ZwUGOZdsFFnQ5uLvR0hWqqhyE7Q9M8=
4+
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7/go.mod h1:QraP0UcVlQJsmHfioCrveWOC1nbiWUl3ej08h4mXWoc=
5+
github.com/aws/aws-sdk-go-v2/config v1.28.6 h1:D89IKtGrs/I3QXOLNTH93NJYtDhm8SYa9Q5CsPShmyo=
6+
github.com/aws/aws-sdk-go-v2/config v1.28.6/go.mod h1:GDzxJ5wyyFSCoLkS+UhGB0dArhb9mI+Co4dHtoTxbko=
7+
github.com/aws/aws-sdk-go-v2/credentials v1.17.47 h1:48bA+3/fCdi2yAwVt+3COvmatZ6jUDNkDTIsqDiMUdw=
8+
github.com/aws/aws-sdk-go-v2/credentials v1.17.47/go.mod h1:+KdckOejLW3Ks3b0E3b5rHsr2f9yuORBum0WPnE5o5w=
9+
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.21 h1:AmoU1pziydclFT/xRV+xXE/Vb8fttJCLRPv8oAkprc0=
10+
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.21/go.mod h1:AjUdLYe4Tgs6kpH4Bv7uMZo7pottoyHMn4eTcIcneaY=
11+
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.25 h1:s/fF4+yDQDoElYhfIVvSNyeCydfbuTKzhxSXDXCPasU=
12+
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.25/go.mod h1:IgPfDv5jqFIzQSNbUEMoitNooSMXjRSDkhXv8jiROvU=
13+
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.25 h1:ZntTCl5EsYnhN/IygQEUugpdwbhdkom9uHcbCftiGgA=
14+
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.25/go.mod h1:DBdPrgeocww+CSl1C8cEV8PN1mHMBhuCDLpXezyvWkE=
15+
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 h1:VaRN3TlFdd6KxX1x3ILT5ynH6HvKgqdiXoTxAF4HQcQ=
16+
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc=
17+
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.25 h1:r67ps7oHCYnflpgDy2LZU0MAQtQbYIOqNNnqGO6xQkE=
18+
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.25/go.mod h1:GrGY+Q4fIokYLtjCVB/aFfCVL6hhGUFl8inD18fDalE=
19+
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1 h1:iXtILhvDxB6kPvEXgsDhGaZCSC6LQET5ZHSdJozeI0Y=
20+
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1/go.mod h1:9nu0fVANtYiAePIBh2/pFUSwtJ402hLnp854CNoDOeE=
21+
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.6 h1:HCpPsWqmYQieU7SS6E9HXfdAMSud0pteVXieJmcpIRI=
22+
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.6/go.mod h1:ngUiVRCco++u+soRRVBIvBZxSMMvOVMXA4PJ36JLfSw=
23+
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.6 h1:50+XsN70RS7dwJ2CkVNXzj7U2L1HKP8nqTd3XWEXBN4=
24+
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.6/go.mod h1:WqgLmwY7so32kG01zD8CPTJWVWM+TzJoOVHwTg4aPug=
25+
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.6 h1:BbGDtTi0T1DYlmjBiCr/le3wzhA37O8QTC5/Ab8+EXk=
26+
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.6/go.mod h1:hLMJt7Q8ePgViKupeymbqI0la+t9/iYFBjxQCFwuAwI=
27+
github.com/aws/aws-sdk-go-v2/service/s3 v1.71.0 h1:nyuzXooUNJexRT0Oy0UQY6AhOzxPxhtt4DcBIHyCnmw=
28+
github.com/aws/aws-sdk-go-v2/service/s3 v1.71.0/go.mod h1:sT/iQz8JK3u/5gZkT+Hmr7GzVZehUMkRZpOaAwYXeGY=
29+
github.com/aws/aws-sdk-go-v2/service/sso v1.24.7 h1:rLnYAfXQ3YAccocshIH5mzNNwZBkBo+bP6EhIxak6Hw=
30+
github.com/aws/aws-sdk-go-v2/service/sso v1.24.7/go.mod h1:ZHtuQJ6t9A/+YDuxOLnbryAmITtr8UysSny3qcyvJTc=
31+
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.6 h1:JnhTZR3PiYDNKlXy50/pNeix9aGMo6lLpXwJ1mw8MD4=
32+
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.6/go.mod h1:URronUEGfXZN1VpdktPSD1EkAL9mfrV+2F4sjH38qOY=
33+
github.com/aws/aws-sdk-go-v2/service/sts v1.33.2 h1:s4074ZO1Hk8qv65GqNXqDjmkf4HSQqJukaLuuW0TpDA=
34+
github.com/aws/aws-sdk-go-v2/service/sts v1.33.2/go.mod h1:mVggCnIWoM09jP71Wh+ea7+5gAp53q+49wDFs1SW5z8=
35+
github.com/aws/smithy-go v1.22.1 h1:/HPHZQ0g7f4eUeK6HKglFz8uwVfZKgoI25rb/J+dnro=
36+
github.com/aws/smithy-go v1.22.1/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg=

0 commit comments

Comments
 (0)