feat(infra): add binary mirroring workflows and tool

.github/workflows/generate-manifests-from-r2.yml

@@ -0,0 +1,148 @@
+name: Generate Manifests from R2
+
+on:
+  # Trigger after mirror workflows complete
+  workflow_run:
+    workflows: ["Mirror Sync", "Mirror All Binaries"]
+    types:
+      - completed
+  # Manual trigger
+  workflow_dispatch:
+    inputs:
+      runtime:
+        description: 'Runtime to generate (node, python, ruby, or all)'
+        required: true
+        default: 'all'
+        type: choice
+        options:
+          - all
+          - node
+          - python
+          - ruby
+      dry_run:
+        description: 'Dry run (report only, no file changes)'
+        required: false
+        default: false
+        type: boolean
+
+jobs:
+  generate:
+    name: Generate Manifests
+    runs-on: ubuntu-latest
+    if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }}
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Go
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: 'go.mod'
+
+      - name: Build manifest generator
+        run: |
+          cd scripts/generate-manifests-from-r2
+          go build -o generate-manifests .
+
+      - name: Generate manifests (dry run)
+        if: ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run }}
+        env:
+          R2_ENDPOINT: https://${{ secrets.CLOUDFLARE_ACCOUNT_ID }}.r2.cloudflarestorage.com
+          R2_BUCKET: ${{ secrets.CLOUDFLARE_R2_BUILDS_BUCKET }}
+          R2_ACCESS_KEY: ${{ secrets.CLOUDFLARE_R2_ACCESS_KEY_ID }}
+          R2_SECRET_KEY: ${{ secrets.CLOUDFLARE_R2_SECRET_ACCESS_KEY }}
+        run: |
+          RUNTIME="${{ inputs.runtime || 'all' }}"
+          ./scripts/generate-manifests-from-r2/generate-manifests \
+            --runtime="$RUNTIME" \
+            --output-dir=src/internal/manifest/data \
+            --base-url="https://builds.dtvem.io" \
+            --r2-endpoint="$R2_ENDPOINT" \
+            --r2-bucket="$R2_BUCKET" \
+            --r2-access-key="$R2_ACCESS_KEY" \
+            --r2-secret-key="$R2_SECRET_KEY" \
+            --dry-run
+
+      - name: Generate manifests
+        if: ${{ github.event_name != 'workflow_dispatch' || !inputs.dry_run }}
+        env:
+          R2_ENDPOINT: https://${{ secrets.CLOUDFLARE_ACCOUNT_ID }}.r2.cloudflarestorage.com
+          R2_BUCKET: ${{ secrets.CLOUDFLARE_R2_BUILDS_BUCKET }}
+          R2_ACCESS_KEY: ${{ secrets.CLOUDFLARE_R2_ACCESS_KEY_ID }}
+          R2_SECRET_KEY: ${{ secrets.CLOUDFLARE_R2_SECRET_ACCESS_KEY }}
+        run: |
+          RUNTIME="${{ inputs.runtime || 'all' }}"
+          ./scripts/generate-manifests-from-r2/generate-manifests \
+            --runtime="$RUNTIME" \
+            --output-dir=src/internal/manifest/data \
+            --base-url="https://builds.dtvem.io" \
+            --r2-endpoint="$R2_ENDPOINT" \
+            --r2-bucket="$R2_BUCKET" \
+            --r2-access-key="$R2_ACCESS_KEY" \
+            --r2-secret-key="$R2_SECRET_KEY"
+
+      - name: Deploy manifests to R2
+        if: ${{ github.event_name != 'workflow_dispatch' || !inputs.dry_run }}
+        env:
+          R2_ENDPOINT: https://${{ secrets.CLOUDFLARE_ACCOUNT_ID }}.r2.cloudflarestorage.com
+          R2_BUCKET: ${{ secrets.CLOUDFLARE_R2_MANIFESTS_BUCKET }}
+        run: |
+          aws configure set aws_access_key_id ${{ secrets.CLOUDFLARE_R2_ACCESS_KEY_ID }}
+          aws configure set aws_secret_access_key ${{ secrets.CLOUDFLARE_R2_SECRET_ACCESS_KEY }}
+          aws configure set default.region auto
+
+          echo "Deploying manifests to R2..."
+          for file in src/internal/manifest/data/*.json; do
+            filename=$(basename "$file")
+            echo "Uploading $filename..."
+            aws s3 cp "$file" "s3://${R2_BUCKET}/${filename}" \
+              --endpoint-url "${R2_ENDPOINT}" \
+              --content-type "application/json" \
+              --cache-control "public, max-age=300"
+          done
+          echo "Manifests deployed to R2!"
+
+      - name: Check for changes
+        id: check-changes
+        if: ${{ github.event_name != 'workflow_dispatch' || !inputs.dry_run }}
+        run: |
+          git diff --quiet src/internal/manifest/data/ || echo "changed=true" >> $GITHUB_OUTPUT
+
+      - name: Create Pull Request
+        if: ${{ steps.check-changes.outputs.changed == 'true' }}
+        uses: peter-evans/create-pull-request@v7
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          commit-message: 'chore(manifest): regenerate manifests from R2'
+          title: 'chore(manifest): regenerate manifests from R2'
+          body: |
+            Regenerated manifests from binaries hosted on `builds.dtvem.io`.
+
+            This PR was created by the [Generate Manifests from R2 workflow](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}).
+
+            ## Changes
+            - URLs now point to `builds.dtvem.io` (our hosted binaries)
+            - Includes `sha256_source` field indicating checksum origin ("upstream" or "dtvem")
+
+            Please review the changes before merging.
+          branch: chore/regenerate-manifests-from-r2
+          delete-branch: true
+          labels: |
+            automated
+            manifest
+
+      - name: Generate summary
+        if: always()
+        run: |
+          echo "## Manifest Generation" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          if [ "${{ inputs.dry_run }}" = "true" ]; then
+            echo "**Mode:** Dry run (no changes made)" >> $GITHUB_STEP_SUMMARY
+          elif [ "${{ steps.check-changes.outputs.changed }}" = "true" ]; then
+            echo "**Result:** Changes detected" >> $GITHUB_STEP_SUMMARY
+            echo "- Manifests deployed to R2 (live immediately)" >> $GITHUB_STEP_SUMMARY
+            echo "- PR created for embedded manifests" >> $GITHUB_STEP_SUMMARY
+          else
+            echo "**Result:** No changes detected" >> $GITHUB_STEP_SUMMARY
+          fi

.github/workflows/mirror-all.yml

@@ -0,0 +1,80 @@
+name: Mirror All Binaries
+
+on:
+  workflow_dispatch:
+    inputs:
+      runtime:
+        description: 'Runtime to mirror (node, python, ruby, or all)'
+        required: true
+        default: 'all'
+        type: choice
+        options:
+          - all
+          - node
+          - python
+          - ruby
+      dry_run:
+        description: 'Dry run (report only, no uploads)'
+        required: false
+        default: false
+        type: boolean
+
+jobs:
+  mirror:
+    name: Mirror ${{ matrix.runtime }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 360  # 6 hours max
+    strategy:
+      fail-fast: false
+      matrix:
+        runtime: ${{ inputs.runtime == 'all' && fromJson('["node", "python", "ruby"]') || fromJson(format('["{0}"]', inputs.runtime)) }}
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Go
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: 'go.mod'
+
+      - name: Build mirror tool
+        run: |
+          cd scripts/mirror-binaries
+          go build -o mirror-binaries .
+
+      - name: Mirror binaries (dry run)
+        if: inputs.dry_run
+        run: |
+          ./scripts/mirror-binaries/mirror-binaries \
+            --runtime=${{ matrix.runtime }} \
+            --manifest-dir=src/internal/manifest/data \
+            --dry-run
+
+      - name: Mirror binaries
+        if: ${{ !inputs.dry_run }}
+        env:
+          R2_ENDPOINT: https://${{ secrets.CLOUDFLARE_ACCOUNT_ID }}.r2.cloudflarestorage.com
+          R2_BUCKET: ${{ secrets.CLOUDFLARE_R2_BUILDS_BUCKET }}
+          R2_ACCESS_KEY: ${{ secrets.CLOUDFLARE_R2_ACCESS_KEY_ID }}
+          R2_SECRET_KEY: ${{ secrets.CLOUDFLARE_R2_SECRET_ACCESS_KEY }}
+        run: |
+          ./scripts/mirror-binaries/mirror-binaries \
+            --runtime=${{ matrix.runtime }} \
+            --manifest-dir=src/internal/manifest/data \
+            --r2-endpoint="$R2_ENDPOINT" \
+            --r2-bucket="$R2_BUCKET" \
+            --r2-access-key="$R2_ACCESS_KEY" \
+            --r2-secret-key="$R2_SECRET_KEY" \
+            --workers=20
+
+      - name: Generate summary
+        if: always()
+        run: |
+          echo "## Mirror Results for ${{ matrix.runtime }}" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          if [ "${{ inputs.dry_run }}" = "true" ]; then
+            echo "**Mode:** Dry run (no uploads)" >> $GITHUB_STEP_SUMMARY
+          else
+            echo "**Mode:** Live upload to R2" >> $GITHUB_STEP_SUMMARY
+          fi

.github/workflows/mirror-sync.yml

@@ -0,0 +1,69 @@
+name: Mirror Sync
+
+on:
+  # Run daily to catch new upstream versions
+  schedule:
+    - cron: '0 4 * * *'  # Every day at 4 AM UTC
+  # Manual trigger
+  workflow_dispatch:
+    inputs:
+      runtime:
+        description: 'Runtime to sync (node, python, ruby, or all)'
+        required: true
+        default: 'all'
+        type: choice
+        options:
+          - all
+          - node
+          - python
+          - ruby
+
+jobs:
+  sync:
+    name: Sync ${{ matrix.runtime }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 180  # 3 hours max for sync
+    strategy:
+      fail-fast: false
+      matrix:
+        runtime: ${{ (github.event_name == 'workflow_dispatch' && inputs.runtime != 'all') && fromJson(format('["{0}"]', inputs.runtime)) || fromJson('["node", "python", "ruby"]') }}
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Go
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: 'go.mod'
+
+      - name: Build mirror tool
+        run: |
+          cd scripts/mirror-binaries
+          go build -o mirror-binaries .
+
+      - name: Sync new binaries to R2
+        env:
+          R2_ENDPOINT: https://${{ secrets.CLOUDFLARE_ACCOUNT_ID }}.r2.cloudflarestorage.com
+          R2_BUCKET: ${{ secrets.CLOUDFLARE_R2_BUILDS_BUCKET }}
+          R2_ACCESS_KEY: ${{ secrets.CLOUDFLARE_R2_ACCESS_KEY_ID }}
+          R2_SECRET_KEY: ${{ secrets.CLOUDFLARE_R2_SECRET_ACCESS_KEY }}
+        run: |
+          ./scripts/mirror-binaries/mirror-binaries \
+            --runtime=${{ matrix.runtime }} \
+            --manifest-dir=src/internal/manifest/data \
+            --r2-endpoint="$R2_ENDPOINT" \
+            --r2-bucket="$R2_BUCKET" \
+            --r2-access-key="$R2_ACCESS_KEY" \
+            --r2-secret-key="$R2_SECRET_KEY" \
+            --sync-only \
+            --workers=10
+
+      - name: Generate summary
+        if: always()
+        run: |
+          echo "## Sync Results for ${{ matrix.runtime }}" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "Synced new binaries from upstream that were not already in R2." >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "To generate updated manifests, run the 'Generate Manifests from R2' workflow." >> $GITHUB_STEP_SUMMARY

schemas/manifest.schema.json

@@ -47,7 +47,7 @@
     "download": {
       "type": "object",
       "description": "Download information for a pre-built binary",
-      "required": ["url", "sha256"],
+      "required": ["url"],
       "additionalProperties": false,
       "properties": {
         "url": {
@@ -59,6 +59,11 @@
           "type": "string",
           "pattern": "^[a-fA-F0-9]{64}$",
           "description": "SHA256 checksum (64 hex characters)"
+        },
+        "sha256_source": {
+          "type": "string",
+          "enum": ["upstream", "dtvem"],
+          "description": "Origin of the SHA256 checksum: 'upstream' if from the original provider, 'dtvem' if generated by us during mirroring"
         }
       }
     }
@@ -69,13 +74,15 @@
       "versions": {
         "3.13.1": {
           "windows-amd64": {
-            "url": "https://www.python.org/ftp/python/3.13.1/python-3.13.1-embed-amd64.zip",
-            "sha256": "a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2"
+            "url": "https://builds.dtvem.io/python/3.13.1/windows-amd64.zip",
+            "sha256": "a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2",
+            "sha256_source": "upstream"
           },
           "darwin-arm64": null,
           "linux-amd64": {
-            "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20251209/cpython-3.13.1.tar.gz",
-            "sha256": "b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3"
+            "url": "https://builds.dtvem.io/python/3.13.1/linux-amd64.tar.gz",
+            "sha256": "b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3",
+            "sha256_source": "dtvem"
           }
         }
       }

scripts/generate-manifests-from-r2/go.mod

@@ -0,0 +1,27 @@
+module github.com/dtvem/dtvem/scripts/generate-manifests-from-r2
+
+go 1.23.0
+
+require (
+	github.com/aws/aws-sdk-go-v2 v1.32.6
+	github.com/aws/aws-sdk-go-v2/config v1.28.6
+	github.com/aws/aws-sdk-go-v2/credentials v1.17.47
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.71.0
+)
+
+require (
+	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.21 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.25 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.25 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.25 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.6 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.6 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.6 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.24.7 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.6 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.33.2 // indirect
+	github.com/aws/smithy-go v1.22.1 // indirect
+)