deploy-droplet.yml

# Deploy to DigitalOcean Droplet
# Deploys the application to a DigitalOcean Droplet via rsync + SSH
# Requires UDP ports open for BitTorrent DHT to work
#
# This workflow runs AFTER the CI workflow completes successfully
# to avoid duplicate test runs

name: Deploy to Droplet

on:
  # Run after CI workflow completes successfully
  workflow_run:
    workflows: ["CI Pipeline"]
    types: [completed]
    branches: [main, master]
  workflow_dispatch: # Allow manual trigger

concurrency:
  group: deploy-droplet
  cancel-in-progress: true

env:
  # Deploy path: /home/ubuntu/www/:domain/:repo
  DEPLOY_PATH: /home/ubuntu/www/bittorrented.com/media-streamer
  SERVICE_NAME: bittorrented
  # Run actions/checkout etc. on Node 24 instead of their bundled Node 20.
  FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: 'true'

jobs:
  # Deploy to Droplet (only if CI passed)
  deploy:
    name: Deploy to Droplet
    runs-on: ubuntu-latest
    # Only deploy if CI workflow succeeded (or manual trigger)
    if: |
      github.event_name == 'workflow_dispatch' ||
      (github.event.workflow_run.conclusion == 'success' &&
       (github.event.workflow_run.head_branch == 'main' || github.event.workflow_run.head_branch == 'master'))
    steps:
      - name: Checkout code
        uses: actions/checkout@v7

      - name: Write .env file
        env:
          ENV_FILE_CONTENT: ${{ secrets.ENV_FILE }}
        run: |
          printf '%s\n' "$ENV_FILE_CONTENT" > .env

      - name: Write proxies.txt
        env:
          PROXIES_LIST: ${{ secrets.PROXIES_LIST }}
        run: |
          # Static residential proxy list (host:port:user:pass per line) used
          # by the SiriusXM client to pin one upstream IP per user. File is
          # gitignored locally; rsync ships it to the droplet alongside .env.
          if [ -n "$PROXIES_LIST" ]; then
            printf '%s\n' "$PROXIES_LIST" > proxies.txt
            echo "wrote proxies.txt ($(wc -l < proxies.txt) lines)"
          else
            echo "PROXIES_LIST secret is empty; skipping proxies.txt"
          fi

      - name: Setup SSH key
        run: |
          mkdir -p ~/.ssh
          echo "${{ secrets.DROPLET_SSH_KEY }}" > ~/.ssh/deploy_key
          chmod 600 ~/.ssh/deploy_key
          
          # Add host to known_hosts (with retry for transient failures)
          for i in 1 2 3; do
            ssh-keyscan -p ${{ secrets.DROPLET_PORT || 22 }} -H ${{ secrets.DROPLET_HOST }} >> ~/.ssh/known_hosts 2>/dev/null && break
            sleep 2
          done

      - name: Sync files to Droplet (with retry)
        uses: nick-fields/retry@v4
        with:
          timeout_minutes: 15
          max_attempts: 5
          retry_wait_seconds: 5
          command: |
            SSH_OPTS="-i ~/.ssh/deploy_key -p ${{ secrets.DROPLET_PORT || 22 }} -o StrictHostKeyChecking=no -o ConnectTimeout=60 -o ServerAliveInterval=30 -o ServerAliveCountMax=5"
            
            # Create deploy directory if it doesn't exist
            ssh $SSH_OPTS ${{ secrets.DROPLET_USER }}@${{ secrets.DROPLET_HOST }} "mkdir -p ${{ env.DEPLOY_PATH }}"
            
            # Sync files
            rsync -azv --partial --progress --delete \
              --exclude='.git' \
              --exclude='node_modules' \
              --exclude='.next' \
              --exclude='.env.local' \
              --exclude='*.log' \
              -e "ssh $SSH_OPTS" \
              ./ ${{ secrets.DROPLET_USER }}@${{ secrets.DROPLET_HOST }}:${{ env.DEPLOY_PATH }}/

      - name: Start build on Droplet (background)
        run: |
          SSH_OPTS="-i ~/.ssh/deploy_key -p ${{ secrets.DROPLET_PORT || 22 }} -o StrictHostKeyChecking=no -o ConnectTimeout=60"
          
          # Create the build script on the server
          ssh $SSH_OPTS ${{ secrets.DROPLET_USER }}@${{ secrets.DROPLET_HOST }} << 'SETUP_SCRIPT'
          cat > /tmp/deploy-build.sh << 'BUILD_SCRIPT'
          #!/bin/bash
          set -e
          
          LOG_FILE="/tmp/deploy-build.log"
          STATUS_FILE="/tmp/deploy-build.status"
          
          # Clear previous status
          echo "running" > "$STATUS_FILE"
          
          {
            cd /home/ubuntu/www/bittorrented.com/media-streamer
            
            # Run idempotent setup script
            echo "=== Running setup script ==="
            bash scripts/setup-server.sh
            
            # Source profile to get pnpm in PATH
            export PNPM_HOME="$HOME/.local/share/pnpm"
            export PATH="$PNPM_HOME:$HOME/.pnpm:$HOME/.npm-global/bin:/usr/local/bin:$PATH"
            [ -f ~/.bashrc ] && source ~/.bashrc 2>/dev/null || true
            [ -f ~/.profile ] && source ~/.profile 2>/dev/null || true
            
            echo ""
            echo "=== Environment ==="
            echo "PATH: $PATH"
            echo "Node: $(node --version 2>/dev/null || echo 'not found')"
            echo "pnpm: $(which pnpm 2>/dev/null || echo 'not found')"
            
            echo ""
            echo "=== System resources ==="
            free -h
            df -h /
            
            echo ""
            echo "=== Cleaning build cache ==="
            # Acquire exclusive lock — wait for any previous deploy to finish
            exec 200>/tmp/deploy-build.lock
            flock -w 300 200 || { echo "✗ Lock timeout after 300s"; echo "failed" > "$STATUS_FILE"; exit 1; }
            
            # Kill any lingering next build processes
            pkill -f "next build" 2>/dev/null || true
            sleep 1
            rm -rf .next 2>/dev/null || true
            echo "✓ Build cache cleaned"
            
            echo ""
            echo "=== Installing dependencies ==="
            rm -rf node_modules
            pnpm install --frozen-lockfile
            echo "✓ Dependencies installed"
            
            echo ""
            echo "=== Building application ==="
            nice -n 10 pnpm build || { echo "✗ Build FAILED"; echo "failed" > "$STATUS_FILE"; exit 1; }
            echo "✓ Build complete"
            
            echo ""
            # Verify build output exists before restarting.
            # Runtime is `next start` (not standalone), so check for .next/BUILD_ID.
            if [ ! -f .next/BUILD_ID ]; then
              echo "✗ Build output missing (.next/BUILD_ID not found)"
              echo "failed" > "$STATUS_FILE"
              exit 1
            fi
            echo "=== Restarting services ==="
            # The on-disk unit is stale (a hand-edited base unit with an
            # EnvironmentFile that no longer exists + an ExecStart pointing at a pnpm
            # path removed by the mise migration), and the service has hit systemd's
            # start limiter from crash-looping. A drop-in can't override the base
            # unit's EnvironmentFile, so REWRITE the whole base unit cleanly with
            # absolute pnpm/node paths discovered from this (mise) deploy shell,
            # drop any stale drop-ins, reset the failure limiter, then restart.
            PNPM_BIN="$(command -v pnpm || true)"
            NODE_BIN="$(command -v node || true)"
            echo "Resolved pnpm=${PNPM_BIN} node=${NODE_BIN}"
            sudo rm -rf /etc/systemd/system/bittorrented.service.d
            if [ -n "$PNPM_BIN" ] && [ -n "$NODE_BIN" ]; then
              NODE_DIR="$(dirname "$NODE_BIN")"
              PNPM_DIR="$(dirname "$PNPM_BIN")"
              APP_DIR="$(pwd)"
              printf '[Unit]\nDescription=BitTorrented Media Streamer\nAfter=network.target\n\n[Service]\nType=simple\nUser=%s\nWorkingDirectory=%s\nExecStart=%s start\nRestart=on-failure\nRestartSec=10\nStartLimitIntervalSec=0\nEnvironment=NODE_ENV=production\nEnvironment=PATH=%s:%s:/usr/local/bin:/usr/bin:/bin\nEnvironment=NODE_OPTIONS=--max-old-space-size=4096 --expose-gc\nStandardOutput=append:/var/log/bittorrented.com.log\nStandardError=append:/var/log/bittorrented.com.error.log\nLimitNOFILE=65535\n\n[Install]\nWantedBy=multi-user.target\n' \
                "$(whoami)" "$APP_DIR" "$PNPM_BIN" "$NODE_DIR" "$PNPM_DIR" \
                | sudo tee /etc/systemd/system/bittorrented.service > /dev/null
              echo "--- wrote clean unit ---"; cat /etc/systemd/system/bittorrented.service
            else
              echo "WARNING: could not resolve pnpm/node; leaving unit as-is"
            fi
            sudo systemctl daemon-reload || true
            sudo systemctl reset-failed bittorrented || true
            sudo systemctl restart bittorrented || echo "Warning: Could not restart main service"
            echo "✓ Main service restart attempted"

            sudo systemctl reset-failed bittorrented-iptv-worker || true
            sudo systemctl restart bittorrented-iptv-worker || echo "Warning: Could not restart IPTV worker"
            echo "✓ IPTV worker restart attempted"

            # The podcast worker is a long-running process that is NOT recreated by
            # `next start`; if it crash-loops (e.g. the mise/pnpm path drift) it hits
            # systemd's start limiter and stays `failed` forever because nothing here
            # restarts it. Reset the limiter and restart it on every deploy.
            sudo systemctl reset-failed bittorrented-podcast-worker || true
            sudo systemctl restart bittorrented-podcast-worker || echo "Warning: Could not restart podcast worker"
            echo "✓ Podcast worker restart attempted"

            echo ""
            echo "=== Verifying deployment ==="
            # Give the unit a moment to either become active or crash so the
            # journal has the real reason.
            sleep 12
            if ! systemctl is-active --quiet bittorrented; then
              echo "✗ Main service is not active — dumping diagnostics:"
              echo "----- systemctl status -----"
              sudo systemctl status bittorrented --no-pager -l 2>&1 | head -40 || true
              echo "----- journalctl -xeu (last 60) -----"
              sudo journalctl -xeu bittorrented -n 60 --no-pager 2>&1 | tail -60 || true
              echo "----- app error log (last 40) -----"
              tail -40 "${ERROR_LOG_FILE}" 2>/dev/null || true
              echo "failed" > "$STATUS_FILE"; exit 1
            fi
            systemctl is-active bittorrented-iptv-worker || echo "IPTV worker status unknown"
            systemctl is-active bittorrented-podcast-worker || echo "Podcast worker status unknown"

            echo "Waiting for HTTP health check..."
            HEALTH_OK=false
            for attempt in {1..30}; do
              if curl -fsS --max-time 5 http://127.0.0.1:3000/api/health >/tmp/bittorrented-health.json; then
                HEALTH_OK=true
                echo "✓ HTTP health check passed on attempt ${attempt}"
                cat /tmp/bittorrented-health.json
                break
              fi

              echo "Health check not ready yet (${attempt}/30)"
              sleep 2
            done

            if [ "$HEALTH_OK" != "true" ]; then
              echo "✗ HTTP health check failed after restart"
              systemctl status bittorrented --no-pager || true
              tail -80 /var/log/bittorrented.com.error.log || true
              echo "failed" > "$STATUS_FILE"
              exit 1
            fi
            
            echo ""
            echo "=== Deployment complete! ==="
            echo "success" > "$STATUS_FILE"
          } >> "$LOG_FILE" 2>&1 || {
            echo "failed" > "$STATUS_FILE"
            exit 1
          }
          BUILD_SCRIPT
          chmod +x /tmp/deploy-build.sh
          SETUP_SCRIPT
          
          # Start the build in background using nohup
          ssh $SSH_OPTS ${{ secrets.DROPLET_USER }}@${{ secrets.DROPLET_HOST }} \
            "nohup /tmp/deploy-build.sh > /dev/null 2>&1 &"
          
          echo "Build started in background on droplet"

      - name: Wait for build to complete
        run: |
          SSH_OPTS="-i ~/.ssh/deploy_key -p ${{ secrets.DROPLET_PORT || 22 }} -o StrictHostKeyChecking=no -o ConnectTimeout=30"
          
          echo "Waiting for build to complete..."
          MAX_WAIT=600  # 10 minutes
          ELAPSED=0
          POLL_INTERVAL=10
          
          while [ $ELAPSED -lt $MAX_WAIT ]; do
            # Check build status
            STATUS=$(ssh $SSH_OPTS ${{ secrets.DROPLET_USER }}@${{ secrets.DROPLET_HOST }} \
              "cat /tmp/deploy-build.status 2>/dev/null || echo 'unknown'" || echo "ssh_error")
            
            if [ "$STATUS" = "success" ]; then
              echo "✓ Build completed successfully!"
              # Show the log
              ssh $SSH_OPTS ${{ secrets.DROPLET_USER }}@${{ secrets.DROPLET_HOST }} \
                "tail -50 /tmp/deploy-build.log" || true
              exit 0
            elif [ "$STATUS" = "failed" ]; then
              echo "✗ Build failed!"
              ssh $SSH_OPTS ${{ secrets.DROPLET_USER }}@${{ secrets.DROPLET_HOST }} \
                "cat /tmp/deploy-build.log" || true
              exit 1
            elif [ "$STATUS" = "ssh_error" ]; then
              echo "SSH connection failed, retrying in ${POLL_INTERVAL}s..."
            else
              echo "Build still running... (${ELAPSED}s elapsed)"
            fi
            
            sleep $POLL_INTERVAL
            ELAPSED=$((ELAPSED + POLL_INTERVAL))
          done
          
          echo "✗ Build timed out after ${MAX_WAIT}s"
          ssh $SSH_OPTS ${{ secrets.DROPLET_USER }}@${{ secrets.DROPLET_HOST }} \
            "cat /tmp/deploy-build.log" || true
          exit 1

      - name: Cleanup SSH key
        if: always()
        run: rm -f ~/.ssh/deploy_key