diff --git a/.github/workflows/build-wolfprovider-debian-nightly.yml b/.github/workflows/build-wolfprovider-debian-nightly.yml new file mode 100644 index 00000000..194bde35 --- /dev/null +++ b/.github/workflows/build-wolfprovider-debian-nightly.yml @@ -0,0 +1,66 @@ +name: Nightly Build wolfProvider + +on: + schedule: + # Jenkins: 2AM UTC nightly - Actual nightly build + # GitHub Actions: 3AM UTC nightly - Update build in github + - cron: "0 3 * * *" + workflow_dispatch: + inputs: + wolfssl_ref: + description: 'wolfSSL ref (tag/branch)' + required: false + default: 'v5.8.2-stable' + type: string + openssl_ref: + description: 'OpenSSL ref (tag/branch)' + required: false + default: 'openssl-3.5.2' + type: string + fips_ref: + description: 'Build type' + required: false + default: 'both' + type: choice + options: + - 'both' + - 'FIPS' + - 'non-FIPS' + replace_default: + description: 'Replace default provider' + required: false + default: true + type: boolean + build_packages: + description: 'build fresh via Jenkins (true) or use downloaded packages (false)' + required: false + type: boolean + default: true + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + build_wolfprovider_debian: + name: Build wolfProvider Debian + # Only run Jenkins builds on schedule/dispatch from protected branches + if: ${{ github.event_name != 'pull_request' && github.event_name != 'pull_request_target' }} + uses: ./.github/workflows/build-wolfprovider-debian.yml + strategy: + matrix: + # When manually triggered with 'both', or on schedule, build both FIPS and non-FIPS + # When manually triggered with specific type, only build that type + fips_ref: >- + ${{ (github.event_name == 'schedule' + || github.event.inputs.fips_ref == 'both' + || github.event.inputs.fips_ref == '') + && fromJSON('["FIPS", "non-FIPS"]') + || fromJSON(format('["{0}"]', github.event.inputs.fips_ref)) }} + with: + wolfssl_ref: ${{ github.event.inputs.wolfssl_ref || 'v5.8.2-stable' }} + openssl_ref: ${{ github.event.inputs.openssl_ref || 'openssl-3.5.2' }} + fips_ref: ${{ matrix.fips_ref }} + replace_default: ${{ github.event.inputs.replace_default != 'false' }} + build_packages: ${{ github.event.inputs.build_packages == 'true' }} + secrets: inherit diff --git a/.github/workflows/build-wolfprovider-debian.yml b/.github/workflows/build-wolfprovider-debian.yml new file mode 100644 index 00000000..336162a8 --- /dev/null +++ b/.github/workflows/build-wolfprovider-debian.yml @@ -0,0 +1,380 @@ +name: Build wolfProvider Debian + +on: + workflow_call: + inputs: + wolfssl_ref: + required: true + type: string + openssl_ref: + required: true + type: string + fips_ref: + required: true + type: string + replace_default: + required: false + type: boolean + default: false + build_packages: + required: false + type: boolean + default: false + +jobs: + build_wolfprovider_debian: + name: Build wolfProvider Debian (${{ inputs.fips_ref }}) + runs-on: ubuntu-22.04 + # Run inside Debian Bookworm to match packaging environment + container: + image: debian:bookworm + env: + DEBIAN_FRONTEND: noninteractive + timeout-minutes: 30 + env: + WOLFSSL_PACKAGES_PATH: /tmp/wolfssl-packages + OPENSSL_PACKAGES_PATH: /tmp/openssl-packages + WOLFPROV_PACKAGES_PATH: /tmp/wolfprov-packages + steps: + # Install git prior to cloning to ensure we have the full repo + # TODO: create a docker with these pre-installed + - name: Install common dependencies + run: | + apt-get update && apt-get install -y --no-install-recommends \ + build-essential \ + devscripts \ + debhelper \ + dh-autoreconf \ + libtool \ + pkg-config \ + git \ + wget \ + curl \ + ca-certificates \ + openssl \ + dpkg-dev \ + lintian \ + fakeroot \ + dh-exec \ + equivs \ + expect \ + xxd \ + libdistro-info-perl \ + autoconf \ + automake \ + quilt \ + patch \ + python3 \ + file \ + gdebi-core \ + jq + + # Step 1: Trigger Jenkins and download wolfSSL packages + - name: Trigger Jenkins build + if: ${{ inputs.build_packages }} + id: trigger + shell: bash + run: | + JOB_URL="https://cloud.wolfssl-test.com/jenkins/job/wolfProvider/job/debian-extraction" + JENKINS_BASE="https://cloud.wolfssl-test.com/jenkins" + + echo "Jenkins URL: $JENKINS_BASE" + echo "Job URL: $JOB_URL" + + # Test authentication first + echo "Testing Jenkins authentication..." + AUTH_TEST=$(curl -s -o /dev/null -w "%{http_code}" \ + "$JENKINS_BASE/api/json" \ + --user "${{ secrets.USERNAME }}:${{ secrets.JENKINS_TOKEN }}") + echo "Auth test status: $AUTH_TEST" + + if [ "$AUTH_TEST" != "200" ]; then + echo "ERROR: Authentication failed (status: $AUTH_TEST)" + exit 1 + fi + + # Get CSRF token (crumb) first + echo "Getting CSRF token..." + CRUMB=$(curl -s \ + "$JENKINS_BASE/crumbIssuer/api/json" \ + --user "${{ secrets.USERNAME }}:${{ secrets.JENKINS_TOKEN }}" | jq -r '.crumb') + + if [ "$CRUMB" = "null" ] || [ -z "$CRUMB" ]; then + echo "ERROR: Failed to get CSRF token" + exit 1 + fi + + # Mask the crumb token to prevent leakage + echo "::add-mask::$CRUMB" + echo "CSRF token obtained" + + # Try simple build first (without parameters) + echo "Trying simple build trigger..." + SIMPLE_URL="${JOB_URL}/build" + echo "Simple URL: $SIMPLE_URL" + + RESPONSE=$(curl -s -X POST -i \ + "$SIMPLE_URL?token=${{ secrets.JENKINS_API_TOKEN }}" \ + --user "${{ secrets.USERNAME }}:${{ secrets.JENKINS_TOKEN }}" \ + -H "Jenkins-Crumb: $CRUMB") + + # Check if simple build worked + HTTP_STATUS=$(echo "$RESPONSE" | grep -i "^HTTP" | head -1) + echo "Simple build status: $HTTP_STATUS" + + # If simple build failed, try with parameters + if [[ "$HTTP_STATUS" == *"500"* ]] || [[ "$HTTP_STATUS" == *"400"* ]]; then + echo "Simple build failed, trying with parameters..." + FULL_JOB_URL="${JOB_URL}/buildWithParameters" + echo "Parameters URL: $FULL_JOB_URL" + + # Try with form data instead of query parameters + RESPONSE=$(curl -s -X POST -i \ + "$FULL_JOB_URL" \ + --user "${{ secrets.USERNAME }}:${{ secrets.JENKINS_TOKEN }}" \ + -H "Jenkins-Crumb: $CRUMB" \ + -H "Content-Type: application/x-www-form-urlencoded" \ + -d "token=${{ secrets.JENKINS_API_TOKEN }}") + fi + + # Show response for debugging + echo "Response headers:" + echo "$RESPONSE" | head -10 + + # Extract Location header (queue item URL) + QUEUE_URL=$(echo "$RESPONSE" | grep -i "^location:" | awk '{print $2}' | tr -d '\r') + + if [ -z "$QUEUE_URL" ]; then + echo "ERROR: Failed to get queue URL from Jenkins" + HTTP_STATUS=$(echo "$RESPONSE" | grep -i "^HTTP" | head -1) + echo "HTTP Status: $HTTP_STATUS" + echo "Response body (first 500 chars):" + echo "$RESPONSE" | tail -n +20 | head -c 500 + exit 1 + fi + + echo "Jenkins job queued successfully" + echo "Queue URL: $QUEUE_URL" + echo "Waiting for job to start..." + + # Wait for job to move from queue to actual build + BUILD_URL="" + for i in {1..30}; do + echo "Checking queue status (attempt $i/30)..." + QUEUE_STATUS=$(curl -s "$QUEUE_URL/api/json" --user "${{ secrets.USERNAME }}:${{ secrets.JENKINS_TOKEN }}" | jq -r '.executable.url // empty') + + if [ -n "$QUEUE_STATUS" ] && [ "$QUEUE_STATUS" != "null" ]; then + BUILD_URL="$QUEUE_STATUS" + echo "Build started: $BUILD_URL" + break + fi + + echo "Still in queue..." + sleep 10 + done + + if [ -z "$BUILD_URL" ]; then + echo "ERROR: Job did not start within 5 minutes" + exit 1 + fi + + # Check status until build finishes + echo "Checking build status..." + while true; do + STATUS=$(curl -s "$BUILD_URL/api/json" --user "${{ secrets.USERNAME }}:${{ secrets.JENKINS_TOKEN }}" | jq -r '.result') + if [[ "$STATUS" == "SUCCESS" ]]; then + echo "Jenkins build succeeded." + echo "build_url=$BUILD_URL" >> $GITHUB_OUTPUT + break + elif [[ "$STATUS" == "FAILURE" ]]; then + echo "Jenkins build failed!" + exit 1 + elif [[ "$STATUS" == "null" ]]; then + echo "Build still running..." + sleep 20 + else + echo "Unknown status: $STATUS" + sleep 10 + fi + done + + - name: Download .deb files from Jenkins + if: ${{ inputs.build_packages }} + shell: bash + run: | + echo "Downloading .deb files from Jenkins build..." + echo "Build URL: ${{ steps.trigger.outputs.build_url }}" + + mkdir -p ./artifacts + cd ./artifacts + + # Get list of artifacts to find real filenames + echo "Getting artifact list..." + ARTIFACTS_JSON=$(curl -s "${{ steps.trigger.outputs.build_url }}/api/json" --user "aidan:${{ secrets.JENKINS_TOKEN }}" | jq '.artifacts') + + # Find all .deb files and their real names + echo "All available artifacts:" + echo "$ARTIFACTS_JSON" | jq -r '.[] | .fileName' | grep '\.deb$' | sort + + # Find FIPS packages + FIPS_PACKAGES=$(echo "$ARTIFACTS_JSON" | jq -r '.[] | select(.fileName | contains("fips") and endswith(".deb")) | .fileName') + NONFIPS_PACKAGES=$(echo "$ARTIFACTS_JSON" | jq -r '.[] | select(.fileName | (contains("fips") | not) and endswith(".deb")) | .fileName') + + echo "" + echo "FIPS packages found:" + echo "$FIPS_PACKAGES" | while read -r file; do + if [ -n "$file" ] && [ "$file" != "null" ]; then + echo " - $file" + fi + done + + echo "" + echo "Non-FIPS packages found:" + echo "$NONFIPS_PACKAGES" | while read -r file; do + if [ -n "$file" ] && [ "$file" != "null" ]; then + echo " - $file" + fi + done + + # Create subdirectories + mkdir -p fips-packages nonfips-packages + + # Download FIPS packages + echo "" + echo "Downloading FIPS packages..." + echo "$FIPS_PACKAGES" | while read -r file; do + if [ -n "$file" ] && [ "$file" != "null" ]; then + echo " Downloading: $file" + curl -u "${{ secrets.USERNAME }}:${{ secrets.JENKINS_TOKEN }}" -L -o "fips-packages/$file" "${{ steps.trigger.outputs.build_url }}/artifact/artifacts/$file" + fi + done + + # Download Non-FIPS packages + echo "" + echo "Downloading Non-FIPS packages..." + echo "$NONFIPS_PACKAGES" | while read -r file; do + if [ -n "$file" ] && [ "$file" != "null" ]; then + echo " Downloading: $file" + curl -u "${{ secrets.USERNAME }}:${{ secrets.JENKINS_TOKEN }}" -L -o "nonfips-packages/$file" "${{ steps.trigger.outputs.build_url }}/artifact/artifacts/$file" + fi + done + + echo "" + echo "Downloaded files:" + echo "FIPS packages:" + ls -lh fips-packages/ 2>/dev/null || echo " No FIPS packages found" + echo "" + echo "Non-FIPS packages:" + ls -lh nonfips-packages/ 2>/dev/null || echo " No Non-FIPS packages found" + + - name: Setup wolfSSL packages from Jenkins + if: ${{ inputs.build_packages }} + run: | + mkdir -p ${{ env.WOLFSSL_PACKAGES_PATH }} + + # Copy packages based on build type + if [ "${{ inputs.fips_ref }}" = "FIPS" ]; then + if [ -d "./artifacts/fips-packages" ] && [ "$(ls -A ./artifacts/fips-packages/*.deb 2>/dev/null)" ]; then + echo "Copying FIPS packages..." + cp ./artifacts/fips-packages/*.deb ${{ env.WOLFSSL_PACKAGES_PATH }}/ + else + echo "ERROR: No FIPS packages found" + exit 1 + fi + else + if [ -d "./artifacts/nonfips-packages" ] && [ "$(ls -A ./artifacts/nonfips-packages/*.deb 2>/dev/null)" ]; then + echo "Copying non-FIPS packages..." + cp ./artifacts/nonfips-packages/*.deb ${{ env.WOLFSSL_PACKAGES_PATH }}/ + else + echo "ERROR: No non-FIPS packages found" + exit 1 + fi + fi + + echo "Jenkins wolfSSL packages available:" + ls -la ${{ env.WOLFSSL_PACKAGES_PATH }} + + - name: Download previously built wolfSSL artifacts + if: ${{ !inputs.build_packages }} + uses: dawidd6/action-download-artifact@v6 + with: + name: debian-packages-${{ inputs.fips_ref }}${{ inputs.replace_default && '-replace-default' || '' }}-${{ inputs.wolfssl_ref }}-${{ inputs.openssl_ref }} + path: ./cached-artifacts + workflow: build-wolfprovider-nightly.yml + workflow_conclusion: success + + - name: Setup cached wolfSSL artifacts + if: ${{ !inputs.build_packages }} + run: | + mkdir -p ${{ env.WOLFSSL_PACKAGES_PATH }} + + # Copy files from downloaded artifact subdirectories + cp ./cached-artifacts/wolfssl-packages/*.deb ${{ env.WOLFSSL_PACKAGES_PATH }}/ + + echo "Cached packages restored:" + ls -la ${{ env.WOLFSSL_PACKAGES_PATH }} + + - name: Install wolfSSL packages + run: | + echo "Installing wolfSSL packages (${{ inputs.fips_ref }})..." + + # Install packages + dpkg -i ${{ env.WOLFSSL_PACKAGES_PATH }}/*.deb || true + + # Fix any dependency issues + apt-get install -f -y + + echo "wolfSSL packages installed successfully:" + dpkg -l | grep wolfssl + + - name: Checkout wolfProvider + uses: actions/checkout@v4 + with: + fetch-depth: 1 + fetch-tags: true + + # Avoid "detected dubious ownership" warning + - name: Ensure the working directory safe + run: | + git config --global --add safe.directory "$GITHUB_WORKSPACE" + + # When running on a fork the upstream tags are not present, so fetch them explicitly + - name: Fetch tags from upstream(for Debian versioning) + run: | + git remote add upstream https://github.com/wolfSSL/wolfProvider.git || true + git fetch upstream --tags --no-recurse-submodules + + - name: Build wolfProvider + run: | + echo "Building ${{ inputs.fips_ref }} wolfProvider..." + WOLFSSL_TAG=${{ inputs.wolfssl_ref }} OPENSSL_TAG=${{ inputs.openssl_ref }} \ + yes "Y" | $GITHUB_WORKSPACE/scripts/build-wolfprovider.sh --debian \ + ${{ inputs.fips_ref == 'FIPS' && '--enable-fips' || '' }} \ + ${{ inputs.replace_default && '--replace-default' || '' }} + + - name: Setup packages directory + run: | + mkdir -p ${{ env.OPENSSL_PACKAGES_PATH }} + cp $GITHUB_WORKSPACE/../openssl*.deb ${{ env.OPENSSL_PACKAGES_PATH }} + cp $GITHUB_WORKSPACE/../libssl*.deb ${{ env.OPENSSL_PACKAGES_PATH }} + + mkdir -p ${{ env.WOLFPROV_PACKAGES_PATH }} + cp $GITHUB_WORKSPACE/../libwolfprov*.deb ${{ env.WOLFPROV_PACKAGES_PATH }} + cp $GITHUB_WORKSPACE/../libwolfprov*.dsc ${{ env.WOLFPROV_PACKAGES_PATH }} + cp $GITHUB_WORKSPACE/../libwolfprov*.tar.gz ${{ env.WOLFPROV_PACKAGES_PATH }} + + printf "Listing packages directory:\n" + ls -la ${{ env.WOLFPROV_PACKAGES_PATH }} + ls -la ${{ env.WOLFSSL_PACKAGES_PATH }} + ls -la ${{ env.OPENSSL_PACKAGES_PATH }} + + # Save all packages as artifacts for consumers + - name: Upload wolfProvider packages + uses: actions/upload-artifact@v4 + with: + name: debian-packages-${{ inputs.fips_ref }}${{ inputs.replace_default && '-replace-default' || '' }}-${{ inputs.wolfssl_ref }}-${{ inputs.openssl_ref }} + path: | + ${{ env.WOLFSSL_PACKAGES_PATH }} + ${{ env.OPENSSL_PACKAGES_PATH }} + ${{ env.WOLFPROV_PACKAGES_PATH }} + retention-days: 1