python-app.yaml

name: Python Package CI/CD

on:
  push:
    branches: [main]        # Only main branch
    tags: ['v*.*.*']
  pull_request:
    branches: [main, 'dev*', 'dev/**', 'feature/**']
    types: [opened, synchronize, reopened]
    paths-ignore:
      - 'docs/**'
      - '*.md'
      - 'README*'
  workflow_dispatch:  # Allow manual triggering

# Set permissions for security
permissions:
  contents: read

# Cancel previous runs on new push
concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

env:
  PYTHON_VERSION: "3.11"

jobs:
  lint:
    runs-on: ubuntu-24.04
    steps:
      - name: Check out code
        uses: actions/checkout@v5

      - name: Set up uv
        uses: astral-sh/setup-uv@v6
        with:
          version: "0.8.19"
          enable-cache: true

      - name: Set up Python
        uses: actions/setup-python@v6
        with:
          python-version: ${{ env.PYTHON_VERSION }}

      - name: Install Ruff
        run: |
          uvx ruff --version

      - name: Run Ruff Linting
        run: |
          echo "::group::Ruff Linting"
          uvx ruff check . --output-format=github
          echo "::endgroup::"

      - name: Run Ruff Formatting Check
        run: |
          echo "::group::Ruff Formatting"
          uvx ruff format --check --diff .
          echo "::endgroup::"

  test:
    runs-on: ubuntu-24.04
    timeout-minutes: 30
    needs: lint  # Run tests only after linting passes
    strategy:
      fail-fast: false  # Continue testing other Python versions if one fails
      matrix:
        python-version: ['3.10', '3.11', '3.12', '3.13']

    steps:
      - name: Check out code
        uses: actions/checkout@v5

      - name: Set up uv
        uses: astral-sh/setup-uv@v6
        with:
          version: "0.8.19"
          enable-cache: true

      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v6
        with:
          python-version: ${{ matrix.python-version }}

      - name: Install dependencies
        run: |
          uv pip install --system .[dev]

      - name: Run tests
        run: pytest -v --numprocesses=auto

  test-examples:
    runs-on: ubuntu-24.04
    timeout-minutes: 45
    needs: lint
    # Only run examples on releases (tags)
    if: startsWith(github.ref, 'refs/tags/v')

    steps:
      - name: Check out code
        uses: actions/checkout@v5

      - name: Set up uv
        uses: astral-sh/setup-uv@v6
        with:
          version: "0.8.19"
          enable-cache: true

      - name: Set up Python ${{ env.PYTHON_VERSION }}
        uses: actions/setup-python@v6
        with:
          python-version: ${{ env.PYTHON_VERSION }}

      - name: Install dependencies
        run: |
          uv pip install --system .[dev]

      - name: Run example tests
        run: pytest -v -p no:warnings -m examples

  security:
    name: Security Scan
    runs-on: ubuntu-24.04
    needs: lint
    steps:
      - name: Check out code
        uses: actions/checkout@v5

      - name: Set up uv
        uses: astral-sh/setup-uv@v6
        with:
          version: "0.8.19"
          enable-cache: true

      - name: Set up Python
        uses: actions/setup-python@v6
        with:
          python-version: ${{ env.PYTHON_VERSION }}

      - name: Run Bandit security scan
        run: |
          # Gate on HIGH severity & MEDIUM confidence; produce JSON artifact
          uvx bandit -r flixopt/ -c pyproject.toml -f json -o bandit-report.json -q --severity-level high --confidence-level medium
          # Human-readable output without affecting job status
          uvx bandit -r flixopt/ -c pyproject.toml -q --exit-zero

      - name: Upload security reports
        uses: actions/upload-artifact@v4
        if: always()
        with:
          name: security-report
          path: bandit-report.json
          retention-days: 30

  create-release:
    name: Create GitHub Release
    runs-on: ubuntu-24.04
    permissions:
      contents: write
    needs: [lint, test, test-examples, security]
    if: startsWith(github.ref, 'refs/tags/v')

    steps:
      - name: Checkout repository
        uses: actions/checkout@v5
        with:
          fetch-depth: 0

      - name: Set up uv
        uses: astral-sh/setup-uv@v6
        with:
          version: "0.8.19"
          enable-cache: true

      - name: Set up Python
        uses: actions/setup-python@v6
        with:
          python-version: ${{ env.PYTHON_VERSION }}

      - name: Extract release notes
        run: |
          VERSION=${GITHUB_REF#refs/tags/v}
          echo "Extracting release notes for version: $VERSION"
          python scripts/extract_release_notes.py $VERSION > current_release_notes.md

      - name: Create GitHub Release
        uses: softprops/action-gh-release@v2
        with:
          body_path: current_release_notes.md
          draft: false
          prerelease: ${{ contains(github.ref, 'alpha') || contains(github.ref, 'beta') || contains(github.ref, 'rc') }}
          generate_release_notes: true
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

  publish-testpypi:
    name: Publish to TestPyPI
    runs-on: ubuntu-24.04
    needs: [test, test-examples, create-release]  # Run after tests and release creation
    if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')  # Only on tag push
    environment:
      name: testpypi
      url: https://test.pypi.org/p/flixopt
    env:
      SKIP_TESTPYPI_UPLOAD: "false"

    steps:
      - name: Checkout repository
        uses: actions/checkout@v5

      - name: Set up uv
        uses: astral-sh/setup-uv@v6
        with:
          version: "0.8.19"
          enable-cache: true

      - name: Set up Python
        uses: actions/setup-python@v6
        with:
          python-version: ${{ env.PYTHON_VERSION }}

      - name: Install dependencies
        run: |
          uv pip install --system twine

      - name: Build the distribution
        run: |
          uv build

      - name: Upload to TestPyPI
        run: |
          twine upload --repository-url https://test.pypi.org/legacy/ dist/* --verbose --skip-existing
        env:
          TWINE_USERNAME: __token__
          TWINE_PASSWORD: ${{ secrets.TEST_PYPI_API_TOKEN }}
          TWINE_NON_INTERACTIVE: "1"

      - name: Test install from TestPyPI
        if: env.SKIP_TESTPYPI_UPLOAD != 'true'
        run: |
          set -Eeuo pipefail
          # Create a temporary environment to test installation
          uv venv test_env
          source test_env/bin/activate

          # Get project name from pyproject.toml (PEP 621)
          PACKAGE_NAME=$(python - <<'PY'
          import sys, tomllib, pathlib
          data = tomllib.loads(pathlib.Path("pyproject.toml").read_text(encoding="utf-8"))
          print(data["project"]["name"])
          PY
          )

          # Extract version from git tag
          VERSION=${GITHUB_REF#refs/tags/v}

          # Wait and retry while TestPyPI indexes the package
          INSTALL_SUCCESS=false
          for d in 10 20 40 80 120; do
            sleep "$d"
            echo "Attempting to install $PACKAGE_NAME==$VERSION from TestPyPI (retry after ${d}s)..."

            # Install specific version and verify it matches
            if uv pip install --index-strategy unsafe-best-match --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple/ "$PACKAGE_NAME==$VERSION" && \
               python -c "from importlib.metadata import version; installed = version('$PACKAGE_NAME'); print(f'Installed: {installed}'); assert '$VERSION' == installed"; then
              INSTALL_SUCCESS=true
              break
            fi
          done

          # Check if installation succeeded
          if [ "$INSTALL_SUCCESS" = "false" ]; then
            echo "ERROR: Failed to install $PACKAGE_NAME==$VERSION from TestPyPI after all retries"
            echo "This could indicate:"
            echo "  - TestPyPI indexing issues"
            echo "  - Package upload problems"
            echo "  - Version mismatch between tag and package"
            exit 1
          fi

          # Final success confirmation
          python -c "import flixopt; print('TestPyPI installation successful!')"

  publish-pypi:
    name: Publish to PyPI
    runs-on: ubuntu-24.04
    needs: [publish-testpypi]  # Only run after TestPyPI publish succeeds
    if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')  # Only on tag push
    environment:
      name: pypi
      url: https://pypi.org/p/flixopt

    steps:
      - name: Checkout repository
        uses: actions/checkout@v5

      - name: Set up uv
        uses: astral-sh/setup-uv@v6
        with:
          version: "0.8.19"
          enable-cache: true

      - name: Set up Python
        uses: actions/setup-python@v6
        with:
          python-version: ${{ env.PYTHON_VERSION }}

      - name: Install dependencies
        run: |
          uv pip install --system twine

      - name: Build the distribution
        run: |
          uv build

      - name: Upload to PyPI
        run: |
          twine upload dist/* --verbose --skip-existing
        env:
          TWINE_USERNAME: __token__
          TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }}
          TWINE_NON_INTERACTIVE: "1"

      - name: Verify PyPI installation
        run: |
          set -Eeuo pipefail
          # Create a temporary environment to test installation
          uv venv prod_test_env
          source prod_test_env/bin/activate

          # Get project name from pyproject.toml (PEP 621)
          PACKAGE_NAME=$(python - <<'PY'
          import sys, tomllib, pathlib
          data = tomllib.loads(pathlib.Path("pyproject.toml").read_text(encoding="utf-8"))
          print(data["project"]["name"])
          PY
          )

          # Extract version from git tag
          VERSION=${GITHUB_REF#refs/tags/v}

          # Wait and retry while PyPI indexes the package
          INSTALL_SUCCESS=false
          for d in 10 20 40 80 120 240 480; do
            sleep "$d"
            echo "Attempting to install $PACKAGE_NAME==$VERSION from PyPI (retry after ${d}s)..."

            # Install specific version and verify it matches
            if uv pip install "$PACKAGE_NAME==$VERSION" && \
               python -c "from importlib.metadata import version; installed = version('$PACKAGE_NAME'); print(f'Installed: {installed}'); assert '$VERSION' == installed"; then
              INSTALL_SUCCESS=true
              break
            fi
          done

          # Check if installation succeeded
          if [ "$INSTALL_SUCCESS" = "false" ]; then
            echo "ERROR: Failed to install $PACKAGE_NAME==$VERSION from PyPI after all retries"
            echo "This could indicate:"
            echo "  - PyPI indexing issues"
            echo "  - Package upload problems"
            echo "  - Version mismatch between tag and package"
            exit 1
          fi

          # Final success confirmation
          python -c "import flixopt; print('PyPI installation successful!')"

  deploy-docs:
    name: Deploy Documentation
    runs-on: ubuntu-24.04
    permissions:
      contents: write
    needs: [publish-pypi]  # Deploy docs after successful PyPI publishing
    if: startsWith(github.ref, 'refs/tags/v') && !contains(github.ref, 'alpha') && !contains(github.ref, 'beta') && !contains(github.ref, 'rc')

    steps:
      - name: Checkout repository
        uses: actions/checkout@v5
        with:
          fetch-depth: 0  # Fetch all history for proper versioning

      - name: Set up uv
        uses: astral-sh/setup-uv@v6
        with:
          version: "0.8.19"
          enable-cache: true

      - name: Set up Python
        uses: actions/setup-python@v6
        with:
          python-version: ${{ env.PYTHON_VERSION }}

      - name: Extract changelog to docs
        run: |
          # Install packaging dependency for changelog extraction
          uv pip install --system packaging

          # Extract individual release files
          python scripts/extract_changelog.py

          echo "✅ Extracted changelog to docs/changelog/"

      - name: Install documentation dependencies
        run: |
          uv pip install --system ".[docs]"

      - name: Configure Git Credentials
        run: |
          git config user.name github-actions[bot]
          git config user.email 41898282+github-actions[bot]@users.noreply.github.com

      - name: Deploy docs
        run: |
          VERSION=${GITHUB_REF#refs/tags/v}
          echo "Deploying docs after successful PyPI publish: $VERSION"
          mike deploy --push --update-aliases $VERSION latest
          mike set-default --push latest