documentation.yml

name: Code Coverage and Documentation

on:
  push:
  pull_request:
    types:
      - closed
    branches:
      - main

jobs:
  run:
    name: Code Coverage and Documentation on ${{ matrix.os }}-${{ matrix.python }}
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        os: [ubuntu-latest]
        python: ['3.13']

    steps:
      - uses: actions/checkout@v3

      - uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python }}

      - name: install graphviz
        uses: ts-graphviz/setup-graphviz@v2

      - name: Install pandoc
        run: sudo apt-get install -y pandoc

      - name: Install pytorch
        run: python -m pip install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/cpu

      # - name: Install numpy, onnxruntime-training
      #   run: python -m pip install numpy onnxruntime-training

      - name: Install requirements
        run: python -m pip install -r requirements.txt

      - name: Install requirements dev
        run: python -m pip install -r requirements-dev.txt

      - name: Uninstall onnx-diagnostic
        run: python -m pip uninstall -y onnx-diagnostic

      - name: Uninstall onnx and install onnx-weekly
        run: |
          python -m pip uninstall -y onnx
          python -m pip install onnx-weekly

      - name: pip freeze
        run: python -m pip freeze

      - name: tiny-llm torch.export.export
        run: PYTHONPATH=. python _unittests/ut_torch_models/test_tiny_llms.py

      - name: tiny-llm onnx
        run: PYTHONPATH=. python _unittests/ut_torch_models/test_tiny_llms_onnx.py
        continue-on-error: true

      - name: tiny-llm example
        run: PYTHONPATH=. python _doc/examples/plot_export_tiny_llm.py
        continue-on-error: true  # connectivity issues

      - name: tiny-llm bypass
        run: PYTHONPATH=. python _doc/examples/plot_export_tiny_llm_patched.py
        continue-on-error: true  # connectivity issues

      - name: run tests bypassed
        run: PYTHONPATH=. python _unittests/ut_torch_models/test_tiny_llms_bypassed.py

      - name: Generate coverage report
        run: |
          pip install pytest pytest-cov          
          PYTHONPATH=. UNITTEST_GOING=1 UNITTEST_DOT=1 pytest --cov=./onnx_diagnostic/ --cov-report=xml --durations=10 _unittests --ignore _unittests/ut_reference/test_backend_extended_reference_evaluator.py  --ignore _unittests/ut_reference/test_backend_onnxruntime_evaluator.py --ignore _unittests/ut_torch_models/test_tiny_llms_bypassed.py

      - name: run backend tests python
        run: PYTHONPATH=. UNITTEST_GOING=1 pytest --durations=10 _unittests/ut_reference/test_backend_extended_reference_evaluator.py

      - name: run backend tests onnxruntime
        run: PYTHONPATH=. UNITTEST_GOING=1 pytest --durations=10 _unittests/ut_reference/test_backend_onnxruntime_evaluator.py --maxfail=15

      - name: Upload coverage reports to Codecov
        uses: codecov/codecov-action@v3
        env:
          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

      - name: Install
        run: python -m pip install -e . -v

      - name: Copy license, changelogs
        run: |
          cp LICENSE* ./_doc
          cp CHANGELOGS* ./_doc

      - name: Documentation
        continue-on-error: true
        run: UNITTEST_GOING=1 python -m sphinx ./_doc ./dist/html -n -w doc.txt

      - name: Summary
        run: cat doc.txt

      - name: Check for errors and warnings
        run: |
          if [[ $(grep ERROR doc.txt | grep -v 'l-plot-tiny-llm-export' | grep -v 'Unexpected section title or transition.') ]]; then
            echo "Documentation produces errors."
            grep ERROR doc.txt | grep -v 'l-plot-tiny-llm-export' | grep -v 'Unexpected section title or transition.'
            exit 1
          fi
          if [[ $(grep WARNING doc.txt | grep -v 'l-plot-tiny-llm-export' | grep -v 'Inline emphasis start-string' | grep -v 'Definition list ends without a blank line' | grep -v 'Unexpected section title or transition' | grep -v 'Inline strong start-string' | grep -v 'MambaCache' ) ]]; then
            echo "Documentation produces warnings."
            grep WARNING doc.txt | grep -v 'l-plot-tiny-llm-export' | grep -v 'Inline emphasis start-string' | grep -v 'Definition list ends without a blank line' | grep -v 'Unexpected section title or transition' | grep -v 'Inline strong start-string' | grep -v 'MambaCache'
            exit 1
          fi

      - uses: actions/upload-artifact@v4
        with:
          path: ./dist/html/**