PyTest Migration (pulp-platform#144)

* Add PyTest-base testing infra for generic platform

* Remove nvidia-pyindex install (not needed anymore)

* Add nvidia channel to the pip config instead of via nvidia-pyindex (depreciated)

* Update banshee.patch

* Fix patch

* Update banshee patch

* Update CI docker

* Update generic platform CI to use PyTest

* Lint and format

* Cleanup pytest.ini

* Apply Calin's comments

* Refactor to better support multiple platforms and add cortexm to pytest suite

* Add PyTest suite for Siracusa and Siracusa Tiled

* Alpha version of CI suite using PyTest for Siracusa and Siracusa Tiled

* Add L2_DOUBLEBUFFER_MODELS to the pytest suite, add -s to debug slow test, fix generic runner

* Fix typo

* Make test use common build folder among a worker to improve compilation speed

* Cleanup unused runners and increase timeout for L3 models

* format and lint

* Migrate Mempool tests to PyTest

* Migrate Snitch, Chimera, and SoftHier tests to PyTest

* Format and Lint

* Migrate Snitch tiled tests to PyTest

* Use the pytest suite in CI for CortexM platform

* Add Siracusa Tiled with Neureka to the pytest suite

* Format and lint

* Fix SoftHier test runner call

* Format and lint

* Remove useless comments and add README for PyTest suite

* Cleanup GitHub action for siracusa with neureka

* Simplify siracusa tiled platform GitHub CI

* Remove unused testRunner

* Update ccache generation to use PyTest suite

* Cleanup generate_test_param function

* DMA test migration to pytest works but only when running test one by one (state contamination)

* Move DMA test to pytest and update CI

* Format and lint

* Adapt the pytest suite and CI for the new test structure

* Fix test path

* Format and lint

* Move internal deeploy test to pytest

* Improve DeeployTest Readme

* Apply code rabbit suggestions

* Fix typo

* Centralize runner logic for test suite and CLI

* Add deeploy runners for the other platforms

* Update README

* Fix import

* Format and lint

* Format and lint

* Fix import and sanitize build folder path

* Move pytest.ini to project root and restrict tests to live in DeeployTest dir

* Add back --profileUntiled flag

* Linting

* Update changelog and revert docker link (CI will fail)

* Change docker link

Author: Victor-Jung

.github/workflows/_runner-chimera.yml

@@ -14,10 +14,7 @@ name: _runner-chimera
       docker-image:
         required: true
         type: string
-      test-names:
-        required: true
-        type: string
-      simulators:
+      pytest-marker:
         required: true
         type: string
 
@@ -39,22 +36,11 @@ jobs:
         with:
           path: /app/.ccache
           key: ccache-ci
-      - name: Run Test
+      - name: Run Test # VJUNG: Run tests with 4 parallel threads as GitHub action VM has 4 cores.
         run: |
-          testNames="${{ inputs.test-names }}"
-          simulators="${{inputs.simulators}}"
           cd DeeployTest
           mkdir -p /app/.ccache
           export CCACHE_DIR=/app/.ccache
           export CHIMERA_SDK_HOME=/app/install/chimera-sdk
-          echo "$simulators" | while IFS= read -r simulator; do
-            if [[ -n "$simulator" ]]; then
-              echo "$testNames" | while IFS= read -r testName; do
-                if [[ -n "$testName" ]]; then
-                  echo "Running test $testName using $simulator"
-                  python testRunner_chimera.py -t Tests/$testName --simulator=$simulator
-                fi
-              done
-            fi
-          done
+          pytest test_platforms.py -v -n 4 -m "chimera and ${{ inputs.pytest-marker }}"
         shell: bash

.github/workflows/_runner-cortexm.yml

@@ -14,7 +14,7 @@ name: _runner-cortexm
       docker-image:
         required: true
         type: string
-      test-names:
+      pytest-marker:
         required: true
         type: string
 
@@ -36,16 +36,10 @@ jobs:
         with:
           path: /app/.ccache
           key: ccache-ci
-      - name: Run Test
+      - name: Run Test # VJUNG: Run tests with 4 parallel threads as GitHub action VM has 4 cores.
         run: |
-          testNames="${{ inputs.test-names }}"
           cd DeeployTest
           mkdir -p /app/.ccache
           export CCACHE_DIR=/app/.ccache
-          echo "$testNames" | while IFS= read -r testName; do
-            if [[ -n "$testName" ]]; then
-              echo "Running test: $testName"
-              python testRunner_cortexm.py -t Tests/$testName
-            fi
-          done
+          pytest test_platforms.py -v -n 4 -m "cortexm and ${{ inputs.pytest-marker }}"
         shell: bash

.github/workflows/_runner-generic.yml

@@ -14,7 +14,7 @@ name: _runner-generic
       docker-image:
         required: true
         type: string
-      test-names:
+      pytest-marker:
         required: true
         type: string
 
@@ -36,16 +36,10 @@ jobs:
         with:
           path: /app/.ccache
           key: ccache-ci
-      - name: Run Test
+      - name: Run Test # VJUNG: Run tests with 4 parallel threads as GitHub action VM has 4 cores.
         run: |
-          testNames="${{ inputs.test-names }}"
           cd DeeployTest
           mkdir -p /app/.ccache
           export CCACHE_DIR=/app/.ccache
-          echo "$testNames" | while IFS= read -r testName; do
-            if [[ -n "$testName" ]]; then
-              echo "Running test: $testName"
-              python testRunner_generic.py -t Tests/$testName
-            fi
-          done
+          pytest test_platforms.py -v -n 4 -m "generic and ${{ inputs.pytest-marker }}"
         shell: bash

.github/workflows/_runner-mempool.yml

@@ -14,7 +14,7 @@ name: _runner-mempool
       docker-image:
         required: true
         type: string
-      test-names:
+      pytest-marker:
         required: true
         type: string
 
@@ -36,16 +36,10 @@ jobs:
         with:
           path: /app/.ccache
           key: ccache-ci
-      - name: Run Test
+      - name: Run Test # VJUNG: Run tests with 4 parallel threads as GitHub action VM has 4 cores.
         run: |
-          testNames="${{ inputs.test-names }}"
           cd DeeployTest
           mkdir -p /app/.ccache
           export CCACHE_DIR=/app/.ccache
-          echo "$testNames" | while IFS= read -r testName; do
-            if [[ -n "$testName" ]]; then
-              echo "Running test: $testName"
-              python testRunner_mempool.py -t Tests/$testName
-            fi
-          done
+          pytest test_platforms.py -v -n 4 -m "mempool and ${{ inputs.pytest-marker }}"
         shell: bash

.github/workflows/_runner-siracusa-neureka-tiled-sequential.yml

@@ -3,7 +3,7 @@
 # SPDX-License-Identifier: Apache-2.0
 
 ---
-name: _runner-siracusa-neureka-tiled
+name: _runner-siracusa-neureka-tiled-sequential
 
 "on":
   workflow_call:
@@ -14,44 +14,12 @@ name: _runner-siracusa-neureka-tiled
       docker-image:
         required: true
         type: string
-      test-name:
+      pytest-marker:
         required: true
         type: string
-      num-cores:
-        required: false
-        default: 8
-        type: number
-      L1:
-        required: false
-        default: "[64000]"
-        type: string
-      default-memory-level:
-        required: false
-        default: "L2"
-        type: string
-      double-buffer:
-        required: false
-        default: false
-        type: boolean
-      memory-allocation-strategy:
-        required: false
-        default: "MiniMalloc"
-        type: string
-      search-strategy:
-        required: false
-        default: "random-max"
-        type: string
-      neureka-wmem:
-        required: false
-        default: false
-        type: boolean
 
 jobs:
   test-runner-siracusa-neureka-tiled:
-    strategy:
-      fail-fast: false
-      matrix:
-        L1: ${{ fromJSON(inputs.L1) }}
     runs-on: ${{ inputs.runner }}
     container:
       image: ${{ inputs.docker-image }}
@@ -63,20 +31,10 @@ jobs:
       - name: Build Deeploy
         shell: bash
         run: pip install -e .
-      - name: Cache ccache
-        uses: actions/cache/restore@v4
-        with:
-          path: /app/.ccache
-          key: ccache-ci
-      - name: Run Test
-        uses: nick-fields/retry@v3
-        with:
-          timeout_minutes: 5
-          max_attempts: 3
-          retry_on: timeout
-          command: |
-            cd DeeployTest
-            mkdir -p /app/.ccache
-            export CCACHE_DIR=/app/.ccache
-            python testRunner_tiled_siracusa_w_neureka.py -t Tests/${{ inputs.test-name }} --cores=${{ inputs.num-cores }} --l1 ${{ matrix.L1 }} --defaultMemLevel=${{ inputs.default-memory-level }} ${{ inputs.double-buffer && '--doublebuffer' || '' }} ${{ inputs.neureka-wmem && '--neureka-wmem' || '' }} --memAllocStrategy=${{ inputs.memory-allocation-strategy }} --searchStrategy=${{ inputs.search-strategy }}
-          shell: bash
+      - name: Run Test # VJUNG: Run tests with 4 parallel threads as GitHub action VM has 4 cores.
+        run: |
+          cd DeeployTest
+          mkdir -p /app/.ccache
+          export CCACHE_DIR=/app/.ccache
+          pytest test_platforms.py -v -n 4 -m "siracusa_neureka_tiled and ${{ inputs.pytest-marker }}"
+        shell: bash