Arm backend: Split smaller stories tests per backend (pytorch#19972)

### Summary
Splitup smaller stories tests per backend and adds runtime info

### Test plan
This updated GitHub tests and get tested by it.

Signed-off-by: Zingo Andersen <Zingo.Andersen@arm.com>

Author: zingo

.github/workflows/trunk.yml

@@ -257,7 +257,7 @@ jobs:
           - test_arm_backend: test_pytest_ops_ethos_u85
           - test_arm_backend: test_pytest_models_ethos_u85
           - test_arm_backend: test_run_ethos_u85
-          - test_arm_backend: test_smaller_stories_llama
+          - test_arm_backend: test_smaller_stories_llama_tosa
           - test_arm_backend: test_memory_allocation
           - test_arm_backend: test_ootb_tests_ethos_u
           - test_arm_backend: test_ootb_tests_tosa
@@ -305,6 +305,7 @@ jobs:
           - test_arm_backend: test_pytest_ops_vkml
           - test_arm_backend: test_pytest_models_vkml
           - test_arm_backend: test_ootb_tests_vgf
+          - test_arm_backend: test_smaller_stories_llama_vkml
       fail-fast: false
     with:
       runner: linux.2xlarge.memory

backends/arm/README.md

@@ -6,7 +6,7 @@ PyTorch models to a TOSA representation. This representation is used to
 deploy to the following targets:
 
 - **Arm® Ethos™-U55/65/85** - Compiled using the Ethos-U Vela compiler.
-- **VGF Format, for ML extensions for Vulkan®** – a format containing SPIR-V™ ML operators for Vulkan-capable devices.
+- **VKML using VGF Format, for ML extensions for Vulkan®** – a format containing SPIR-V™ ML operators for Vulkan Machine Learning (VKML) devices.
 
 The backend provides an ahead-of-time (AOT) flow, that produces a PTE file for your
 chosen target. The AOT flow supports the following development operating systems:
@@ -248,15 +248,16 @@ Below is an overview of some of the testing options this script provides:
 | `test_arm_backend.sh test_pytest_ops_ethos_u85`    | Runs operator unit tests for Ethos-U85 specific use-cases.   |
 | `test_arm_backend.sh test_pytest_models_ethos_u85` | Runs model unit tests for Ethos-U85 specific use-cases.      |
 | `test_arm_backend.sh test_run_ethos_u85`           | Runs end-to-end unit tests for Ethos-U85 specific use-cases. |
-| `test_arm_backend.sh test_pytest_ops_vkml`         | Runs operator unit tests for VGF specific use-cases.         |
-| `test_arm_backend.sh test_pytest_models_vkml`      | Runs model unit tests for VGF specific use-cases.            |
-| `test_arm_backend.sh test_run_vkml`                | Runs end-to-end unit tests for VGF specific use-cases.       |
+| `test_arm_backend.sh test_pytest_ops_vkml`         | Runs operator unit tests for VKML/VGF specific use-cases.    |
+| `test_arm_backend.sh test_pytest_models_vkml`      | Runs model unit tests for VKML/VGF specific use-cases.       |
+| `test_arm_backend.sh test_run_vkml`                | Runs end-to-end unit tests for VKML/VGF specific use-cases.  |
 | `test_arm_backend.sh test_model_smollm2_135M`      | Runs some models with Corstone FVP.                          |
 | `test_arm_backend.sh test_ootb_tests_ethos_u`      | Runs out-of-the-box tests for Ethos-U.                       |
 | `test_arm_backend.sh test_ootb_tests_tosa`         | Runs out-of-the-box tests for TOSA.                          |
-| `test_arm_backend.sh test_ootb_tests_vgf`          | Runs out-of-the-box tests for VGF.                           |
+| `test_arm_backend.sh test_ootb_tests_vgf`          | Runs out-of-the-box tests for VKML/VGF.                      |
 | `test_arm_backend.sh test_deit_e2e_ethos_u`        | Runs DEiT end-to-end tests on Ethos-U.                       |
-| `test_arm_backend.sh test_smaller_stories_llama`   | Runs E2E model tests on Corstone FVP.                        |
+| `test_arm_backend.sh test_smaller_stories_llama_tosa` | Runs Llama model tests for TOSA.                          |
+| `test_arm_backend.sh test_smaller_stories_llama_vkml` | Runs Llama model tests for VKML/VGF.                      |
 | `test_arm_backend.sh test_memory_allocation`       | Runs memory allocation tests for Ethos-U specific targets    |
 
 For more information, please refer to the `backends/arm/test/test_arm_backend.sh` script.

backends/arm/test/test_arm_backend.sh

@@ -336,14 +336,16 @@ test_model_smollm2_135M() {
         -a "${et_root_dir}"/arm_test/ethos-u85-256_${pte_addr}/cmake-out/arm_executor_runner \
         -C mps4_board.subsystem.ethosu.extra_args="--fast" \
         --data smollm2.pte@"${pte_addr}"
-    
+
     echo "${TEST_SUITE_NAME}: PASS"
 }
 
-test_smaller_stories_llama() {
-    echo "${TEST_SUITE_NAME}: Test smaller_stories_llama"
+_test_smaller_stories_llama() {
+    local backend=$1
 
-    backends/arm/scripts/build_executorch.sh
+    echo "${TEST_SUITE_NAME}: Test smaller_stories_llama for ${backend}"
+
+    # This model might consume a lot of memory so --numprocesses=auto is not used to avoid parallel testing
 
     mkdir -p stories110M
     pushd stories110M
@@ -357,14 +359,29 @@ test_smaller_stories_llama() {
     "${PYTEST_RETRY_ARGS[@]}" \
     --verbose \
     --color=yes \
-    --numprocesses=auto \
-    --junit-xml=stories110M/test-reports/unittest.xml \
+    --durations=0 \
     backends/arm/test/models/test_llama.py \
+    -k "${backend}" \
     --llama_inputs stories110M/stories110M.pt stories110M/params.json stories110m
 
     echo "${TEST_SUITE_NAME}: PASS"
 }
 
+test_smaller_stories_llama_tosa() {
+    _test_smaller_stories_llama tosa
+}
+
+test_smaller_stories_llama_vkml() {
+    source backends/arm/test/setup_testing_vkml.sh
+
+    _test_smaller_stories_llama vgf
+}
+
+test_smaller_stories_llama() {
+    test_smaller_stories_llama_tosa
+    test_smaller_stories_llama_vkml
+}
+
 test_memory_allocation() {
     echo "${TEST_SUITE_NAME}: Test ethos-u memory allocation with run.sh"