feat(test-framework): Add a test-framework to bao

.github/workflows/test-framework.yaml

@@ -0,0 +1,37 @@
+name: Test Franewrork
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+  workflow_dispatch:
+
+
+jobs:
+
+  run-tests:
+    runs-on: ubuntu-latest
+    container:
+      image: diogo21costa/bao-tests:latest
+      options: --user root
+    strategy:
+      matrix:
+        platform: [
+          "qemu-aarch64-virt",
+        ]
+        recipe: [
+          "single-baremetal/default.nix"
+        ]
+        gic: [
+          "GICV2",
+        ]
+        cross_compile: [
+          "CROSS_COMPILE=clang"
+        ]
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: recursive
+
+      - name: Run tests inside nix shell
+        run: nix develop ./tests --command make tests PLATFORM=${{ matrix.platform }} RECIPE=${{ matrix.recipe }} ${{ matrix.cross_compile }} GIC_VERSION=${{ matrix.gic }}

.gitignore

@@ -1,5 +1,6 @@
 bin/*
 build/*
+**/wrkdir/
 *.o
 *.elf
 *.bin

.gitmodules

@@ -1,3 +1,9 @@
 [submodule "ci"]
 	path = ci
 	url = git@github.com:bao-project/bao-ci.git
+[submodule "tests/tf"]
+	path = tests/tf
+	url = https://github.com/bao-project/bao-tf.git
+[submodule "tests/benchs"]
+	path = tests/benchs
+	url = git@github.com:osyxtechnologies/bao-benchmarks.git

Makefile

@@ -76,7 +76,7 @@ targets:=$(MAKECMDGOALS)
 ifeq ($(targets),)
 targets:=all
 endif
-non_build_targets+=ci clean
+non_build_targets+=ci clean tests
 build_targets:=$(strip $(foreach target, $(targets), \
 	$(if $(findstring $(target),$(non_build_targets)),,$(target))))
 
@@ -436,4 +436,10 @@ $(call ci, format, $(all_c_files))
 .PHONY: ci
 ci: license-check format-check
 
-endif
+tests_dir := $(cur_dir)/tests
+-include $(tests_dir)/tests.mk
+
+.PHONY: tests
+tests: framework 
+
+endif

README.md

@@ -144,3 +144,23 @@ In IEEE Access, 2024. https://ieeexplore.ieee.org/document/10781314
 
 10. Hidemasa Kawasaki and Soramichi Akiyama. "**Running Bao Hypervisor on gem5**"
 In gem5 blog, 2024. https://www.gem5.org/2024/11/12/bao-on-gem5.html
+
+
+Testing Framework Usage
+-----------------------
+
+### Prerequisites
+- Ensure [Nix package manager](https://nixos.org/download.html) is installed.
+- Ensure that all dependencies (check [here](./tests/README.md)) are installed.
+
+### Run the test framework locally 
+(Follow [this instructions](./tests/README.md) to use Docker)
+```sh
+make tests PLATFORM=<platform_name> RECIPE=<target_recipe>
+```
+
+Example using `qemu-aarch64-virt` with a single baremetal setup:
+
+```sh
+make tests PLATFORM=qemu-aarch64-virt RECIPE=single-baremetal/default.nix IRQC=gicv3
+```

tests/README.md

@@ -0,0 +1,49 @@
+# Bao Test Framework
+
+### Prerequesites
+If you prefer not to use Docker or Nix, you can install the required dependencies manually:
+```sh
+sudo apt-get update && sudo apt-get install -y \
+  git curl wget tree vim nano sudo make net-tools socat xterm \
+  python3 python3-pip nodejs npm bison flex gawk device-tree-compiler \
+  ninja-build pkg-config enchant
+
+pip install psutil pyserial
+```
+
+
+### Clone docker
+To pull the Docker image, run the following command:
+
+```sh
+docker pull baoproject/bao-tests:latest
+```
+
+### Run the Test framework inside Docker
+
+First, start Docker with the following command in the repo's root directory:
+```sh
+docker run --rm -it \
+  -v "$(pwd):$(pwd)" \
+  -w "$(pwd)" \
+  -u $(id -u):$(id -g) \
+  -e DISPLAY=$DISPLAY \
+  -v /tmp/.X11-unix:/tmp/.X11-unix \
+  baoproject/bao-tests:latest
+```
+
+Then, initialize the Nix environment:
+```sh
+nix develop ./tests/
+```
+
+Finally, run the test framework with:
+```sh
+make tests PLATFORM=<platform_name> RECIPE=<target_recipe>
+```
+
+Currently, the following platforms are available:
+- QEMU RV64 virt (qemu-aarch64-virt)
+- QEMU Aarch64 virt (qemu-aarch64-virt)
+- FVP-A AArch64 (fvp-a)
+- FVP-R AArch64 (fvp-r)

tests/tests.mk

@@ -0,0 +1,46 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright (c) Bao Project and Contributors. All rights reserved
+
+bao-tests_dir = $(tests_dir)/bao-tests
+bao-nix_dir = $(tests_dir)/bao-nix
+recipes_dir = $(tests_dir)/recipes
+
+LOG_LEVEL?=2
+ECHO?=tf
+SETUP?=baremetal
+TEST?=boot
+
+irq_flags:=
+
+ifdef GIC_VERSION
+irq_flags+=-gicv
+irq_flags+=$(GIC_VERSION)
+endif
+
+ifdef IRQC
+irq_flags+=-irqc
+irq_flags+=$(IRQC)
+endif
+
+ifdef IPIC
+irq_flags+=-ipic
+irq_flags+=$(IPIC)
+endif
+
+
+.PHONY: test-framework
+framework:
+
+ifndef RECIPE
+	@echo "Error: RECIPE variable is not defined. Please specify the path to the recipe file."
+	@exit 1
+endif
+
+	@echo "Running Bao Tests Framework..."
+	python3 $(bao-tests_dir)/framework/test_framework.py \
+		-log_level $(LOG_LEVEL) \
+		-echo $(ECHO) \
+		-platform $(PLATFORM) \
+		-test $(TEST) \
+		-setup $(SETUP) \
+		$(irq_flags)

tests/tests/configs/baremetal/fvp-r.c

@@ -0,0 +1,49 @@
+#include <config.h>
+
+
+struct config config = {
+
+    .vmlist_size = 1,
+    .vmlist = (struct vm_config[]) {
+        {
+            .image = VM_IMAGE_LOADED(0x10000000, 0x10000000, 100*1024),
+
+            .entry = 0x10000000,
+
+            .platform = {
+                .cpu_num = 4,
+
+                .region_num = 1,
+                .regions =  (struct vm_mem_region[]) {
+                    {
+                        .base = 0x10000000,
+                        .size = 0x4000000,
+                    },
+                },
+
+                .dev_num = 2,
+                .devs =  (struct vm_dev_region[]) {
+                    {   
+                        /* UART2, PL011 */
+                        .pa = 0x9c0A0000,
+                        .va = 0x9c0A0000,
+                        .size = 0x10000,
+                        .interrupt_num = 1,
+                        .interrupts = (irqid_t[]) {38} 
+                    },
+                    {   
+                        .interrupt_num = 1,
+                        .interrupts = (irqid_t[]) {27} 
+                    }
+                },
+
+                .arch = {
+                    .gic = {
+                        .gicd_addr = 0xAF000000,
+                        .gicr_addr = 0xAF100000,
+                    }
+                }
+            },
+        }
+    }
+};

tests/tests/configs/baremetal/qemu-aarch64-virt.c

@@ -0,0 +1,58 @@
+#include <config.h>
+
+VM_IMAGE(baremetal_image, XSTR(BAO_WRKDIR_IMGS/baremetal.bin))
+
+struct config config = {
+
+    CONFIG_HEADER
+
+    .vmlist_size = 1,
+    .vmlist = (struct vm_config[]) {
+        {
+            .image = {
+                .base_addr = 0x50000000,
+                .load_addr = VM_IMAGE_OFFSET(baremetal_image),
+                .size = VM_IMAGE_SIZE(baremetal_image)
+            },
+
+            .entry = 0x50000000,
+
+            .platform = {
+                .cpu_num = 4,
+
+                .region_num = 1,
+                .regions =  (struct vm_mem_region[]) {
+                    {
+                        .base = 0x50000000,
+                        .size = 0x4000000
+                    }
+                },
+
+                .dev_num = 2,
+                .devs =  (struct vm_dev_region[]) {
+                    {
+                        /* PL011 */
+                        .pa = 0x9000000,
+                        .va = 0x9000000,
+                        .size = 0x10000,
+                        .interrupt_num = 1,
+                        .interrupts = (irqid_t[]) {33}
+                    },
+                    {
+                        /* Arch timer interrupt */
+                        .interrupt_num = 1,
+                        .interrupts =
+                            (irqid_t[]) {27}
+                    }
+                },
+
+                .arch = {
+                    .gic = {
+                        .gicd_addr = (paddr_t) 0x08000000,
+                        .gicr_addr = (paddr_t) 0x080A0000,
+                    }
+                }
+            },
+        }
+    },
+};

tests/tests/configs/baremetal/qemu-riscv64-virt.c

@@ -0,0 +1,48 @@
+#include <config.h>
+
+VM_IMAGE(baremetal_image, XSTR(BAO_WRKDIR_IMGS/baremetal.bin))
+
+struct config config  = {
+
+    CONFIG_HEADER
+
+    .vmlist_size = 1,
+    .vmlist = (struct vm_config[]) {
+        { 
+            .image = {
+                .base_addr = 0x80200000,
+                .load_addr = VM_IMAGE_OFFSET(baremetal_image),
+                .size = VM_IMAGE_SIZE(baremetal_image)
+            },
+
+            .entry = 0x80200000,
+
+            .platform = {
+                .cpu_num = 4,
+
+                .region_num = 1,
+                .regions =  (struct vm_mem_region[]) {
+                    {
+                        .base = 0x80200000,
+                        .size = 0x4000000
+                    }
+                },
+
+                .dev_num = 1,
+                .devs =  (struct vm_dev_region[]) {
+                    {
+                        .pa = 0x10000000,   
+                        .va = 0x10000000,  
+                        .size = 0x1000, 
+                        .interrupt_num = 1,
+                        .interrupts = (irqid_t[]) {10}
+                    },
+                },
+
+                .arch = {
+                   .irqc.plic.base = 0xc000000,
+                }
+            },
+        },
+     }
+};