Skip to content

Commit e4efa8a

Browse files
fpjentzschfpjentzsch
authored
CI for Regression Testing & Benchmarking (#34)
* Add benchmarking scripts that wrap the builder to perform gridsearches defined in a simple YAML format * Add infrastructure for running bitstreams on an FPGA board and logging all results via DVC * Add new custom_step_library to collect all custom steps required for the included end2end models (stored via DVC) * Extend the validate.py script for on-board Top-1 accuracy validation on MNIST, CIFAR-10, CIFAR-100, ImageNet (if synthesized with IODMA shell) * Changes to the logging infrastructure and launch_process_helper
1 parent b226ce1 commit e4efa8a

45 files changed

Lines changed: 4996 additions & 269 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -106,3 +106,4 @@ bench_input
106106
bench_output
107107
bench_save
108108
bench_work
109+
/models

‎.gitlab-ci.yml‎

Lines changed: 82 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
include: ci/.gitlab-setup.yml
2+
13
stages:
24
- sync
35
- build
@@ -17,6 +19,9 @@ variables:
1719
CPU_CORES:
1820
description: "Select number of CPU cores and test workers"
1921
value: "32"
22+
CPU_CORES_BENCH:
23+
description: "Select number of CPU cores for benchmark runs"
24+
value: "8"
2025
PARALLEL_JOBS:
2126
description: "Number of parallel Slurm array jobs per Benchmark job"
2227
value: "1"
@@ -30,15 +35,14 @@ variables:
3035
description: "Optional QoS option (include --qos, e.g., --qos express)"
3136
value: ""
3237
MANUAL_CFG_PATH:
33-
description: "Use this config file instead of configs stored in the repo. Path must be accessible to runner"
38+
description: "Name (in ci/cfg/) or path (relative to LOCAL_CFG_DIR) of benchmarking config to run"
3439
value: ""
3540

3641
workflow:
3742
name: '$PIPELINE_NAME'
3843
rules:
39-
# Run pipeline for GitHub PRs to dev or main (does not support PRs from forks)
44+
# Run pipeline for GitHub PRs to dev (does not support PRs from forks)
4045
- if: $CI_PIPELINE_SOURCE == "external_pull_request_event" && $CI_EXTERNAL_PULL_REQUEST_TARGET_BRANCH_NAME == "dev"
41-
- if: $CI_PIPELINE_SOURCE == "external_pull_request_event" && $CI_EXTERNAL_PULL_REQUEST_TARGET_BRANCH_NAME == "main"
4246
# Run pipeline for pushes to dev or main
4347
- if: $CI_COMMIT_BRANCH == "dev" || $CI_COMMIT_BRANCH == "main"
4448
# Run pipeline if manually triggered via API or web GUI
@@ -67,38 +71,6 @@ Sync finn-dev:
6771
- git pull upstream dev
6872
- git push origin finn-dev
6973

70-
.n2_setup_general: &n2_setup_general
71-
- module load lang/Python/3.10.4-GCCcore-11.3.0
72-
- module load devel/Autoconf/2.71-GCCcore-11.3.0
73-
- module load lang/Bison/3.8.2-GCCcore-11.3.0
74-
- module load lang/flex/2.6.4-GCCcore-11.3.0
75-
- module load compiler/GCC/11.3.0
76-
- module load lib/pybind11/2.9.2-GCCcore-11.3.0
77-
- module load devel/Boost/1.79.0-GCC-11.3.0
78-
- module load lib/fmt/9.1.0-GCCcore-11.3.0
79-
- ulimit -s unlimited # Increase stack size limit
80-
81-
.n2_setup_xilinx_2022_2: &n2_setup_xilinx_2022_2
82-
- module load fpga
83-
- module load xilinx/xrt/2.14 # includes Vitis/Vivado 2022.2
84-
# module load will set PLATFORM_REPO_PATHS to one specific platform, revert to top-level PLATFORM_PATH
85-
- export PLATFORM_REPO_PATHS=$PLATFORM_PATH
86-
87-
.n2_setup_xilinx_2024_2: &n2_setup_xilinx_2024_2
88-
- module load fpga
89-
- module load xilinx/xrt/2.14 # includes Vitis/Vivado 2022.2
90-
- module swap xilinx/vitis/24.2 # switch to Vitis/Vivado 2024.2
91-
# module load will set PLATFORM_REPO_PATHS to one specific platform, revert to top-level PLATFORM_PATH
92-
- export PLATFORM_REPO_PATHS=$PLATFORM_PATH
93-
94-
.setup_venv_from_whl: &setup_venv_from_whl
95-
# Move everything to working directory (e.g., RAMdisk)
96-
- cp -dfR . $PATH_WORKDIR
97-
- cd $PATH_WORKDIR
98-
# Create fresh virtual environment and install finn-plus from .whl (artifact)
99-
- python3 -m venv finn-plus-venv
100-
- finn-plus-venv/bin/pip install dist/*.whl
101-
10274
Build:
10375
id_tokens:
10476
CI_JOB_JWT:
@@ -113,8 +85,8 @@ Build:
11385
# Otherwise run
11486
- when: always
11587
before_script:
116-
- *n2_setup_general
117-
- *n2_setup_xilinx_2022_2
88+
- !reference [.n2_setup_general, before_script]
89+
- !reference [.n2_setup_xilinx_2022_2, before_script]
11890
# Install current version of Poetry
11991
- python3 -m venv poetry-install
12092
- poetry-install/bin/pip install poetry
@@ -151,6 +123,9 @@ FINN Test Suite 2022.2:
151123
# Do not run if test suite has been deselected
152124
- if: $TEST_SUITE == "none"
153125
when: never
126+
# Do not run for PRs to dev (run only for pushes)
127+
- if: $CI_PIPELINE_SOURCE == "external_pull_request_event" && $CI_EXTERNAL_PULL_REQUEST_TARGET_BRANCH_NAME == "dev"
128+
when: never
154129
# Always run, as long as there was no prior failure
155130
- when: on_success
156131
cache:
@@ -159,13 +134,10 @@ FINN Test Suite 2022.2:
159134
paths:
160135
- deps
161136
variables:
162-
GIT_STRATEGY: empty # Do not pull repository, use PyPI installation instead
137+
GIT_STRATEGY: empty # Do not pull repository, install from wheel (artifact) instead
163138
SCHEDULER_PARAMETERS: "-A $PROJECT_ACCOUNT -p $SLURM_PARTITION -t $SLURM_TIMEOUT $SLURM_QOS --nodes 1 --ntasks 1 --cpus-per-task $CPU_CORES --exclusive"
164139
PYTEST_PARALLEL: "$CPU_CORES"
165-
before_script:
166-
- *n2_setup_general
167-
- *n2_setup_xilinx_2022_2
168-
- *setup_venv_from_whl
140+
extends: .setup_full_2022_2
169141
script:
170142
# Launch additional monitoring
171143
- $JOB_MONITORING_DIR/monitor.sh $JOB_MONITORING_DIR/$CI_PIPELINE_ID/$HOSTNAME.log &
@@ -182,8 +154,71 @@ FINN Test Suite 2022.2:
182154
junit: reports/*.xml
183155

184156
FINN Test Suite 2024.2:
185-
extends: FINN Test Suite 2022.2
186-
before_script:
187-
- *n2_setup_general
188-
- *n2_setup_xilinx_2024_2
189-
- *setup_venv_from_whl
157+
extends:
158+
- FINN Test Suite 2022.2
159+
- .setup_full_2024_2
160+
rules:
161+
# Do not run on a schedule
162+
- if: $CI_PIPELINE_SOURCE == "schedule"
163+
when: never
164+
# Do not run if test suite has been deselected
165+
- if: $TEST_SUITE == "none"
166+
when: never
167+
# Always run, as long as there was no prior failure
168+
- when: on_success
169+
170+
Bench (Manual):
171+
stage: test
172+
rules:
173+
# Do not run on a schedule
174+
- if: $CI_PIPELINE_SOURCE == "schedule"
175+
when: never
176+
- if: $MANUAL_CFG_PATH != ""
177+
trigger:
178+
include: ci/.gitlab-bench.yml
179+
strategy: depend
180+
forward:
181+
pipeline_variables: true
182+
variables:
183+
PARENT_PIPELINE_ID: $CI_PIPELINE_ID
184+
BENCH_CFG: "manual"
185+
186+
Bench (Basic):
187+
stage: test
188+
rules:
189+
# Do not run on a schedule
190+
- if: $CI_PIPELINE_SOURCE == "schedule"
191+
when: never
192+
- if: $MANUAL_CFG_PATH == ""
193+
trigger:
194+
include: ci/.gitlab-bench.yml
195+
strategy: depend
196+
forward:
197+
pipeline_variables: true
198+
variables:
199+
PARENT_PIPELINE_ID: $CI_PIPELINE_ID
200+
parallel:
201+
matrix:
202+
- BENCH_CFG: [regression_basic]
203+
204+
Bench (Extended):
205+
stage: test
206+
rules:
207+
# Do not run on a schedule
208+
- if: $CI_PIPELINE_SOURCE == "schedule"
209+
when: never
210+
# Do not run for PRs to dev (run only for pushes)
211+
- if: $CI_PIPELINE_SOURCE == "external_pull_request_event" && $CI_EXTERNAL_PULL_REQUEST_TARGET_BRANCH_NAME == "dev"
212+
when: never
213+
- if: $MANUAL_CFG_PATH == ""
214+
trigger:
215+
include: ci/.gitlab-bench.yml
216+
strategy: depend
217+
forward:
218+
pipeline_variables: true
219+
variables:
220+
PARENT_PIPELINE_ID: $CI_PIPELINE_ID
221+
PARALLEL_JOBS: "4"
222+
parallel:
223+
matrix:
224+
- BENCH_CFG: [regression_extended, microbenchmark_basic]

‎.pre-commit-config.yaml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@ repos:
4343
- id: check-merge-conflict
4444
- id: check-xml
4545
- id: check-yaml
46+
args: ['--unsafe']
4647
- id: debug-statements
4748
exclude: '^src/finn/builder/build_dataflow.py$'
4849
- id: end-of-file-fixer

‎ci/.gitlab-bench.yml‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
include: ci/.gitlab-setup.yml
2+
3+
stages:
4+
- build
5+
- measure
6+
- collect
7+
8+
variables:
9+
BENCH_CFG:
10+
description: "Select config, usually provided by parent pipeline"
11+
value: ""
12+
13+
workflow:
14+
name: "bench_$BENCH_CFG"
15+
16+
FINN Build:
17+
id_tokens:
18+
CI_JOB_JWT:
19+
aud: https://git.uni-paderborn.de
20+
stage: build
21+
needs:
22+
- job: Build
23+
pipeline: $PARENT_PIPELINE_ID
24+
variables:
25+
SCHEDULER_PARAMETERS: "-A $PROJECT_ACCOUNT -p $SLURM_PARTITION -t $SLURM_TIMEOUT $SLURM_QOS --nodes 1 --ntasks 1 --cpus-per-task $CPU_CORES_BENCH --exclusive --array 0-$( expr $PARALLEL_JOBS - 1 )"
26+
NUM_DEFAULT_WORKERS: "$CPU_CORES_BENCH"
27+
extends: .setup_full_2022_2
28+
script:
29+
# Launch additional monitoring
30+
- $JOB_MONITORING_DIR/monitor.sh $JOB_MONITORING_DIR/$CI_PIPELINE_ID/$HOSTNAME.log &
31+
# Launch benchmarking script via FINN CLI, includes deps update and environment preparation
32+
# TODO: cache dvc pull
33+
- |
34+
source finn-plus-venv/bin/activate
35+
dvc pull
36+
finn bench --dependency-path ./deps --build-path $FINN_BUILD_DIR --num-workers $CPU_CORES_BENCH --bench_config $BENCH_CFG
37+
cache:
38+
key: $CI_COMMIT_SHA
39+
policy: pull
40+
paths:
41+
- deps
42+
artifacts:
43+
name: "build_artifacts"
44+
when: always
45+
paths:
46+
- build_artifacts/
47+
48+
Measurement:
49+
id_tokens:
50+
CI_JOB_JWT:
51+
aud: https://git.uni-paderborn.de
52+
stage: measure
53+
tags:
54+
- board
55+
rules:
56+
# Also run on failure of previous tasks to measure partial results
57+
- when: always
58+
script:
59+
# Run as root and activate the PYNQ venv manually to use PYNQ outside of the typical Jupyter environment
60+
- sudo bash -c "source /etc/profile.d/pynq_venv.sh && export XILINX_XRT=/usr && python ci/measure.py"
61+
artifacts:
62+
name: "measurement_artifacts"
63+
when: always
64+
paths:
65+
- measurement_artifacts/
66+
67+
Result Collection:
68+
id_tokens:
69+
CI_JOB_JWT:
70+
aud: https://git.uni-paderborn.de
71+
stage: collect
72+
tags:
73+
- image_build
74+
rules:
75+
# Also run on failure of previous tasks to collect partial results
76+
- when: always
77+
script:
78+
# pulling models seems to be needed for dvclive to save experiments, even though they are not used or modified
79+
- dvc pull
80+
- python3.10 ci/collect.py
81+
- dvc exp push -f -j 4 -r push git@github.com:eki-project/finn-plus.git

‎ci/.gitlab-setup.yml‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
# This file defines some basic scripts used to setup the FINN environment on the runner
2+
3+
.n2_setup_general:
4+
before_script:
5+
- module load lang/Python/3.10.4-GCCcore-11.3.0
6+
- module load devel/Autoconf/2.71-GCCcore-11.3.0
7+
- module load lang/Bison/3.8.2-GCCcore-11.3.0
8+
- module load lang/flex/2.6.4-GCCcore-11.3.0
9+
- module load compiler/GCC/11.3.0
10+
- module load lib/pybind11/2.9.2-GCCcore-11.3.0
11+
- module load devel/Boost/1.79.0-GCC-11.3.0
12+
- module load lib/fmt/9.1.0-GCCcore-11.3.0
13+
- ulimit -s unlimited # Increase stack size limit
14+
15+
.n2_setup_xilinx_2022_2:
16+
before_script:
17+
- module load fpga
18+
- module load xilinx/xrt/2.14 # includes Vitis/Vivado 2022.2
19+
# module load will set PLATFORM_REPO_PATHS to one specific platform, revert to top-level PLATFORM_PATH
20+
- export PLATFORM_REPO_PATHS=$PLATFORM_PATH
21+
22+
.n2_setup_xilinx_2024_2:
23+
before_script:
24+
- module load fpga
25+
- module load xilinx/xrt/2.14 # includes Vitis/Vivado 2022.2
26+
- module swap xilinx/vitis/24.2 # switch to Vitis/Vivado 2024.2
27+
# module load will set PLATFORM_REPO_PATHS to one specific platform, revert to top-level PLATFORM_PATH
28+
- export PLATFORM_REPO_PATHS=$PLATFORM_PATH
29+
30+
.setup_venv_from_whl:
31+
before_script:
32+
# Move everything to working directory (e.g., RAMdisk)
33+
- cp -dfR . $PATH_WORKDIR
34+
- cd $PATH_WORKDIR
35+
# Create fresh virtual environment and install finn-plus from .whl (artifact)
36+
- python3 -m venv finn-plus-venv
37+
- finn-plus-venv/bin/pip install dist/*.whl
38+
39+
.setup_full_2022_2:
40+
before_script:
41+
- !reference [.n2_setup_general, before_script]
42+
- !reference [.n2_setup_xilinx_2022_2, before_script]
43+
- !reference [.setup_venv_from_whl, before_script]
44+
45+
.setup_full_2024_2:
46+
before_script:
47+
- !reference [.n2_setup_general, before_script]
48+
- !reference [.n2_setup_xilinx_2024_2, before_script]
49+
- !reference [.setup_venv_from_whl, before_script]

‎ci/cfg/live_fifosizing.yml‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
[
2+
# Real models
3+
{
4+
"dut": ["vgg10"],
5+
"live_fifo_sizing": [True],
6+
"generate_outputs": [["bitfile", "pynq_driver", "deployment_package"]]
7+
},
8+
{
9+
"dut": ["mobilenetv1"],
10+
"live_fifo_sizing": [True],
11+
"generate_outputs": [["bitfile", "pynq_driver", "deployment_package"]]
12+
},
13+
{
14+
"dut": ["resnet50"],
15+
"live_fifo_sizing": [True],
16+
"generate_outputs": [["bitfile", "pynq_driver", "deployment_package"]]
17+
},
18+
19+
# Synthetic non-linear models
20+
{
21+
"dut": ["synthetic_nonlinear"],
22+
"dim": [64],
23+
"kernel_size": [5],
24+
"ch": [8],
25+
"simd": [8],
26+
"pe": [8],
27+
"parallel_window": [1],
28+
29+
"lb_num_layers": [1],
30+
"rb_num_layers": [4, 8, 16],
31+
32+
"live_fifo_sizing": [True],
33+
"generate_outputs": [["bitfile", "pynq_driver", "deployment_package"]]
34+
},
35+
{
36+
"dut": ["synthetic_nonlinear"],
37+
"dim": [64],
38+
"kernel_size": [5],
39+
"ch": [8],
40+
"simd": [1],
41+
"pe": [1],
42+
"parallel_window": [0],
43+
44+
"lb_num_layers": [1],
45+
"rb_num_layers": [4, 8, 16],
46+
47+
"live_fifo_sizing": [True],
48+
"generate_outputs": [["bitfile", "pynq_driver", "deployment_package"]]
49+
}
50+
]

0 commit comments

Comments
 (0)