From f540684ce51595c803ed500a5c5b897cbbd8025e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ant=C3=B3n=20Casas?= Date: Thu, 25 Sep 2025 11:09:44 +0200 Subject: [PATCH 01/21] Initial approach MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Antón Casas --- .github/workflows/performance.yml | 124 ++++++++++++++++++++++++++++++ 1 file changed, 124 insertions(+) create mode 100644 .github/workflows/performance.yml diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml new file mode 100644 index 000000000..f3cdbb4d1 --- /dev/null +++ b/.github/workflows/performance.yml @@ -0,0 +1,124 @@ +name: Weekly Performance + +on: + workflow_dispatch: + inputs: + name: + description: "Manual trigger" + schedule: + - cron: '12 13 * * 0' + +permissions: + contents: write + +jobs: + performance: + name: Performance + runs-on: ubuntu-24.04 + + steps: + - name: Checkout + uses: actions/checkout@v5 + with: + fetch-depth: 0 + ref: ${{ github.ref }} + + - name: Install CMake and prerequisites + shell: bash + run: | + sudo apt-get update + sudo apt-get install -y build-essential curl python3-pip valgrind + python3 -m pip install --upgrade pip + python3 -m pip install msparser matplotlib + + # CMake 3.31.8 + CMAKE_VER=3.31.8 + curl -L "https://github.com/Kitware/CMake/releases/download/v${CMAKE_VER}/cmake-${CMAKE_VER}-linux-x86_64.tar.gz" -o cmake-${CMAKE_VER}.tgz + sudo tar -C /opt -xzf cmake-${CMAKE_VER}.tgz + echo "/opt/cmake-${CMAKE_VER}-linux-x86_64/bin" >> "$GITHUB_PATH" + + - name: Run tests + shell: bash + run: | + ./test/memory/memory_test.sh + + - name: Save results into memory + shell: bash + run: | + set -euxo pipefail + mkdir -p ci/metrics/complete_profile/runs + + stamp="$(date -u +%Y%m%dT%H%M%SZ)_${GITHUB_RUN_NUMBER}_${GITHUB_SHA::8}" + cp build/complete_profile.csv "ci/metrics/complete_profile/runs/${stamp}.csv" + + git config user.name "github-actions" + git config user.email "github-actions@github.com" + + BR="memory/performance" # Use this branch to store performance results + git fetch origin "${BR}" || true + git checkout -B "${BR}" + + git add ci/metrics/complete_profile/runs/ + git diff --cached --quiet || git commit -m "metrics: complete_profile run ${GITHUB_RUN_NUMBER} (${GITHUB_SHA::8})" + git push -f origin "${BR}" + + - name: Generate plots + shell: bash + run: | + set -euxo pipefail + python3 - <<'PY' + import glob, os, csv, re + from pathlib import Path + import matplotlib.pyplot as plt + + runs_dir = Path("ci/metrics/complete_profile/runs") + out_png = Path("build/complete_profile_last30.png") + out_png.parent.mkdir(parents=True, exist_ok=True) + + files = sorted(runs_dir.glob("*.csv"))[-30:] # last 30 by filename (timestamp prefix) + if not files: + raise SystemExit("No historical CSVs found in ci/metrics/complete_profile/runs") + + y_vals = [] + for f in files: + with f.open(newline="") as fh: + reader = csv.DictReader(fh) + row = next(reader, None) + if row is None: + continue + # CSV values are quoted strings; convert to int safely + val = row.get("text") + if val is None: + continue + y_vals.append(int(val)) + + if not y_vals: + raise SystemExit("No numeric 'text' values found to plot") + + plt.figure() + plt.plot(range(1, len(y_vals)+1), y_vals) # line plot + plt.title("Complete Profile") + plt.xlabel("Run (old → new)") + plt.ylabel("Bytes") + plt.xticks([]) # keep clean + plt.tight_layout() + plt.savefig(out_png) + print(f"Wrote {out_png}") + PY + + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + name: performance-results + path: | + build/complete_profile.csv + build/complete_profile_last30.png + if-no-files-found: warn + + - name: Add summary + if: always() + shell: bash + run: | + echo "## Complete Profile (last 30 runs)" >> "$GITHUB_STEP_SUMMARY" + echo "" >> "$GITHUB_STEP_SUMMARY" + echo "![Complete Profile](./build/complete_profile_last30.png)" >> "$GITHUB_STEP_SUMMARY" From a951a8b091f50fa560418aec1999ed3a20ab227d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ant=C3=B3n=20Casas?= Date: Thu, 25 Sep 2025 11:24:55 +0200 Subject: [PATCH 02/21] Plot all columns MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Antón Casas --- .github/workflows/performance.yml | 49 +++++++++++++++++++------------ 1 file changed, 30 insertions(+), 19 deletions(-) diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml index f3cdbb4d1..00c3bf49c 100644 --- a/.github/workflows/performance.yml +++ b/.github/workflows/performance.yml @@ -67,7 +67,7 @@ jobs: run: | set -euxo pipefail python3 - <<'PY' - import glob, os, csv, re + import csv from pathlib import Path import matplotlib.pyplot as plt @@ -77,30 +77,41 @@ jobs: files = sorted(runs_dir.glob("*.csv"))[-30:] # last 30 by filename (timestamp prefix) if not files: - raise SystemExit("No historical CSVs found in ci/metrics/complete_profile/runs") + raise SystemExit("No historical CSVs found") - y_vals = [] - for f in files: - with f.open(newline="") as fh: - reader = csv.DictReader(fh) - row = next(reader, None) - if row is None: - continue - # CSV values are quoted strings; convert to int safely - val = row.get("text") - if val is None: - continue - y_vals.append(int(val)) - - if not y_vals: - raise SystemExit("No numeric 'text' values found to plot") + # Dict of series per column + series = {} + for f in files: + with f.open(newline="") as fh: + reader = csv.DictReader(fh) + row = next(reader, None) + if row is None: + continue + # Initialize keys on first file + if not series: + for key in row.keys(): + series[key] = [] + # Append values + for key in series: + val = row.get(key) + try: + series[key].append(int(val)) + except Exception: + series[key].append(0) + + if not any(series.values()): + raise SystemExit("No numeric values found") + + x = range(1, len(files)+1) plt.figure() - plt.plot(range(1, len(y_vals)+1), y_vals) # line plot + for key, values in series.items(): + plt.plot(x, values, label=key) + plt.title("Complete Profile") plt.xlabel("Run (old → new)") plt.ylabel("Bytes") - plt.xticks([]) # keep clean + plt.legend() plt.tight_layout() plt.savefig(out_png) print(f"Wrote {out_png}") From 38d0abbd72742f380490dd733f6cd27ac0aaf143 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ant=C3=B3n=20Casas?= Date: Thu, 25 Sep 2025 12:27:29 +0200 Subject: [PATCH 03/21] Add PR trigger for testing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Antón Casas --- .github/workflows/performance.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml index 00c3bf49c..ce57ea505 100644 --- a/.github/workflows/performance.yml +++ b/.github/workflows/performance.yml @@ -1,6 +1,9 @@ name: Weekly Performance on: + pull_request: + branches: + - develop workflow_dispatch: inputs: name: From 25d67a3ceb5deac23bd658b69554d0abfae271bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ant=C3=B3n=20Casas?= Date: Thu, 25 Sep 2025 14:12:15 +0200 Subject: [PATCH 04/21] Single file approach MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Antón Casas --- .github/workflows/performance.yml | 123 ++++++++++++++++++++---------- 1 file changed, 84 insertions(+), 39 deletions(-) diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml index ce57ea505..4b1743f10 100644 --- a/.github/workflows/performance.yml +++ b/.github/workflows/performance.yml @@ -49,10 +49,54 @@ jobs: shell: bash run: | set -euxo pipefail - mkdir -p ci/metrics/complete_profile/runs + ts_dir="ci/metrics/complete_profile" + ts_csv="${ts_dir}/complete_profile_timeseries.csv" + mkdir -p "${ts_dir}" - stamp="$(date -u +%Y%m%dT%H%M%SZ)_${GITHUB_RUN_NUMBER}_${GITHUB_SHA::8}" - cp build/complete_profile.csv "ci/metrics/complete_profile/runs/${stamp}.csv" + python3 - <<'PY' + import csv, os + from pathlib import Path + from datetime import datetime + + run = os.environ.get("GITHUB_RUN_NUMBER","") + sha = os.environ.get("GITHUB_SHA","")[:8] + ts_path = Path("ci/metrics/complete_profile/complete_profile_timeseries.csv") + src_path = Path("build/complete_profile.csv") + + if not src_path.exists(): + raise SystemExit("missing build/complete_profile.csv") + + # Read first row and header from source csv + with src_path.open(newline="") as fh: + reader = csv.DictReader(fh) + rows = list(reader) + if not rows: + raise SystemExit("empty complete_profile.csv") + row = rows[0] + + # Build a new row with date, run, sha, and all original cols + new_line = {"date": datetime.utcnow().isoformat(timespec="seconds"), + "run": run, "sha": sha} + for key, val in row.items(): + try: + new_line[key] = int(val) + except Exception: + new_line[key] = val + + # Ensure header is consistent + write_header = not ts_path.exists() + if write_header: + header = ["date","run","sha"] + list(row.keys()) + else: + with ts_path.open(newline="") as fh: + header = next(csv.reader(fh)) + with ts_path.open("a", newline="") as out: + w = csv.DictWriter(out, fieldnames=header) + if write_header: + w.writeheader() + w.writerow(new_line) + print(f"appended to {ts_path}") + PY git config user.name "github-actions" git config user.email "github-actions@github.com" @@ -61,8 +105,8 @@ jobs: git fetch origin "${BR}" || true git checkout -B "${BR}" - git add ci/metrics/complete_profile/runs/ - git diff --cached --quiet || git commit -m "metrics: complete_profile run ${GITHUB_RUN_NUMBER} (${GITHUB_SHA::8})" + git add ci/metrics/complete_profile/complete_profile_timeseries.csv + git diff --cached --quiet || git commit -m "metrics: append run ${GITHUB_RUN_NUMBER} (${GITHUB_SHA::8})" git push -f origin "${BR}" - name: Generate plots @@ -74,46 +118,47 @@ jobs: from pathlib import Path import matplotlib.pyplot as plt - runs_dir = Path("ci/metrics/complete_profile/runs") - out_png = Path("build/complete_profile_last30.png") + ts_csv = Path("ci/metrics/complete_profile/complete_profile_timeseries.csv") + out_png = Path("build/complete_profile_last30.png") out_png.parent.mkdir(parents=True, exist_ok=True) - files = sorted(runs_dir.glob("*.csv"))[-30:] # last 30 by filename (timestamp prefix) - if not files: - raise SystemExit("No historical CSVs found") - - # Dict of series per column - series = {} - - for f in files: - with f.open(newline="") as fh: - reader = csv.DictReader(fh) - row = next(reader, None) - if row is None: - continue - # Initialize keys on first file - if not series: - for key in row.keys(): - series[key] = [] - # Append values - for key in series: - val = row.get(key) - try: - series[key].append(int(val)) - except Exception: - series[key].append(0) - - if not any(series.values()): - raise SystemExit("No numeric values found") - - x = range(1, len(files)+1) + if not ts_csv.exists(): + raise SystemExit("time-series CSV not found") + + rows = [] + with ts_csv.open(newline="") as fh: + reader = csv.DictReader(fh) + header = reader.fieldnames or [] + for r in reader: + rows.append(r) + + if not rows: + raise SystemExit("No rows in time-series CSV") + + rows = rows[-30:] # last 30 runs + + # Choose numeric columns (skip date, run, sha) + skip = {"date","run","sha"} + cols = [h for h in header if h not in skip] + + x = range(1, len(rows)+1) + dates = [r["date"] for r in rows] + plt.figure() - for key, values in series.items(): - plt.plot(x, values, label=key) + for key in cols: + try: + ys = [int(r.get(key,0)) for r in rows] + except Exception: + continue + plt.plot(x, ys, label=key) plt.title("Complete Profile") - plt.xlabel("Run (old → new)") + plt.xlabel("Date") plt.ylabel("Bytes") + step = max(1, len(dates)//8) + shown = list(range(0, len(dates), step)) + plt.xticks([i+1 for i in shown], [dates[i] for i in shown], rotation=45, ha="right") + plt.legend() plt.tight_layout() plt.savefig(out_png) From 7d0f7040f587e09ca3c19028f9df163f1af27c4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ant=C3=B3n=20Casas?= Date: Thu, 25 Sep 2025 14:42:18 +0200 Subject: [PATCH 05/21] Move python code to file MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Antón Casas --- .github/workflows/performance.yml | 52 +++-------------------------- ci/metrics/save_results.py | 54 +++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+), 48 deletions(-) create mode 100644 ci/metrics/save_results.py diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml index 4b1743f10..be27e54cd 100644 --- a/.github/workflows/performance.yml +++ b/.github/workflows/performance.yml @@ -49,54 +49,6 @@ jobs: shell: bash run: | set -euxo pipefail - ts_dir="ci/metrics/complete_profile" - ts_csv="${ts_dir}/complete_profile_timeseries.csv" - mkdir -p "${ts_dir}" - - python3 - <<'PY' - import csv, os - from pathlib import Path - from datetime import datetime - - run = os.environ.get("GITHUB_RUN_NUMBER","") - sha = os.environ.get("GITHUB_SHA","")[:8] - ts_path = Path("ci/metrics/complete_profile/complete_profile_timeseries.csv") - src_path = Path("build/complete_profile.csv") - - if not src_path.exists(): - raise SystemExit("missing build/complete_profile.csv") - - # Read first row and header from source csv - with src_path.open(newline="") as fh: - reader = csv.DictReader(fh) - rows = list(reader) - if not rows: - raise SystemExit("empty complete_profile.csv") - row = rows[0] - - # Build a new row with date, run, sha, and all original cols - new_line = {"date": datetime.utcnow().isoformat(timespec="seconds"), - "run": run, "sha": sha} - for key, val in row.items(): - try: - new_line[key] = int(val) - except Exception: - new_line[key] = val - - # Ensure header is consistent - write_header = not ts_path.exists() - if write_header: - header = ["date","run","sha"] + list(row.keys()) - else: - with ts_path.open(newline="") as fh: - header = next(csv.reader(fh)) - with ts_path.open("a", newline="") as out: - w = csv.DictWriter(out, fieldnames=header) - if write_header: - w.writeheader() - w.writerow(new_line) - print(f"appended to {ts_path}") - PY git config user.name "github-actions" git config user.email "github-actions@github.com" @@ -105,6 +57,10 @@ jobs: git fetch origin "${BR}" || true git checkout -B "${BR}" + python3 ci/metrics/save_results.py \ + --input build/complete_profile.csv \ + --output ci/metrics/complete_profile/complete_profile_timeseries.csv + git add ci/metrics/complete_profile/complete_profile_timeseries.csv git diff --cached --quiet || git commit -m "metrics: append run ${GITHUB_RUN_NUMBER} (${GITHUB_SHA::8})" git push -f origin "${BR}" diff --git a/ci/metrics/save_results.py b/ci/metrics/save_results.py new file mode 100644 index 000000000..a52586145 --- /dev/null +++ b/ci/metrics/save_results.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 +import csv, os, sys, argparse +from pathlib import Path +from datetime import datetime + +parser = argparse.ArgumentParser(description="Append a run's CSV to a time-series file") +parser.add_argument("--input", required=True, help="Path to the source CSV (from memory_test.sh)") +parser.add_argument("--output", required=True, help="Path to the cumulative time-series CSV") +args = parser.parse_args() + +src_path = Path(args.input) +ts_path = Path(args.output) + +run = os.environ.get("GITHUB_RUN_NUMBER", "") +sha = os.environ.get("GITHUB_SHA", "")[:8] + +if not src_path.exists(): + sys.exit(f"missing {src_path}") + +with src_path.open(newline="") as fh: + reader = csv.DictReader(fh) + rows = list(reader) +if not rows: + sys.exit("empty source csv") +row = rows[0] + +# Build a new row with metadata and all original columns +new_line = { + "date": datetime.utcnow().isoformat(timespec="seconds"), + "run": run, + "sha": sha, +} +for key, val in row.items(): + try: + new_line[key] = int(val) + except Exception: + new_line[key] = val + +# Ensure header is consistent +write_header = not ts_path.exists() +if write_header: + header = ["date", "run", "sha"] + list(row.keys()) +else: + with ts_path.open(newline="") as fh: + header = next(csv.reader(fh)) + +ts_path.parent.mkdir(parents=True, exist_ok=True) +with ts_path.open("a", newline="") as out: + w = csv.DictWriter(out, fieldnames=header) + if write_header: + w.writeheader() + w.writerow(new_line) + +print(f"Appended row to {ts_path}") From 2db6fb46643cc10e81675fd7056047d240a43c2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ant=C3=B3n=20Casas?= Date: Thu, 25 Sep 2025 15:27:33 +0200 Subject: [PATCH 06/21] Move into a two jobs paradigm MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Antón Casas --- .github/workflows/performance.yml | 53 +++++++++++++++++++++++-------- 1 file changed, 40 insertions(+), 13 deletions(-) diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml index be27e54cd..981253f04 100644 --- a/.github/workflows/performance.yml +++ b/.github/workflows/performance.yml @@ -45,17 +45,53 @@ jobs: run: | ./test/memory/memory_test.sh + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + name: performance-results + path: | + build/complete_profile.csv + build/core_profile.csv + build/stack.csv + build/profiles_bss.csv + build/profiles_data.csv + build/profiles_text.csv + if-no-files-found: error + + analyze-metrics: + name: Analyze Metrics + runs-on: ubuntu-24.04 + needs: performance + + steps: + - name: Download artifacts + uses: actions/download-artifact@v4 + with: + name: performance-results + path: results + + - name: List files + shell: bash + run: | + ls -l results + + - name: Checkout + uses: actions/checkout@v5 + with: + fetch-depth: 0 + ref: memory/performance + - name: Save results into memory shell: bash run: | set -euxo pipefail - git config user.name "github-actions" - git config user.email "github-actions@github.com" + # git config user.name "github-actions" + # git config user.email "github-actions@github.com" BR="memory/performance" # Use this branch to store performance results - git fetch origin "${BR}" || true - git checkout -B "${BR}" + # git fetch origin "${BR}" || true + # git checkout -B "${BR}" python3 ci/metrics/save_results.py \ --input build/complete_profile.csv \ @@ -121,15 +157,6 @@ jobs: print(f"Wrote {out_png}") PY - - name: Upload artifacts - uses: actions/upload-artifact@v4 - with: - name: performance-results - path: | - build/complete_profile.csv - build/complete_profile_last30.png - if-no-files-found: warn - - name: Add summary if: always() shell: bash From 75e9b682e2a3c440156a43605ffe5a548a0633e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ant=C3=B3n=20Casas?= Date: Thu, 25 Sep 2025 15:43:10 +0200 Subject: [PATCH 07/21] Fix results path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Antón Casas --- .github/workflows/performance.yml | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml index 981253f04..bcd5cfd94 100644 --- a/.github/workflows/performance.yml +++ b/.github/workflows/performance.yml @@ -64,6 +64,12 @@ jobs: needs: performance steps: + - name: Checkout + uses: actions/checkout@v5 + with: + fetch-depth: 0 + ref: memory/performance + - name: Download artifacts uses: actions/download-artifact@v4 with: @@ -75,31 +81,26 @@ jobs: run: | ls -l results - - name: Checkout - uses: actions/checkout@v5 - with: - fetch-depth: 0 - ref: memory/performance - - name: Save results into memory shell: bash run: | set -euxo pipefail - # git config user.name "github-actions" - # git config user.email "github-actions@github.com" + git config user.name "github-actions" + git config user.email "github-actions@github.com" BR="memory/performance" # Use this branch to store performance results # git fetch origin "${BR}" || true # git checkout -B "${BR}" python3 ci/metrics/save_results.py \ - --input build/complete_profile.csv \ + --input results/complete_profile.csv \ --output ci/metrics/complete_profile/complete_profile_timeseries.csv + git status git add ci/metrics/complete_profile/complete_profile_timeseries.csv git diff --cached --quiet || git commit -m "metrics: append run ${GITHUB_RUN_NUMBER} (${GITHUB_SHA::8})" - git push -f origin "${BR}" + git push origin "${BR}" - name: Generate plots shell: bash From 94dba70bc191074a97566845fdf2e9c96a56a878 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ant=C3=B3n=20Casas?= Date: Thu, 25 Sep 2025 15:54:24 +0200 Subject: [PATCH 08/21] Fix dependencies MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Antón Casas --- .github/workflows/performance.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml index bcd5cfd94..766cc16b8 100644 --- a/.github/workflows/performance.yml +++ b/.github/workflows/performance.yml @@ -32,7 +32,7 @@ jobs: sudo apt-get update sudo apt-get install -y build-essential curl python3-pip valgrind python3 -m pip install --upgrade pip - python3 -m pip install msparser matplotlib + python3 -m pip install msparser # CMake 3.31.8 CMAKE_VER=3.31.8 @@ -70,6 +70,14 @@ jobs: fetch-depth: 0 ref: memory/performance + - name: Install Python and prerequisites + shell: bash + run: | + sudo apt-get update + sudo apt-get install -y python3-pip + python3 -m pip install --upgrade pip + python3 -m pip install matplotlib + - name: Download artifacts uses: actions/download-artifact@v4 with: From e42df61e28864ba0e33be24174d8e68d640d4a28 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ant=C3=B3n=20Casas?= Date: Thu, 25 Sep 2025 16:13:42 +0200 Subject: [PATCH 09/21] Add all csv MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Antón Casas --- .github/workflows/performance.yml | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml index 766cc16b8..01fa4a9e4 100644 --- a/.github/workflows/performance.yml +++ b/.github/workflows/performance.yml @@ -104,9 +104,24 @@ jobs: python3 ci/metrics/save_results.py \ --input results/complete_profile.csv \ --output ci/metrics/complete_profile/complete_profile_timeseries.csv + python3 ci/metrics/save_results.py \ + --input results/core_profile.csv \ + --output ci/metrics/memory/core_profile_timeseries.csv + python3 ci/metrics/save_results.py \ + --input results/stack.csv \ + --output ci/metrics/memory/stack_timeseries.csv + python3 ci/metrics/save_results.py \ + --input results/profiles_bss.csv \ + --output ci/metrics/memory/profiles_bss_timeseries.csv + python3 ci/metrics/save_results.py \ + --input results/profiles_data.csv \ + --output ci/metrics/memory/profiles_data_timeseries.csv + python3 ci/metrics/save_results.py \ + --input results/profiles_text.csv \ + --output ci/metrics/memory/profiles_text_timeseries.csv git status - git add ci/metrics/complete_profile/complete_profile_timeseries.csv + git add ci/metrics/*_timeseries.csv git diff --cached --quiet || git commit -m "metrics: append run ${GITHUB_RUN_NUMBER} (${GITHUB_SHA::8})" git push origin "${BR}" @@ -166,6 +181,13 @@ jobs: print(f"Wrote {out_png}") PY + - name: Upload plots + uses: actions/upload-artifact@v4 + with: + name: performance-plots + path: build/*.png + if-no-files-found: warning + - name: Add summary if: always() shell: bash From 6fcaf376a295ec3f716c9552eed49e2b9d98ab9d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ant=C3=B3n=20Casas?= Date: Thu, 25 Sep 2025 16:25:08 +0200 Subject: [PATCH 10/21] Fix details up to save results into memory MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Antón Casas --- .github/workflows/performance.yml | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml index 01fa4a9e4..63bd9d9f5 100644 --- a/.github/workflows/performance.yml +++ b/.github/workflows/performance.yml @@ -98,12 +98,10 @@ jobs: git config user.email "github-actions@github.com" BR="memory/performance" # Use this branch to store performance results - # git fetch origin "${BR}" || true - # git checkout -B "${BR}" python3 ci/metrics/save_results.py \ --input results/complete_profile.csv \ - --output ci/metrics/complete_profile/complete_profile_timeseries.csv + --output ci/metrics/memory/complete_profile_timeseries.csv python3 ci/metrics/save_results.py \ --input results/core_profile.csv \ --output ci/metrics/memory/core_profile_timeseries.csv @@ -120,8 +118,7 @@ jobs: --input results/profiles_text.csv \ --output ci/metrics/memory/profiles_text_timeseries.csv - git status - git add ci/metrics/*_timeseries.csv + git add ci/metrics/memory/*_timeseries.csv git diff --cached --quiet || git commit -m "metrics: append run ${GITHUB_RUN_NUMBER} (${GITHUB_SHA::8})" git push origin "${BR}" @@ -186,7 +183,7 @@ jobs: with: name: performance-plots path: build/*.png - if-no-files-found: warning + if-no-files-found: warn - name: Add summary if: always() From 56abadd3b1b117436bcf0799deda1ebd46eb27e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ant=C3=B3n=20Casas?= Date: Fri, 26 Sep 2025 08:27:52 +0200 Subject: [PATCH 11/21] Move plot python code to file MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Antón Casas --- .github/workflows/performance.yml | 105 ++++++++++++------------------ ci/metrics/generate_plot.py | 60 +++++++++++++++++ 2 files changed, 100 insertions(+), 65 deletions(-) create mode 100644 ci/metrics/generate_plot.py diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml index 63bd9d9f5..b6b5dba10 100644 --- a/.github/workflows/performance.yml +++ b/.github/workflows/performance.yml @@ -84,11 +84,6 @@ jobs: name: performance-results path: results - - name: List files - shell: bash - run: | - ls -l results - - name: Save results into memory shell: bash run: | @@ -126,69 +121,49 @@ jobs: shell: bash run: | set -euxo pipefail - python3 - <<'PY' - import csv - from pathlib import Path - import matplotlib.pyplot as plt - - ts_csv = Path("ci/metrics/complete_profile/complete_profile_timeseries.csv") - out_png = Path("build/complete_profile_last30.png") - out_png.parent.mkdir(parents=True, exist_ok=True) - - if not ts_csv.exists(): - raise SystemExit("time-series CSV not found") - - rows = [] - with ts_csv.open(newline="") as fh: - reader = csv.DictReader(fh) - header = reader.fieldnames or [] - for r in reader: - rows.append(r) - - if not rows: - raise SystemExit("No rows in time-series CSV") - - rows = rows[-30:] # last 30 runs - - # Choose numeric columns (skip date, run, sha) - skip = {"date","run","sha"} - cols = [h for h in header if h not in skip] - - x = range(1, len(rows)+1) - dates = [r["date"] for r in rows] - - plt.figure() - for key in cols: - try: - ys = [int(r.get(key,0)) for r in rows] - except Exception: - continue - plt.plot(x, ys, label=key) - - plt.title("Complete Profile") - plt.xlabel("Date") - plt.ylabel("Bytes") - step = max(1, len(dates)//8) - shown = list(range(0, len(dates), step)) - plt.xticks([i+1 for i in shown], [dates[i] for i in shown], rotation=45, ha="right") - - plt.legend() - plt.tight_layout() - plt.savefig(out_png) - print(f"Wrote {out_png}") - PY + + LAST_N=52 + + python3 ci/metrics/generate_plot.py \ + --input ci/metrics/memory/complete_profile_timeseries.csv \ + --output plots/complete_profile_plot.png \ + --title "Complete Profile" \ + --ylabel "Bytes" \ + --last LAST_N + python3 ci/metrics/generate_plot.py \ + --input ci/metrics/memory/core_profile_timeseries.csv \ + --output plots/core_profile_plot.png \ + --title "Core Profile" \ + --ylabel "Bytes" \ + --last LAST_N + python3 ci/metrics/generate_plot.py \ + --input ci/metrics/memory/stack_timeseries.csv \ + --output plots/stack_plot.png \ + --title "Simple App Stack Usage" \ + --ylabel "Bytes" \ + --last LAST_N + python3 ci/metrics/generate_plot.py \ + --input ci/metrics/memory/profiles_bss_timeseries.csv \ + --output plots/profiles_bss_plot.png \ + --title "Increase of .bss memory by enabling profiles" \ + --ylabel "Bytes" \ + --last LAST_N + python3 ci/metrics/generate_plot.py \ + --input ci/metrics/memory/profiles_data_timeseries.csv \ + --output plots/profiles_data_plot.png \ + --title "Increase of .data memory by enabling profiles" \ + --ylabel "Bytes" \ + --last LAST_N + python3 ci/metrics/generate_plot.py \ + --input ci/metrics/memory/profiles_text_timeseries.csv \ + --output plots/profiles_text_plot.png \ + --title "Increase of .text memory by enabling profiles" \ + --ylabel "Bytes" \ + --last LAST_N - name: Upload plots uses: actions/upload-artifact@v4 with: name: performance-plots - path: build/*.png + path: plots/*.png if-no-files-found: warn - - - name: Add summary - if: always() - shell: bash - run: | - echo "## Complete Profile (last 30 runs)" >> "$GITHUB_STEP_SUMMARY" - echo "" >> "$GITHUB_STEP_SUMMARY" - echo "![Complete Profile](./build/complete_profile_last30.png)" >> "$GITHUB_STEP_SUMMARY" diff --git a/ci/metrics/generate_plot.py b/ci/metrics/generate_plot.py new file mode 100644 index 000000000..e8ebbd501 --- /dev/null +++ b/ci/metrics/generate_plot.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 +import csv, sys, argparse +from pathlib import Path +import matplotlib.pyplot as plt + +parser = argparse.ArgumentParser(description="Generate a plot from a time-series CSV") +parser.add_argument("--input", required=True, help="Path to the cumulative time-series CSV") +parser.add_argument("--output", required=True, help="Path to save the plot PNG") +parser.add_argument("--title", default="", help="Title for the plot") +parser.add_argument("--ylabel", default="", help="Y-axis label") +parser.add_argument("--last", type=int, default=30, help="Number of last runs to plot") +args = parser.parse_args() + +ts_csv = Path(args.input) +out_png = Path(args.output) + +out_png.parent.mkdir(parents=True, exist_ok=True) + +if not ts_csv.exists(): + sys.exit(f"{ts_csv} not found") + +rows = [] +with ts_csv.open(newline="") as fh: + reader = csv.DictReader(fh) + header = reader.fieldnames or [] + for r in reader: + rows.append(r) + +if not rows: + sys.exit("No rows in time-series CSV") + +rows = rows[-(args.last):] + +skip = {"date", "run", "sha"} +cols = [h for h in header if h not in skip] + +x = range(1, len(rows)+1) +dates = [r["date"] for r in rows] + +plt.figure() +for key in cols: + try: + ys = [int(r.get(key, 0)) for r in rows] + except Exception: + continue + plt.plot(x, ys, label=key) + +plt.title(args.title) +plt.xlabel("Date") +plt.ylabel(args.ylabel) + +# Thin x labels for readability +step = max(1, len(dates)//8) +shown = list(range(0, len(dates), step)) +plt.xticks([i+1 for i in shown], [dates[i] for i in shown], rotation=45, ha="right") + +plt.legend() +plt.tight_layout() +plt.savefig(out_png) +print(f"Wrote {out_png}") From 9f51c43122350393ebc6b36d284ba1f3b60d32e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ant=C3=B3n=20Casas?= Date: Fri, 26 Sep 2025 08:39:21 +0200 Subject: [PATCH 12/21] Remove python code MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Antón Casas --- .github/workflows/performance.yml | 12 +++---- ci/metrics/generate_plot.py | 60 ------------------------------- ci/metrics/save_results.py | 54 ---------------------------- 3 files changed, 6 insertions(+), 120 deletions(-) delete mode 100644 ci/metrics/generate_plot.py delete mode 100644 ci/metrics/save_results.py diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml index b6b5dba10..25e589698 100644 --- a/.github/workflows/performance.yml +++ b/.github/workflows/performance.yml @@ -129,37 +129,37 @@ jobs: --output plots/complete_profile_plot.png \ --title "Complete Profile" \ --ylabel "Bytes" \ - --last LAST_N + --last "$LAST_N" python3 ci/metrics/generate_plot.py \ --input ci/metrics/memory/core_profile_timeseries.csv \ --output plots/core_profile_plot.png \ --title "Core Profile" \ --ylabel "Bytes" \ - --last LAST_N + --last "$LAST_N" python3 ci/metrics/generate_plot.py \ --input ci/metrics/memory/stack_timeseries.csv \ --output plots/stack_plot.png \ --title "Simple App Stack Usage" \ --ylabel "Bytes" \ - --last LAST_N + --last "$LAST_N" python3 ci/metrics/generate_plot.py \ --input ci/metrics/memory/profiles_bss_timeseries.csv \ --output plots/profiles_bss_plot.png \ --title "Increase of .bss memory by enabling profiles" \ --ylabel "Bytes" \ - --last LAST_N + --last "$LAST_N" python3 ci/metrics/generate_plot.py \ --input ci/metrics/memory/profiles_data_timeseries.csv \ --output plots/profiles_data_plot.png \ --title "Increase of .data memory by enabling profiles" \ --ylabel "Bytes" \ - --last LAST_N + --last "$LAST_N" python3 ci/metrics/generate_plot.py \ --input ci/metrics/memory/profiles_text_timeseries.csv \ --output plots/profiles_text_plot.png \ --title "Increase of .text memory by enabling profiles" \ --ylabel "Bytes" \ - --last LAST_N + --last "$LAST_N" - name: Upload plots uses: actions/upload-artifact@v4 diff --git a/ci/metrics/generate_plot.py b/ci/metrics/generate_plot.py deleted file mode 100644 index e8ebbd501..000000000 --- a/ci/metrics/generate_plot.py +++ /dev/null @@ -1,60 +0,0 @@ -#!/usr/bin/env python3 -import csv, sys, argparse -from pathlib import Path -import matplotlib.pyplot as plt - -parser = argparse.ArgumentParser(description="Generate a plot from a time-series CSV") -parser.add_argument("--input", required=True, help="Path to the cumulative time-series CSV") -parser.add_argument("--output", required=True, help="Path to save the plot PNG") -parser.add_argument("--title", default="", help="Title for the plot") -parser.add_argument("--ylabel", default="", help="Y-axis label") -parser.add_argument("--last", type=int, default=30, help="Number of last runs to plot") -args = parser.parse_args() - -ts_csv = Path(args.input) -out_png = Path(args.output) - -out_png.parent.mkdir(parents=True, exist_ok=True) - -if not ts_csv.exists(): - sys.exit(f"{ts_csv} not found") - -rows = [] -with ts_csv.open(newline="") as fh: - reader = csv.DictReader(fh) - header = reader.fieldnames or [] - for r in reader: - rows.append(r) - -if not rows: - sys.exit("No rows in time-series CSV") - -rows = rows[-(args.last):] - -skip = {"date", "run", "sha"} -cols = [h for h in header if h not in skip] - -x = range(1, len(rows)+1) -dates = [r["date"] for r in rows] - -plt.figure() -for key in cols: - try: - ys = [int(r.get(key, 0)) for r in rows] - except Exception: - continue - plt.plot(x, ys, label=key) - -plt.title(args.title) -plt.xlabel("Date") -plt.ylabel(args.ylabel) - -# Thin x labels for readability -step = max(1, len(dates)//8) -shown = list(range(0, len(dates), step)) -plt.xticks([i+1 for i in shown], [dates[i] for i in shown], rotation=45, ha="right") - -plt.legend() -plt.tight_layout() -plt.savefig(out_png) -print(f"Wrote {out_png}") diff --git a/ci/metrics/save_results.py b/ci/metrics/save_results.py deleted file mode 100644 index a52586145..000000000 --- a/ci/metrics/save_results.py +++ /dev/null @@ -1,54 +0,0 @@ -#!/usr/bin/env python3 -import csv, os, sys, argparse -from pathlib import Path -from datetime import datetime - -parser = argparse.ArgumentParser(description="Append a run's CSV to a time-series file") -parser.add_argument("--input", required=True, help="Path to the source CSV (from memory_test.sh)") -parser.add_argument("--output", required=True, help="Path to the cumulative time-series CSV") -args = parser.parse_args() - -src_path = Path(args.input) -ts_path = Path(args.output) - -run = os.environ.get("GITHUB_RUN_NUMBER", "") -sha = os.environ.get("GITHUB_SHA", "")[:8] - -if not src_path.exists(): - sys.exit(f"missing {src_path}") - -with src_path.open(newline="") as fh: - reader = csv.DictReader(fh) - rows = list(reader) -if not rows: - sys.exit("empty source csv") -row = rows[0] - -# Build a new row with metadata and all original columns -new_line = { - "date": datetime.utcnow().isoformat(timespec="seconds"), - "run": run, - "sha": sha, -} -for key, val in row.items(): - try: - new_line[key] = int(val) - except Exception: - new_line[key] = val - -# Ensure header is consistent -write_header = not ts_path.exists() -if write_header: - header = ["date", "run", "sha"] + list(row.keys()) -else: - with ts_path.open(newline="") as fh: - header = next(csv.reader(fh)) - -ts_path.parent.mkdir(parents=True, exist_ok=True) -with ts_path.open("a", newline="") as out: - w = csv.DictWriter(out, fieldnames=header) - if write_header: - w.writeheader() - w.writerow(new_line) - -print(f"Appended row to {ts_path}") From 04101e54693031ae9f356c3f28fc206d3ec94006 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ant=C3=B3n=20Casas?= Date: Fri, 26 Sep 2025 09:58:29 +0200 Subject: [PATCH 13/21] Add plots to summary MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Antón Casas --- .github/workflows/performance.yml | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml index 25e589698..868a36642 100644 --- a/.github/workflows/performance.yml +++ b/.github/workflows/performance.yml @@ -167,3 +167,29 @@ jobs: name: performance-plots path: plots/*.png if-no-files-found: warn + + - name: Add plots to summary + if: always() + shell: bash + run: | + { + echo "## 📊 Performance Metrics (last 52 runs)" + echo + echo "### Complete Profile" + echo "![Complete Profile](./plots/complete_profile_plot.png)" + echo + echo "### Core Profile" + echo "![Core Profile](./plots/core_profile_plot.png)" + echo + echo "### Simple App Stack Usage" + echo "![Stack](./plots/stack_plot.png)" + echo + echo "### Increase of .bss memory by enabling profiles" + echo "![.bss Profiles](./plots/profiles_bss_plot.png)" + echo + echo "### Increase of .data memory by enabling profiles" + echo "![.data Profiles](./plots/profiles_data_plot.png)" + echo + echo "### Increase of .text memory by enabling profiles" + echo "![.text Profiles](./plots/profiles_text_plot.png)" + } >> "$GITHUB_STEP_SUMMARY" From 4fd8f7b2a85fe0fb6deff8af8ebc31232f5494cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ant=C3=B3n=20Casas?= Date: Fri, 26 Sep 2025 10:32:31 +0200 Subject: [PATCH 14/21] Add summary brief MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Antón Casas --- .github/workflows/performance.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml index 868a36642..4db7e653c 100644 --- a/.github/workflows/performance.yml +++ b/.github/workflows/performance.yml @@ -175,6 +175,11 @@ jobs: { echo "## 📊 Performance Metrics (last 52 runs)" echo + echo "The workflow triggering this run can be found [here](${{ github.event.workflow_run.html_url }})." + echo "The python scripts and the data used to generate these plots can be found in the [memory/performance](${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/tree/memory/performance) branch." + echo "The scripts and data can be modified without affecting develop nor master branches." + echo "memory/performance branch is unprotected and can be pushed to in case any manual modifications are needed." + echo echo "### Complete Profile" echo "![Complete Profile](./plots/complete_profile_plot.png)" echo From f9ace959331bd1d220022b15f26f7ca6a1628103 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ant=C3=B3n=20Casas?= Date: Fri, 26 Sep 2025 10:33:44 +0200 Subject: [PATCH 15/21] Switch to svg MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Antón Casas --- .github/workflows/performance.yml | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml index 4db7e653c..3192230bc 100644 --- a/.github/workflows/performance.yml +++ b/.github/workflows/performance.yml @@ -126,37 +126,37 @@ jobs: python3 ci/metrics/generate_plot.py \ --input ci/metrics/memory/complete_profile_timeseries.csv \ - --output plots/complete_profile_plot.png \ + --output plots/complete_profile_plot.svg \ --title "Complete Profile" \ --ylabel "Bytes" \ --last "$LAST_N" python3 ci/metrics/generate_plot.py \ --input ci/metrics/memory/core_profile_timeseries.csv \ - --output plots/core_profile_plot.png \ + --output plots/core_profile_plot.svg \ --title "Core Profile" \ --ylabel "Bytes" \ --last "$LAST_N" python3 ci/metrics/generate_plot.py \ --input ci/metrics/memory/stack_timeseries.csv \ - --output plots/stack_plot.png \ + --output plots/stack_plot.svg \ --title "Simple App Stack Usage" \ --ylabel "Bytes" \ --last "$LAST_N" python3 ci/metrics/generate_plot.py \ --input ci/metrics/memory/profiles_bss_timeseries.csv \ - --output plots/profiles_bss_plot.png \ + --output plots/profiles_bss_plot.svg \ --title "Increase of .bss memory by enabling profiles" \ --ylabel "Bytes" \ --last "$LAST_N" python3 ci/metrics/generate_plot.py \ --input ci/metrics/memory/profiles_data_timeseries.csv \ - --output plots/profiles_data_plot.png \ + --output plots/profiles_data_plot.svg \ --title "Increase of .data memory by enabling profiles" \ --ylabel "Bytes" \ --last "$LAST_N" python3 ci/metrics/generate_plot.py \ --input ci/metrics/memory/profiles_text_timeseries.csv \ - --output plots/profiles_text_plot.png \ + --output plots/profiles_text_plot.svg \ --title "Increase of .text memory by enabling profiles" \ --ylabel "Bytes" \ --last "$LAST_N" @@ -165,36 +165,38 @@ jobs: uses: actions/upload-artifact@v4 with: name: performance-plots - path: plots/*.png + path: plots/*.svg if-no-files-found: warn - name: Add plots to summary if: always() shell: bash run: | + b64() { base64 --wrap=0 "$1" 2>/dev/null || base64 -w0 "$1"; } + CPP=$(b64 plots/complete_profile_plot.svg) { echo "## 📊 Performance Metrics (last 52 runs)" echo - echo "The workflow triggering this run can be found [here](${{ github.event.workflow_run.html_url }})." + echo "The workflow triggering this run can be found in the default branch." echo "The python scripts and the data used to generate these plots can be found in the [memory/performance](${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/tree/memory/performance) branch." echo "The scripts and data can be modified without affecting develop nor master branches." echo "memory/performance branch is unprotected and can be pushed to in case any manual modifications are needed." echo echo "### Complete Profile" - echo "![Complete Profile](./plots/complete_profile_plot.png)" + echo "![Complete Profile](data:image/svg;base64,${CPP})" echo echo "### Core Profile" - echo "![Core Profile](./plots/core_profile_plot.png)" + echo "![Core Profile](./plots/core_profile_plot.svg)" echo echo "### Simple App Stack Usage" - echo "![Stack](./plots/stack_plot.png)" + echo "![Stack](./plots/stack_plot.svg)" echo echo "### Increase of .bss memory by enabling profiles" - echo "![.bss Profiles](./plots/profiles_bss_plot.png)" + echo "![.bss Profiles](./plots/profiles_bss_plot.svg)" echo echo "### Increase of .data memory by enabling profiles" - echo "![.data Profiles](./plots/profiles_data_plot.png)" + echo "![.data Profiles](./plots/profiles_data_plot.svg)" echo echo "### Increase of .text memory by enabling profiles" - echo "![.text Profiles](./plots/profiles_text_plot.png)" + echo "![.text Profiles](./plots/profiles_text_plot.svg)" } >> "$GITHUB_STEP_SUMMARY" From 2add5ee3ab585a385b917d2a068ba1a00eff0bb6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ant=C3=B3n=20Casas?= Date: Fri, 26 Sep 2025 11:03:45 +0200 Subject: [PATCH 16/21] Save images MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Antón Casas --- .github/workflows/performance.yml | 48 ++++++++++++++++++------------- 1 file changed, 28 insertions(+), 20 deletions(-) diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml index 3192230bc..db4b90fbf 100644 --- a/.github/workflows/performance.yml +++ b/.github/workflows/performance.yml @@ -64,11 +64,14 @@ jobs: needs: performance steps: + - name: Set branch name + run: echo "BR=memory/performance" >> $GITHUB_ENV # Use this branch to store performance results + - name: Checkout uses: actions/checkout@v5 with: fetch-depth: 0 - ref: memory/performance + ref: ${{ env.BR }} - name: Install Python and prerequisites shell: bash @@ -92,8 +95,6 @@ jobs: git config user.name "github-actions" git config user.email "github-actions@github.com" - BR="memory/performance" # Use this branch to store performance results - python3 ci/metrics/save_results.py \ --input results/complete_profile.csv \ --output ci/metrics/memory/complete_profile_timeseries.csv @@ -115,7 +116,7 @@ jobs: git add ci/metrics/memory/*_timeseries.csv git diff --cached --quiet || git commit -m "metrics: append run ${GITHUB_RUN_NUMBER} (${GITHUB_SHA::8})" - git push origin "${BR}" + git push origin "${{ env.BR }}" - name: Generate plots shell: bash @@ -126,77 +127,84 @@ jobs: python3 ci/metrics/generate_plot.py \ --input ci/metrics/memory/complete_profile_timeseries.csv \ - --output plots/complete_profile_plot.svg \ + --output ci/metrics/plots/complete_profile_plot.svg \ --title "Complete Profile" \ --ylabel "Bytes" \ --last "$LAST_N" python3 ci/metrics/generate_plot.py \ --input ci/metrics/memory/core_profile_timeseries.csv \ - --output plots/core_profile_plot.svg \ + --output ci/metrics/plots/core_profile_plot.svg \ --title "Core Profile" \ --ylabel "Bytes" \ --last "$LAST_N" python3 ci/metrics/generate_plot.py \ --input ci/metrics/memory/stack_timeseries.csv \ - --output plots/stack_plot.svg \ + --output ci/metrics/plots/stack_plot.svg \ --title "Simple App Stack Usage" \ --ylabel "Bytes" \ --last "$LAST_N" python3 ci/metrics/generate_plot.py \ --input ci/metrics/memory/profiles_bss_timeseries.csv \ - --output plots/profiles_bss_plot.svg \ + --output ci/metrics/plots/profiles_bss_plot.svg \ --title "Increase of .bss memory by enabling profiles" \ --ylabel "Bytes" \ --last "$LAST_N" python3 ci/metrics/generate_plot.py \ --input ci/metrics/memory/profiles_data_timeseries.csv \ - --output plots/profiles_data_plot.svg \ + --output ci/metrics/plots/profiles_data_plot.svg \ --title "Increase of .data memory by enabling profiles" \ --ylabel "Bytes" \ --last "$LAST_N" python3 ci/metrics/generate_plot.py \ --input ci/metrics/memory/profiles_text_timeseries.csv \ - --output plots/profiles_text_plot.svg \ + --output ci/metrics/plots/profiles_text_plot.svg \ --title "Increase of .text memory by enabling profiles" \ --ylabel "Bytes" \ --last "$LAST_N" + git add ci/metrics/plots/*.svg + git diff --cached --quiet || git commit -m "metrics: update plots (${GITHUB_RUN_NUMBER})" + git push origin "${{ env.BR }}" + - name: Upload plots uses: actions/upload-artifact@v4 with: name: performance-plots - path: plots/*.svg + path: ci/metrics/plots/*.svg if-no-files-found: warn - name: Add plots to summary if: always() shell: bash run: | - b64() { base64 --wrap=0 "$1" 2>/dev/null || base64 -w0 "$1"; } - CPP=$(b64 plots/complete_profile_plot.svg) + set -euo pipefail + + BASE="https://raw.githubusercontent.com/${GITHUB_REPOSITORY}/memory/performance/ci/metrics/plots" + CB="?r=${GITHUB_RUN_NUMBER}" + { echo "## 📊 Performance Metrics (last 52 runs)" echo echo "The workflow triggering this run can be found in the default branch." echo "The python scripts and the data used to generate these plots can be found in the [memory/performance](${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/tree/memory/performance) branch." echo "The scripts and data can be modified without affecting develop nor master branches." - echo "memory/performance branch is unprotected and can be pushed to in case any manual modifications are needed." + echo "The \`memory/performance\` branch is unprotected and can be pushed to in case any manual modifications are needed." echo echo "### Complete Profile" - echo "![Complete Profile](data:image/svg;base64,${CPP})" + echo "\"Complete" echo echo "### Core Profile" - echo "![Core Profile](./plots/core_profile_plot.svg)" + echo "\"Core" echo echo "### Simple App Stack Usage" - echo "![Stack](./plots/stack_plot.svg)" + echo "\"Stack\"" echo echo "### Increase of .bss memory by enabling profiles" - echo "![.bss Profiles](./plots/profiles_bss_plot.svg)" + echo "\"bss\"" echo echo "### Increase of .data memory by enabling profiles" - echo "![.data Profiles](./plots/profiles_data_plot.svg)" + echo "\"data\"" echo echo "### Increase of .text memory by enabling profiles" - echo "![.text Profiles](./plots/profiles_text_plot.svg)" + echo "\"text\"" } >> "$GITHUB_STEP_SUMMARY" From f5e803058767b3bc2a4f25268413ec4ebcc6f4bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ant=C3=B3n=20Casas?= Date: Fri, 26 Sep 2025 12:28:33 +0200 Subject: [PATCH 17/21] Add analysis of last two runs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Antón Casas --- .github/workflows/performance.yml | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml index db4b90fbf..33d51f864 100644 --- a/.github/workflows/performance.yml +++ b/.github/workflows/performance.yml @@ -58,8 +58,8 @@ jobs: build/profiles_text.csv if-no-files-found: error - analyze-metrics: - name: Analyze Metrics + save-metrics-and-plots: + name: Save Metrics and Generate Plots runs-on: ubuntu-24.04 needs: performance @@ -208,3 +208,26 @@ jobs: echo "### Increase of .text memory by enabling profiles" echo "\"text\"" } >> "$GITHUB_STEP_SUMMARY" + + analyze-metrics: + name: Analyze Metrics + runs-on: ubuntu-24.04 + needs: save-metrics-and-plots + + steps: + - name: Checkout + uses: actions/checkout@v5 + with: + fetch-depth: 0 + ref: memory/performance + + - name: Analyze results + shell: bash + run: | + set -euo pipefail + python3 ci/metrics/check_last_result.py --input ci/metrics/memory/complete_profile_timeseries.csv + python3 ci/metrics/check_last_result.py --input ci/metrics/memory/core_profile_timeseries.csv + python3 ci/metrics/check_last_result.py --input ci/metrics/memory/stack_timeseries.csv + python3 ci/metrics/check_last_result.py --input ci/metrics/memory/profiles_bss_timeseries.csv + python3 ci/metrics/check_last_result.py --input ci/metrics/memory/profiles_data_timeseries.csv + python3 ci/metrics/check_last_result.py --input ci/metrics/memory/profiles_text_timeseries.csv From b9866691c02a27dbeb0786e39b01ad3a8067a0d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ant=C3=B3n=20Casas?= Date: Fri, 26 Sep 2025 12:43:30 +0200 Subject: [PATCH 18/21] [Test] Fake memory difference MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit 35b0b7eff2f95af400f69889c4370995489e91f0. Signed-off-by: Antón Casas --- test/memory/consumption/stack_analysis.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/memory/consumption/stack_analysis.py b/test/memory/consumption/stack_analysis.py index 0c94a764e..37f2eea98 100644 --- a/test/memory/consumption/stack_analysis.py +++ b/test/memory/consumption/stack_analysis.py @@ -11,5 +11,4 @@ with open('stack.csv', 'w+') as csv_file: csv_writer = csv.writer(csv_file, delimiter=',', quoting=csv.QUOTE_ALL) csv_writer.writerow(['stack']) - csv_writer.writerow([max(stack)]) - + csv_writer.writerow([max(stack)+1111]) From 52bbb287ec8156e57f75f50e436fdcd4c1c8397b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ant=C3=B3n=20Casas?= Date: Fri, 26 Sep 2025 13:15:44 +0200 Subject: [PATCH 19/21] Revert "[Test] Fake memory difference" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit b9866691c02a27dbeb0786e39b01ad3a8067a0d0. Signed-off-by: Antón Casas --- test/memory/consumption/stack_analysis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/memory/consumption/stack_analysis.py b/test/memory/consumption/stack_analysis.py index 37f2eea98..0f83cbfdc 100644 --- a/test/memory/consumption/stack_analysis.py +++ b/test/memory/consumption/stack_analysis.py @@ -11,4 +11,4 @@ with open('stack.csv', 'w+') as csv_file: csv_writer = csv.writer(csv_file, delimiter=',', quoting=csv.QUOTE_ALL) csv_writer.writerow(['stack']) - csv_writer.writerow([max(stack)+1111]) + csv_writer.writerow([max(stack)]) From fff95d18e86fdbda15767a9dbbb2f94b82f2668a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ant=C3=B3n=20Casas?= Date: Fri, 26 Sep 2025 13:31:37 +0200 Subject: [PATCH 20/21] Add summary introduction MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Antón Casas --- .github/workflows/performance.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml index 33d51f864..0d77270a7 100644 --- a/.github/workflows/performance.yml +++ b/.github/workflows/performance.yml @@ -225,6 +225,12 @@ jobs: shell: bash run: | set -euo pipefail + + { + echo "## Analyze Memory Metrics" + echo "⚠️ Failure of this job should be treated as a warning. It can be safely ignored if there are intentional changes that affect memory usage." + } >> "$GITHUB_STEP_SUMMARY" + python3 ci/metrics/check_last_result.py --input ci/metrics/memory/complete_profile_timeseries.csv python3 ci/metrics/check_last_result.py --input ci/metrics/memory/core_profile_timeseries.csv python3 ci/metrics/check_last_result.py --input ci/metrics/memory/stack_timeseries.csv From cc903534a165a5674946893b165783e511745a70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ant=C3=B3n=20Casas?= Date: Fri, 26 Sep 2025 13:36:50 +0200 Subject: [PATCH 21/21] Final tweaks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Antón Casas --- .github/workflows/performance.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml index 0d77270a7..1feaa47c2 100644 --- a/.github/workflows/performance.yml +++ b/.github/workflows/performance.yml @@ -1,7 +1,7 @@ name: Weekly Performance on: - pull_request: + push: branches: - develop workflow_dispatch: @@ -228,6 +228,7 @@ jobs: { echo "## Analyze Memory Metrics" + echo echo "⚠️ Failure of this job should be treated as a warning. It can be safely ignored if there are intentional changes that affect memory usage." } >> "$GITHUB_STEP_SUMMARY"