fix ci fps percent change

Author: gabrielburnworth

.github/workflows/render.yml

@@ -94,6 +94,14 @@ jobs:
             echo "scene_metrics_found=true" >> "$GITHUB_OUTPUT"
           fi
 
+      - name: Restore fps history
+        id: restore_fps_history
+        run: |
+          scripts/ci/git-restore-artifact-branch-file "$FPS_HISTORY.csv"
+          if compgen -G "/tmp/$FPS_HISTORY.csv" >/dev/null; then
+            echo "fps_history_found=true" >> "$GITHUB_OUTPUT"
+          fi
+
       - name: Restore FPS metrics
         if: steps.restore_metric_history.outputs.scene_metrics_found != 'true'
         uses: actions/cache/restore@v5
@@ -124,12 +132,7 @@ jobs:
         run: scripts/ci/run-playwright-render
 
       - name: Track FPS value
-        run: |
-          PREV=$(scripts/ci/previous-fps-value)
-          echo "Previous value: $PREV fps"
-          percent_change=$(scripts/ci/percent-change --new "$FPS_VALUE" --old "$PREV")
-          echo "$FPS_VALUE fps ($percent_change% change)"
-          echo "PERCENT_CHANGE=$percent_change" >> "$GITHUB_ENV"
+        run: scripts/ci/track-fps-history
 
       - name: Plot metric CSVs
         if: always()

scripts/ci/combine-render-images

@@ -18,6 +18,7 @@ const screenshotName = requiredEnv('SCREENSHOT_NAME');
 const fpsSamplesName = requiredEnv('FPS_SAMPLES_NAME');
 const sceneMetricsName = requiredEnv('SCENE_METRICS_NAME');
 const feCoverageName = requiredEnv('FE_COVERAGE_NAME');
+const fpsHistoryName = requiredEnv('FPS_HISTORY');
 const renderImageScale = Number(process.env.COMBINED_RENDER_IMAGE_SCALE || 3);
 if (!Number.isFinite(renderImageScale) || renderImageScale <= 0) {
     console.error('COMBINED_RENDER_IMAGE_SCALE must be a positive number');
@@ -70,7 +71,11 @@ const rows = [
         screenshotCell('404'),
         screenshotCell('dropdown'),
     ],
-    [['fe coverage', imagePath(feCoverageName)], [], []],
+    [
+        ['fe coverage', imagePath(feCoverageName)],
+        ['fps history', imagePath(fpsHistoryName)],
+        [],
+    ],
 ];
 
 const escapeHtml = value => String(value)

scripts/ci/git-publish-render-artifacts

@@ -5,6 +5,7 @@ set -euo pipefail
 : "${COMBINED_RENDER_IMAGE:?COMBINED_RENDER_IMAGE is required}"
 : "${CHOSEN_FPS:?CHOSEN_FPS is required}"
 : "${SCENE_METRICS_NAME:?SCENE_METRICS_NAME is required}"
+: "${FPS_HISTORY:?FPS_HISTORY is required}"
 : "${FE_COVERAGE_NAME:?FE_COVERAGE_NAME is required}"
 : "${GITHUB_REPOSITORY:?GITHUB_REPOSITORY is required}"
 : "${GITHUB_RUN_ID:?GITHUB_RUN_ID is required}"
@@ -28,11 +29,11 @@ for image in "$COMBINED_RENDER_IMAGE"; do
 done
 cp "/tmp/$CHOSEN_FPS.csv" "$worktree/build/latest/$CHOSEN_FPS.csv"
 cp "/tmp/$CHOSEN_FPS.csv" "$worktree/build/${GITHUB_RUN_ID}/$CHOSEN_FPS.csv"
-for metric in "/tmp/$SCENE_METRICS_NAME"*.csv "/tmp/$FE_COVERAGE_NAME.csv"; do
+for metric in "/tmp/$SCENE_METRICS_NAME"*.csv "/tmp/$FPS_HISTORY.csv" "/tmp/$FE_COVERAGE_NAME.csv"; do
   [ -f "$metric" ] && cp "$metric" "$worktree/$(basename "$metric")"
 done
 git -C "$worktree" add build
-for artifact in "$worktree/$SCENE_METRICS_NAME"*.csv "$worktree/$FE_COVERAGE_NAME.csv"; do
+for artifact in "$worktree/$SCENE_METRICS_NAME"*.csv "$worktree/$FPS_HISTORY.csv" "$worktree/$FE_COVERAGE_NAME.csv"; do
   [ -f "$artifact" ] && git -C "$worktree" add "$(basename "$artifact")"
 done
 git -C "$worktree" commit -m "Update CI image artifacts for ${GITHUB_RUN_ID}" || true

scripts/ci/previous-fps-value

@@ -12,9 +12,9 @@ def required_env(name):
     return value
 
 
-chosen_metrics = required_env("CHOSEN_METRICS")
+fps_history = required_env("FPS_HISTORY")
 fallback = required_env("FALLBACK_FPS_VALUE")
-path = f"/tmp/{chosen_metrics}.csv"
+path = f"/tmp/{fps_history}.csv"
 
 try:
     with open(path, newline="") as f:
@@ -28,9 +28,9 @@ except FileNotFoundError:
     rows = []
 
 values = []
-for row in rows[:-1]:
+for row in rows:
     try:
-        values.append(float(row["FPS"]))
+        values.append(float(row["fps"]))
     except (KeyError, TypeError, ValueError):
         pass
 

scripts/ci/render-env

@@ -5,6 +5,7 @@
 : "${CHOSEN_FPS:=${FPS_SAMPLES_NAME}_${CHOSEN_NAME}}"
 : "${SCENE_METRICS_NAME:=scene_metrics}"
 : "${CHOSEN_METRICS:=${SCENE_METRICS_NAME}_${CHOSEN_NAME}}"
+: "${FPS_HISTORY:=fps_history}"
 : "${COMBINED_RENDER_IMAGE:=render_summary}"
 : "${COMBINED_RENDER_IMAGE_SCALE:=3}"
 : "${FE_COVERAGE_NAME:=fe_coverage}"

scripts/ci/test_python_scripts.py

@@ -64,35 +64,67 @@ def test_percent_change_handles_zero_old_value(self):
         self.assertEqual(code, 0)
         self.assertEqual(stdout, "n/a\n")
 
-    def test_previous_fps_value_reads_previous_row(self):
-        metrics_name = f"scene_metrics_test_{os.getpid()}"
-        path = Path("/tmp") / f"{metrics_name}.csv"
+    def test_previous_fps_value_reads_latest_history_row(self):
+        history_name = f"fps_history_test_{os.getpid()}"
+        path = Path("/tmp") / f"{history_name}.csv"
         path.write_text(
-            "epoch, FPS, Calls\n"
-            "1, 80, 1\n"
-            "2, 90, 1\n"
-            "3, 100, 1\n",
+            "fps,% change\n"
+            "80,0.00\n"
+            "90,12.50\n"
+            "100,11.11\n",
         )
         try:
             code, stdout, stderr = run_script("previous-fps-value", env={
-                "CHOSEN_METRICS": metrics_name,
+                "FPS_HISTORY": history_name,
                 "FALLBACK_FPS_VALUE": "100",
             })
         finally:
             path.unlink(missing_ok=True)
 
         self.assertEqual(code, 0)
-        self.assertEqual(stdout, "90.00\n")
+        self.assertEqual(stdout, "100.00\n")
         self.assertEqual(stderr, "")
 
     def test_previous_fps_value_uses_fallback_without_history(self):
         code, stdout, _stderr = run_script("previous-fps-value", env={
-            "CHOSEN_METRICS": f"missing_metrics_{os.getpid()}",
+            "FPS_HISTORY": f"missing_history_{os.getpid()}",
             "FALLBACK_FPS_VALUE": "100",
         })
         self.assertEqual(code, 0)
         self.assertEqual(stdout, "100\n")
 
+    def test_track_fps_history_appends_csv(self):
+        history_name = f"fps_history_track_test_{os.getpid()}"
+        path = Path("/tmp") / f"{history_name}.csv"
+        github_env = Path("/tmp") / f"github_env_track_test_{os.getpid()}"
+        path.write_text(
+            "fps,% change,commit sha\n"
+            "100.00,0.00,old123\n")
+        try:
+            code, stdout, stderr = run_script("track-fps-history", env={
+                "FPS_HISTORY": history_name,
+                "FPS_VALUE": "106.04",
+                "FALLBACK_FPS_VALUE": "100",
+                "GITHUB_SHA": "0123456789abcdef",
+                "GITHUB_ENV": str(github_env),
+                "PATH": os.environ["PATH"],
+            })
+            self.assertEqual(code, 0)
+            self.assertEqual(
+                stdout,
+                "Previous value: 100.00 fps\n"
+                "106.04 fps (6.04% change)\n")
+            self.assertEqual(stderr, "")
+            self.assertEqual(
+                path.read_text(),
+                "fps,% change,commit sha\n"
+                "100.00,0.00,old123\n"
+                "106.04,6.04,0123456789\n")
+            self.assertEqual(github_env.read_text(), "PERCENT_CHANGE=6.04\n")
+        finally:
+            path.unlink(missing_ok=True)
+            github_env.unlink(missing_ok=True)
+
     def test_render_url_records_outputs_field_separated_records(self):
         with tempfile.NamedTemporaryFile("w", delete=False) as file:
             json.dump([{

scripts/ci/track-fps-history

@@ -0,0 +1,38 @@
+#!/usr/bin/env python3
+import csv
+import os
+from pathlib import Path
+import subprocess
+
+
+fps_history = os.environ["FPS_HISTORY"]
+fps_value = os.environ["FPS_VALUE"]
+commit_sha = os.environ["GITHUB_SHA"][:10]
+github_env = os.environ["GITHUB_ENV"]
+csv_path = Path("/tmp") / f"{fps_history}.csv"
+csv_header = ["fps", "% change", "commit sha"]
+
+previous = subprocess.check_output(
+    ["scripts/ci/previous-fps-value"],
+    text=True,
+).strip()
+print(f"Previous value: {previous} fps")
+
+percent_change = subprocess.check_output([
+    "scripts/ci/percent-change",
+    "--new",
+    fps_value,
+    "--old",
+    previous,
+], text=True).strip()
+print(f"{fps_value} fps ({percent_change}% change)")
+
+with open(github_env, "a") as env_file:
+    env_file.write(f"PERCENT_CHANGE={percent_change}\n")
+
+if not csv_path.exists():
+    csv_path.write_text(",".join(csv_header) + "\n")
+
+with csv_path.open("a", newline="") as csv_file:
+    writer = csv.writer(csv_file)
+    writer.writerow([fps_value, percent_change, commit_sha])

scripts/metric_plot.js

@@ -370,6 +370,7 @@ const title = (basename, prefix) => {
 const inferCsvPlot = ({ headers, rows }, filename = '') => {
     const basename = path.basename(filename);
     const isSceneMetricsCsv = /^scene_metrics(?:_[^/]+)?\.csv$/.test(basename);
+    const isFpsHistoryCsv = basename === 'fps_history.csv';
     const firstHeader = headers[0];
     const xHeader = headers.find(header =>
         ['elapsed seconds', 'elapsedSeconds'].includes(header));
@@ -405,6 +406,18 @@ const inferCsvPlot = ({ headers, rows }, filename = '') => {
             latestCommitSha: latestCommitShaFor(rows, ['percent']),
         };
     }
+    if (isFpsHistoryCsv && headers.includes('fps')) {
+        return {
+            title: 'FPS history',
+            xLabel: 'Runs',
+            yMin: 0,
+            yMaxBaseline: 200,
+            yTickInterval: 100,
+            decimalValues: true,
+            samples: rows.map((row, index) => sampleFor(row, index, 'fps')),
+            latestCommitSha: latestCommitShaFor(rows, ['fps']),
+        };
+    }
     if (headers.includes('FPS')) {
         if (isSceneMetricsCsv) {
             return {
@@ -495,6 +508,7 @@ function printUsage() {
         '',
         'Supported CSV inputs:',
         '  fps_samples.csv     Plots the fps column against elapsed seconds.',
+        '  fps_history.csv     Plots historical fps values.',
         '  fe_coverage.csv     Plots the percent column.',
         '  scene_metrics.csv   Plots numeric columns.',
         '',