nix(loadtest): report percentage change

This reports the percentage change between the current head branch and
the main branch, which is exactly the number we'll want to make our
decisions on "success or fail" on.

CI failures will initially be reported for regressions of 5% or more on
an individual number.

Author: wolfgangwalther

.github/workflows/test.yaml

@@ -25,6 +25,10 @@ on:
       - test/**
       - '!**.md'
 
+defaults:
+  run:
+    shell: bash
+
 concurrency:
   # Terminate all previous runs of the same workflow for pull requests
   group: test-${{ github.head_ref || github.run_id }}
@@ -152,10 +156,34 @@ jobs:
             latest_tag=$(git tag --merged HEAD --sort=-creatordate "v*" | head -n1)
           fi
           postgrest-loadtest-against -k ${{ matrix.kind }} "$TARGET_BRANCH" "$latest_tag"
+
+      - name: Report P50
+        # This step checks whether any red cross indicators (:x:) are present in the step summary.
+        # The loadtest reporter writes them when any of individual steps fails the performance
+        # regression threshold.
+        run: |
+          postgrest-loadtest-report -g ${{ matrix.kind }} -p 50 \
+            | tee "$GITHUB_STEP_SUMMARY" \
+            | grep -v ':x:'
+
+      - name: Report P0
+        if: always()
+        run: |
           postgrest-loadtest-report -g ${{ matrix.kind }} -p 0 >> "$GITHUB_STEP_SUMMARY"
-          postgrest-loadtest-report -g ${{ matrix.kind }} -p 50 >> "$GITHUB_STEP_SUMMARY"
+
+      - name: Report P90
+        if: always()
+        run: |
           postgrest-loadtest-report -g ${{ matrix.kind }} -p 90 >> "$GITHUB_STEP_SUMMARY"
+
+      - name: Report P95
+        if: always()
+        run: |
           postgrest-loadtest-report -g ${{ matrix.kind }} -p 95 >> "$GITHUB_STEP_SUMMARY"
+
+      - name: Report CPU/MEM
+        if: always()
+        run: |
           postgrest-loadtest-report-load -g ${{ matrix.kind }} >> "$GITHUB_STEP_SUMMARY"
 
   flake:

nix/tools/loadtest.nix

@@ -274,12 +274,28 @@ let
         import sys
         import pandas as pd
 
+
+        def evaluate_change(df):
+            return ((df['head'] / df['main'] - 1) * 100) \
+              .map(lambda r: "{icon} {ratio:.1f} %".format(
+                ratio=r,
+                # Hardcoded failure threshold for CI is 5% here.
+                icon="" if r < 5 else ":x:"
+              ))
+
+
         pd.read_json(sys.stdin) \
           .rename(columns={'latency': sys.argv[1]}) \
           .set_index(sys.argv[1]) \
           .drop(['branch']) \
           .convert_dtypes() \
-          .to_markdown(sys.stdout, floatfmt='.1f')
+          .assign(change=evaluate_change) \
+          .to_markdown(
+            sys.stdout,
+            floatfmt='.1f',
+            colglobalalign='right',
+            colalign=('left',)
+          )
       '';