nix(loadtest): group results by URL / report minimum latency / merge errors+mixed

.github/workflows/test.yaml

@@ -25,6 +25,10 @@ on:
       - test/**
       - '!**.md'
 
+defaults:
+  run:
+    shell: bash
+
 concurrency:
   # Terminate all previous runs of the same workflow for pull requests
   group: test-${{ github.head_ref || github.run_id }}
@@ -127,7 +131,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        kind: ['mixed', 'errors', 'jwt-hs', 'jwt-hs-cache', 'jwt-hs-cache-worst', 'jwt-rsa', 'jwt-rsa-cache', 'jwt-rsa-cache-worst']
+        kind: ['mixed', 'jwt-hs', 'jwt-hs-cache', 'jwt-hs-cache-worst', 'jwt-rsa', 'jwt-rsa-cache', 'jwt-rsa-cache-worst']
     name: Loadtest
     runs-on: ubuntu-24.04
     steps:
@@ -138,7 +142,7 @@ jobs:
         uses: ./.github/actions/setup-nix
         with:
           authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
-          tools: loadtest.loadtestAgainst.bin loadtest.report.bin cabalTools.update.bin
+          tools: loadtest.loadtestAgainst.bin loadtest.report.bin loadtest.report-load.bin cabalTools.update.bin
 
       - run: postgrest-cabal-update
 
@@ -152,7 +156,35 @@ jobs:
             latest_tag=$(git tag --merged HEAD --sort=-creatordate "v*" | head -n1)
           fi
           postgrest-loadtest-against -k ${{ matrix.kind }} "$TARGET_BRANCH" "$latest_tag"
-          postgrest-loadtest-report -g ${{ matrix.kind }} >> "$GITHUB_STEP_SUMMARY"
+
+      - name: Report P50
+        # This step checks whether any red cross indicators (:x:) are present in the step summary.
+        # The loadtest reporter writes them when any of individual steps fails the performance
+        # regression threshold.
+        run: |
+          postgrest-loadtest-report -g ${{ matrix.kind }} -p 50 \
+            | tee "$GITHUB_STEP_SUMMARY" \
+            | grep -v ':x:'
+
+      - name: Report P0
+        if: always()
+        run: |
+          postgrest-loadtest-report -g ${{ matrix.kind }} -p 0 >> "$GITHUB_STEP_SUMMARY"
+
+      - name: Report P90
+        if: always()
+        run: |
+          postgrest-loadtest-report -g ${{ matrix.kind }} -p 90 >> "$GITHUB_STEP_SUMMARY"
+
+      - name: Report P95
+        if: always()
+        run: |
+          postgrest-loadtest-report -g ${{ matrix.kind }} -p 95 >> "$GITHUB_STEP_SUMMARY"
+
+      - name: Report CPU/MEM
+        if: always()
+        run: |
+          postgrest-loadtest-report-load -g ${{ matrix.kind }} >> "$GITHUB_STEP_SUMMARY"
 
   flake:
     strategy:

nix/tools/loadtest.nix

@@ -46,7 +46,7 @@ let
           "ARG_OPTIONAL_SINGLE([testdir], [t], [Directory to load tests and fixtures from], [./test/load])"
           "ARG_OPTIONAL_SINGLE([kind], [k], [Kind of loadtest], [mixed])"
           "ARG_OPTIONAL_SINGLE([method],, [HTTP method used for the jwt loadtests], [OPTIONS])"
-          "ARG_TYPE_GROUP_SET([KIND], [KIND], [kind], [mixed,errors,jwt-hs,jwt-hs-cache,jwt-hs-cache-worst,jwt-rsa,jwt-rsa-cache,jwt-rsa-cache-worst])"
+          "ARG_TYPE_GROUP_SET([KIND], [KIND], [kind], [mixed,jwt-hs,jwt-hs-cache,jwt-hs-cache-worst,jwt-rsa,jwt-rsa-cache,jwt-rsa-cache-worst])"
           "ARG_TYPE_GROUP_SET([METHOD], [METHOD], [method], [OPTIONS,GET])"
           "ARG_OPTIONAL_SINGLE([monitor], [m], [Monitoring file], [./loadtest/result.csv])"
           "ARG_LEFTOVERS([additional vegeta arguments])"
@@ -139,27 +139,19 @@ let
             ${runner} -lazy -targets gen_targets.http -output \"$abs_output\" \"''${_arg_leftovers[@]}\""
             ;;
 
+          # here we sleep purposefully to check how much memory does the schema cache consume in the final report
           mixed)
             # shellcheck disable=SC2145
             ${withTools.withPg} -f "$_arg_testdir"/fixtures.sql \
-            ${withTools.withPgrst} -m "$_arg_monitor" \
-            sh -c "cd \"$_arg_testdir\" && \
-            ${runner} -targets targets.http -output \"$abs_output\" \"''${_arg_leftovers[@]}\""
-            ;;
-
-          # here we sleep purposefully to check how much memory does the schema cache consume in the final report
-          errors)
-            # shellcheck disable=SC2145
-            ${withTools.withPg} -f "$_arg_testdir"/errors.sql \
             ${withTools.withPgrst} --timeout 2 --sleep 5 -m "$_arg_monitor" \
             sh -c "cd \"$_arg_testdir\" && \
-            ${runner} -targets errors.http -output \"$abs_output\" \"''${_arg_leftovers[@]}\""
+            ${runner} -targets targets.http -output \"$abs_output\" \"''${_arg_leftovers[@]}\""
             ;;
         esac
 
         ${vegeta}/bin/vegeta report -type=text "$_arg_output"
 
-        if [ "$_arg_kind" != "errors" ]; then
+        if [ "$_arg_kind" != "mixed" ]; then
           # fail in case 401 happened on jwt loadtests
           unauthorized_count="$(${vegeta}/bin/vegeta report -type=json "$_arg_output" \
             | ${jq}/bin/jq -r '.status_codes["401"] // 0')"
@@ -262,12 +254,14 @@ let
         docs = "Create a named json report for a single result file.";
         args = [
           "ARG_POSITIONAL_SINGLE([file], [Filename of result to create report for])"
+          "ARG_OPTIONAL_SINGLE([percentile], [p], [Percentile to report latency for], 50)"
           "ARG_LEFTOVERS([additional vegeta arguments])"
         ];
         workingDir = "/";
       }
       ''
-        ${vegeta}/bin/vegeta report -type=json "$_arg_file" \
+        ${vegeta}/bin/vegeta encode "$_arg_file" \
+          | ${jq}/bin/jq --arg percentile "$_arg_percentile" --slurp 'map(select(.url != "")) | group_by("\(.code) \(.method) \(.url)") | map({("\(.[0].code) \(.[0].method) \(.[0].url)" | sub("http://postgrest";"")): map(.latency) | sort | .[(length-1) * ($percentile | tonumber) / 100 | floor] / 10e3 }) | .[]' \
           | ${jq}/bin/jq --arg branch "$(basename "$_arg_file" .bin)" '. + {branch: $branch}'
       '';
 
@@ -280,12 +274,28 @@ let
         import sys
         import pandas as pd
 
+
+        def evaluate_change(df):
+            return ((df['head'] / df['main'] - 1) * 100) \
+              .map(lambda r: "{icon} {ratio:.1f} %".format(
+                ratio=r,
+                # Hardcoded failure threshold for CI is 5% here.
+                icon="" if r < 5 else ":x:"
+              ))
+
+
         pd.read_json(sys.stdin) \
-          .set_index('param') \
-          .drop(['branch', 'earliest', 'end', 'latest']) \
-          .fillna("") \
+          .rename(columns={'latency': sys.argv[1]}) \
+          .set_index(sys.argv[1]) \
+          .drop(['branch']) \
           .convert_dtypes() \
-          .to_markdown(sys.stdout, floatfmt='.0f')
+          .assign(change=evaluate_change) \
+          .to_markdown(
+            sys.stdout,
+            floatfmt='.1f',
+            colglobalalign='right',
+            colalign=('left',)
+          )
       '';
 
 
@@ -296,20 +306,31 @@ let
         docs = "Create a report of all loadtest reports as markdown.";
         args = [
           "ARG_OPTIONAL_SINGLE([group], [g], [Marker to group results])"
+          "ARG_OPTIONAL_SINGLE([percentile], [p], [Percentile to report latency for], 50)"
         ];
         workingDir = "/";
       }
       ''
-        marker=''${_arg_group:+"($_arg_group)"}
-
-        echo -e "## Loadtest results $marker\n"
+        echo -e "## Loadtest results $_arg_group (P$_arg_percentile)\n"
 
-        find loadtest -type f -iname '*.bin' -exec ${reporter} {} \; \
-          | ${jq}/bin/jq '[paths(scalars) as $path | {param: $path | join("."), (.branch): getpath($path)}]' \
-          | ${jq}/bin/jq --slurp 'flatten | group_by(.param) | map(add)' \
-          | ${toMarkdown}
+        find loadtest -type f -iname '*.bin' -exec ${reporter} -p "$_arg_percentile" {} \; \
+          | ${jq}/bin/jq '[paths(scalars) as $path | {latency: $path | join("."), (.branch): getpath($path)}]' \
+          | ${jq}/bin/jq --slurp 'flatten | group_by(.latency) | map(add)' \
+          | ${toMarkdown} "P$_arg_percentile latency [μs]"
+      '';
 
-        echo -e "\n\n## Loadtest elapsed seconds vs CPU/MEM usage $marker\n"
+  report-load =
+    checkedShellScript
+      {
+        name = "postgrest-loadtest-report-load";
+        docs = "Create a report of all CPU/MEM usage as markdown.";
+        args = [
+          "ARG_OPTIONAL_SINGLE([group], [g], [Marker to group results])"
+        ];
+        workingDir = "/";
+      }
+      ''
+        echo -e "\n\n## Loadtest elapsed seconds vs CPU/MEM usage $_arg_group\n"
 
         find loadtest -type f -iname '*.csv' \
           | sort -m \
@@ -341,5 +362,5 @@ let
 in
 buildToolbox {
   name = "postgrest-loadtest";
-  tools = { inherit loadtest loadtestAgainst report; };
+  tools = { inherit loadtest loadtestAgainst report report-load; };
 }

test/load/fixtures.sql

@@ -49,3 +49,9 @@ REVOKE ALL PRIVILEGES ON TABLE
 FROM postgrest_test_anonymous;
 
 GRANT ALL ON TABLE authors_only TO postgrest_test_author;
+
+SELECT format('CREATE TABLE test.actors_%s ();', n)
+FROM generate_series(1, 450) n -- 500 is the upper limit for table not found error hint generation
+\gexec
+
+-- TODO add many function for fuzzy search (somehow this is making the loadtest start slow)

test/load/targets.http

@@ -33,3 +33,19 @@ POST http://postgrest/rpc/call_me
 @rpc.json
 
 OPTIONS http://postgrest/actors
+
+# Misspelled relations
+GET http://postgrest/actoxs?actor=eq.1
+Prefer: tx=commit
+
+# Misspelled relations on embeds
+GET http://postgrest/actors?select=*,rolws(*,films(*))
+Prefer: tx=commit
+
+# Misspelled function names
+GET http://postgrest/rpc/call_me_x?name=John
+Prefer: tx=commit
+
+# Permission denied errors
+GET http://postgrest/actors_1
+Prefer: tx=commit