nix(loadtest): move errors into mixed

Because we separate loadtest results per URL now, we can move the error
tests into the regular mixed bag of loadtests - we will be able to tell
from the misspelled URLs when we hit a regression in that area.

We should be able to do similar things for JWT tests, but we'll need
more infrastructure here.

Author: wolfgangwalther

.github/workflows/test.yaml

@@ -127,7 +127,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        kind: ['mixed', 'errors', 'jwt-hs', 'jwt-hs-cache', 'jwt-hs-cache-worst', 'jwt-rsa', 'jwt-rsa-cache', 'jwt-rsa-cache-worst']
+        kind: ['mixed', 'jwt-hs', 'jwt-hs-cache', 'jwt-hs-cache-worst', 'jwt-rsa', 'jwt-rsa-cache', 'jwt-rsa-cache-worst']
     name: Loadtest
     runs-on: ubuntu-24.04
     steps:

nix/tools/loadtest.nix

@@ -46,7 +46,7 @@ let
           "ARG_OPTIONAL_SINGLE([testdir], [t], [Directory to load tests and fixtures from], [./test/load])"
           "ARG_OPTIONAL_SINGLE([kind], [k], [Kind of loadtest], [mixed])"
           "ARG_OPTIONAL_SINGLE([method],, [HTTP method used for the jwt loadtests], [OPTIONS])"
-          "ARG_TYPE_GROUP_SET([KIND], [KIND], [kind], [mixed,errors,jwt-hs,jwt-hs-cache,jwt-hs-cache-worst,jwt-rsa,jwt-rsa-cache,jwt-rsa-cache-worst])"
+          "ARG_TYPE_GROUP_SET([KIND], [KIND], [kind], [mixed,jwt-hs,jwt-hs-cache,jwt-hs-cache-worst,jwt-rsa,jwt-rsa-cache,jwt-rsa-cache-worst])"
           "ARG_TYPE_GROUP_SET([METHOD], [METHOD], [method], [OPTIONS,GET])"
           "ARG_OPTIONAL_SINGLE([monitor], [m], [Monitoring file], [./loadtest/result.csv])"
           "ARG_LEFTOVERS([additional vegeta arguments])"
@@ -139,27 +139,19 @@ let
             ${runner} -lazy -targets gen_targets.http -output \"$abs_output\" \"''${_arg_leftovers[@]}\""
             ;;
 
+          # here we sleep purposefully to check how much memory does the schema cache consume in the final report
           mixed)
             # shellcheck disable=SC2145
             ${withTools.withPg} -f "$_arg_testdir"/fixtures.sql \
-            ${withTools.withPgrst} -m "$_arg_monitor" \
-            sh -c "cd \"$_arg_testdir\" && \
-            ${runner} -targets targets.http -output \"$abs_output\" \"''${_arg_leftovers[@]}\""
-            ;;
-
-          # here we sleep purposefully to check how much memory does the schema cache consume in the final report
-          errors)
-            # shellcheck disable=SC2145
-            ${withTools.withPg} -f "$_arg_testdir"/errors.sql \
             ${withTools.withPgrst} --timeout 2 --sleep 5 -m "$_arg_monitor" \
             sh -c "cd \"$_arg_testdir\" && \
-            ${runner} -targets errors.http -output \"$abs_output\" \"''${_arg_leftovers[@]}\""
+            ${runner} -targets targets.http -output \"$abs_output\" \"''${_arg_leftovers[@]}\""
             ;;
         esac
 
         ${vegeta}/bin/vegeta report -type=text "$_arg_output"
 
-        if [ "$_arg_kind" != "errors" ]; then
+        if [ "$_arg_kind" != "mixed" ]; then
           # fail in case 401 happened on jwt loadtests
           unauthorized_count="$(${vegeta}/bin/vegeta report -type=json "$_arg_output" \
             | ${jq}/bin/jq -r '.status_codes["401"] // 0')"

test/load/errors.http

@@ -49,3 +49,9 @@ REVOKE ALL PRIVILEGES ON TABLE
 FROM postgrest_test_anonymous;
 
 GRANT ALL ON TABLE authors_only TO postgrest_test_author;
+
+SELECT format('CREATE TABLE test.actors_%s ();', n)
+FROM generate_series(1, 450) n -- 500 is the upper limit for table not found error hint generation
+\gexec
+
+-- TODO add many function for fuzzy search (somehow this is making the loadtest start slow)

test/load/errors.sql

@@ -33,3 +33,19 @@ POST http://postgrest/rpc/call_me
 @rpc.json
 
 OPTIONS http://postgrest/actors
+
+# Misspelled relations
+GET http://postgrest/actoxs?actor=eq.1
+Prefer: tx=commit
+
+# Misspelled relations on embeds
+GET http://postgrest/actors?select=*,rolws(*,films(*))
+Prefer: tx=commit
+
+# Misspelled function names
+GET http://postgrest/rpc/call_me_x?name=John
+Prefer: tx=commit
+
+# Permission denied errors
+GET http://postgrest/actors_1
+Prefer: tx=commit