diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index ad9e7d111d..ca91b3cb74 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -127,7 +127,7 @@ jobs: strategy: fail-fast: false matrix: - kind: ['mixed', 'errors', 'jwt-hs', 'jwt-hs-cache', 'jwt-hs-cache-worst', 'jwt-rsa', 'jwt-rsa-cache', 'jwt-rsa-cache-worst'] + kind: ['mixed', 'jwt-hs', 'jwt-hs-cache', 'jwt-hs-cache-worst', 'jwt-rsa', 'jwt-rsa-cache', 'jwt-rsa-cache-worst'] name: Loadtest runs-on: ubuntu-24.04 steps: diff --git a/nix/tools/loadtest.nix b/nix/tools/loadtest.nix index 034fb63282..9cbc81c11c 100644 --- a/nix/tools/loadtest.nix +++ b/nix/tools/loadtest.nix @@ -46,7 +46,7 @@ let "ARG_OPTIONAL_SINGLE([testdir], [t], [Directory to load tests and fixtures from], [./test/load])" "ARG_OPTIONAL_SINGLE([kind], [k], [Kind of loadtest], [mixed])" "ARG_OPTIONAL_SINGLE([method],, [HTTP method used for the jwt loadtests], [OPTIONS])" - "ARG_TYPE_GROUP_SET([KIND], [KIND], [kind], [mixed,errors,jwt-hs,jwt-hs-cache,jwt-hs-cache-worst,jwt-rsa,jwt-rsa-cache,jwt-rsa-cache-worst])" + "ARG_TYPE_GROUP_SET([KIND], [KIND], [kind], [mixed,jwt-hs,jwt-hs-cache,jwt-hs-cache-worst,jwt-rsa,jwt-rsa-cache,jwt-rsa-cache-worst])" "ARG_TYPE_GROUP_SET([METHOD], [METHOD], [method], [OPTIONS,GET])" "ARG_OPTIONAL_SINGLE([monitor], [m], [Monitoring file], [./loadtest/result.csv])" "ARG_LEFTOVERS([additional vegeta arguments])" @@ -139,27 +139,19 @@ let ${runner} -lazy -targets gen_targets.http -output \"$abs_output\" \"''${_arg_leftovers[@]}\"" ;; + # here we sleep purposefully to check how much memory does the schema cache consume in the final report mixed) # shellcheck disable=SC2145 ${withTools.withPg} -f "$_arg_testdir"/fixtures.sql \ - ${withTools.withPgrst} -m "$_arg_monitor" \ - sh -c "cd \"$_arg_testdir\" && \ - ${runner} -targets targets.http -output \"$abs_output\" \"''${_arg_leftovers[@]}\"" - ;; - - # here we sleep purposefully to check how much memory does the schema cache consume in the final report - errors) - # shellcheck disable=SC2145 - ${withTools.withPg} -f "$_arg_testdir"/errors.sql \ ${withTools.withPgrst} --timeout 2 --sleep 5 -m "$_arg_monitor" \ sh -c "cd \"$_arg_testdir\" && \ - ${runner} -targets errors.http -output \"$abs_output\" \"''${_arg_leftovers[@]}\"" + ${runner} -targets targets.http -output \"$abs_output\" \"''${_arg_leftovers[@]}\"" ;; esac ${vegeta}/bin/vegeta report -type=text "$_arg_output" - if [ "$_arg_kind" != "errors" ]; then + if [ "$_arg_kind" != "mixed" ]; then # fail in case 401 happened on jwt loadtests unauthorized_count="$(${vegeta}/bin/vegeta report -type=json "$_arg_output" \ | ${jq}/bin/jq -r '.status_codes["401"] // 0')" @@ -267,7 +259,8 @@ let workingDir = "/"; } '' - ${vegeta}/bin/vegeta report -type=json "$_arg_file" \ + ${vegeta}/bin/vegeta encode "$_arg_file" \ + | ${jq}/bin/jq --slurp 'map(select(.url != "")) | group_by("\(.code) \(.method) \(.url)") | map({("\(.[0].code) \(.[0].method) \(.[0].url)" | sub("http://postgrest";"")): map(.latency) | sort | .[length * 0.50 | floor] / 10e3 }) | .[]' \ | ${jq}/bin/jq --arg branch "$(basename "$_arg_file" .bin)" '. + {branch: $branch}' ''; @@ -281,11 +274,11 @@ let import pandas as pd pd.read_json(sys.stdin) \ - .set_index('param') \ - .drop(['branch', 'earliest', 'end', 'latest']) \ - .fillna("") \ + .rename(columns={'p50': 'P50 latency [μs]'}) \ + .set_index('P50 latency [μs]') \ + .drop(['branch']) \ .convert_dtypes() \ - .to_markdown(sys.stdout, floatfmt='.0f') + .to_markdown(sys.stdout, floatfmt='.1f') ''; @@ -305,8 +298,8 @@ let echo -e "## Loadtest results $marker\n" find loadtest -type f -iname '*.bin' -exec ${reporter} {} \; \ - | ${jq}/bin/jq '[paths(scalars) as $path | {param: $path | join("."), (.branch): getpath($path)}]' \ - | ${jq}/bin/jq --slurp 'flatten | group_by(.param) | map(add)' \ + | ${jq}/bin/jq '[paths(scalars) as $path | {p50: $path | join("."), (.branch): getpath($path)}]' \ + | ${jq}/bin/jq --slurp 'flatten | group_by(.p50) | map(add)' \ | ${toMarkdown} echo -e "\n\n## Loadtest elapsed seconds vs CPU/MEM usage $marker\n" diff --git a/test/load/errors.http b/test/load/errors.http deleted file mode 100644 index ea5db90569..0000000000 --- a/test/load/errors.http +++ /dev/null @@ -1,15 +0,0 @@ -# Misspelled relations -GET http://postgrest/actoxs?actor=eq.1 -Prefer: tx=commit - -# Misspelled relations on embeds -GET http://postgrest/actors?select=*,rolws(*,films(*)) -Prefer: tx=commit - -# Misspelled function names -GET http://postgrest/rpc/call_me_x?name=John -Prefer: tx=commit - -# Permission denied errors -GET http://postgrest/actors_1 -Prefer: tx=commit diff --git a/test/load/errors.sql b/test/load/errors.sql deleted file mode 100644 index 02abc37f3a..0000000000 --- a/test/load/errors.sql +++ /dev/null @@ -1,7 +0,0 @@ -\ir fixtures.sql - -SELECT format('CREATE TABLE test.actors_%s ();', n) -FROM generate_series(1, 450) n -- 500 is the upper limit for table not found error hint generation -\gexec - --- TODO add many function for fuzzy search (somehow this is making the loadtest start slow) diff --git a/test/load/fixtures.sql b/test/load/fixtures.sql index 75bd6a8bb2..8f844c4835 100644 --- a/test/load/fixtures.sql +++ b/test/load/fixtures.sql @@ -49,3 +49,9 @@ REVOKE ALL PRIVILEGES ON TABLE FROM postgrest_test_anonymous; GRANT ALL ON TABLE authors_only TO postgrest_test_author; + +SELECT format('CREATE TABLE test.actors_%s ();', n) +FROM generate_series(1, 450) n -- 500 is the upper limit for table not found error hint generation +\gexec + +-- TODO add many function for fuzzy search (somehow this is making the loadtest start slow) diff --git a/test/load/targets.http b/test/load/targets.http index ce65472a70..7553ade018 100644 --- a/test/load/targets.http +++ b/test/load/targets.http @@ -33,3 +33,19 @@ POST http://postgrest/rpc/call_me @rpc.json OPTIONS http://postgrest/actors + +# Misspelled relations +GET http://postgrest/actoxs?actor=eq.1 +Prefer: tx=commit + +# Misspelled relations on embeds +GET http://postgrest/actors?select=*,rolws(*,films(*)) +Prefer: tx=commit + +# Misspelled function names +GET http://postgrest/rpc/call_me_x?name=John +Prefer: tx=commit + +# Permission denied errors +GET http://postgrest/actors_1 +Prefer: tx=commit