From 50c7a69536c4d4cabbbbf9ab10def33a7c96b2ac Mon Sep 17 00:00:00 2001 From: "Jonathan D.A. Jewell" <6759885+hyperpolymath@users.noreply.github.com> Date: Sun, 26 Jul 2026 17:56:28 +0100 Subject: [PATCH 1/2] \ --- .gitlab-ci.yml | 15 ++++++++++----- READINESS.md | 9 ++++----- beam/benches/nif_bench.exs | 9 +++++++++ beam/mix.exs | 4 ++-- beam/mix.lock | 6 ++++++ beam/test/aspect_test.exs | 26 ++++++++++++++++++++++++++ beam/test/property_test.exs | 23 +++++++++++++++++++++++ 7 files changed, 80 insertions(+), 12 deletions(-) create mode 100644 beam/benches/nif_bench.exs create mode 100644 beam/mix.lock create mode 100644 beam/test/aspect_test.exs create mode 100644 beam/test/property_test.exs diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7309fa9..62cfd88 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -67,13 +67,14 @@ mix-audit: stage: security image: elixir:latest script: + - cd beam - mix local.hex --force - mix archive.install hex mix_audit --force - mix deps.get - mix deps.audit rules: - exists: - - mix.exs + - beam/mix.exs allow_failure: true # ================== @@ -105,21 +106,23 @@ mix-format: stage: lint image: elixir:latest script: + - cd beam - mix format --check-formatted rules: - exists: - - mix.exs + - beam/mix.exs credo: stage: lint image: elixir:latest script: + - cd beam - mix local.hex --force - mix deps.get - mix credo --strict rules: - exists: - - mix.exs + - beam/mix.exs allow_failure: true # ================== @@ -139,12 +142,13 @@ mix-test: stage: test image: elixir:latest script: + - cd beam - mix local.hex --force - mix deps.get - mix test rules: - exists: - - mix.exs + - beam/mix.exs # ================== # Build @@ -167,9 +171,10 @@ mix-build: stage: build image: elixir:latest script: + - cd beam - mix local.hex --force - mix deps.get - MIX_ENV=prod mix compile rules: - exists: - - mix.exs + - beam/mix.exs diff --git a/READINESS.md b/READINESS.md index b18a7df..6a3a72f 100644 --- a/READINESS.md +++ b/READINESS.md @@ -30,9 +30,9 @@ Zig FFI + zig API + Elixir BEAM NIFs. ## Gaps preventing higher grades ### Blocks C (works reliably + annotated) -- No EXPLAINME.adoc, TEST-NEEDS.md, PROOF-NEEDS.md at repo root. +- ~~No EXPLAINME.adoc, TEST-NEEDS.md, PROOF-NEEDS.md at repo root.~~ *(Added)* - Julia server code has no docstrings. -- Elixir BEAM layer has no dedicated test coverage documented here. +- ~~Elixir BEAM layer has no dedicated test coverage documented here.~~ *(Added Unit, P2P, Aspect, and Benchmark coverage)* - No integration tests spanning the full Idris2 → Zig → V → Julia → Elixir stack. - No dogfooding evidence — has anyone actually driven this end-to-end? - Only 4 commits in history before absorption into nextgen-databases monorepo. @@ -42,9 +42,8 @@ Zig FFI + zig API + Elixir BEAM NIFs. ## What to do for C -1. Add EXPLAINME.adoc explaining the polyglot architecture and its intended - users. -2. Add TEST-NEEDS.md documenting what's tested at each language layer and what isn't. +1. ~~Add EXPLAINME.adoc explaining the polyglot architecture and its intended users.~~ *(Done)* +2. ~~Add TEST-NEEDS.md documenting what's tested at each language layer and what isn't.~~ *(Done)* 3. Write docstrings for `server/serve.jl`, `server/quandle_semantic.jl`. 4. Add per-language READMEs at `src/abi/`, `src/ffi/`, `src/api/`, `beam/` explaining what each layer contributes. diff --git a/beam/benches/nif_bench.exs b/beam/benches/nif_bench.exs new file mode 100644 index 0000000..bb38fba --- /dev/null +++ b/beam/benches/nif_bench.exs @@ -0,0 +1,9 @@ +# benches/nif_bench.exs +Benchee.run( + %{ + "semantic_lookup (stub/nif)" => fn -> QuandleDBNif.semantic_lookup("3_1") end, + "semantic_equivalents (stub/nif)" => fn -> QuandleDBNif.semantic_equivalents("3_1") end + }, + time: 2, + memory_time: 1 +) diff --git a/beam/mix.exs b/beam/mix.exs index 90147e2..c64ba7a 100644 --- a/beam/mix.exs +++ b/beam/mix.exs @@ -22,8 +22,8 @@ defmodule QuandleDBNif.MixProject do # Run "mix help deps" to learn about dependencies. defp deps do [ - # {:dep_from_hexpm, "~> 0.3.0"}, - # {:dep_from_git, git: "https://github.com/elixir-lang/my_dep.git", tag: "0.1.0"} + {:stream_data, "~> 1.1", only: :test}, + {:benchee, "~> 1.3", only: [:dev, :test]} ] end diff --git a/beam/mix.lock b/beam/mix.lock new file mode 100644 index 0000000..7efdde0 --- /dev/null +++ b/beam/mix.lock @@ -0,0 +1,6 @@ +%{ + "benchee": {:hex, :benchee, "1.5.1", "b95cbc36c4b98969a5c592a246e171041eb683c56bad1cb4f49a3b081ba66087", [:mix], [{:deep_merge, "~> 1.0", [hex: :deep_merge, repo: "hexpm", optional: false]}, {:statistex, "~> 1.1", [hex: :statistex, repo: "hexpm", optional: false]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "a539301f8dfd4efc5c5123bfb9d47ebde20092a863a5b5b16c2a60d2243dfce7"}, + "deep_merge": {:hex, :deep_merge, "1.0.2", "476aa7ea61c54de96220051b998d893869069094da65b96101aebf79416f8a1e", [:mix], [], "hexpm", "737a53cdc9758fedbb608bdc213969e65729466c4ef3cd8e8726d0335dff116c"}, + "statistex": {:hex, :statistex, "1.1.1", "73612aa7f79e53c30569be065fd121e380f1cf57bc4c2da5b41be9246da18df9", [:mix], [], "hexpm", "310c4b49b34adf683de3103639006bed233ab54c08a4add65a531448e653857c"}, + "stream_data": {:hex, :stream_data, "1.4.0", "026f929db613aabea6208012ae9b8970d3fd5f88b3bdf26831bc536f98c42036", [:mix], [], "hexpm", "2b0ee3a340dcce1c8cf6302a763ee757d1e01c54d6e16d9069062509d68b1dc9"}, +} diff --git a/beam/test/aspect_test.exs b/beam/test/aspect_test.exs new file mode 100644 index 0000000..a2b6e8b --- /dev/null +++ b/beam/test/aspect_test.exs @@ -0,0 +1,26 @@ +defmodule QuandleDBNifAspectTest do + use ExUnit.Case + + @moduledoc """ + Aspect-based tests for the QuandleDB BEAM NIF boundary. + Asserts resilience against adversarial or extreme inputs. + """ + + test "resists extreme length inputs" do + huge_str = String.duplicate("A", 1_000_000) + result = QuandleDBNif.semantic_lookup(huge_str) + assert match?({:ok, _}, result) or match?({:error, _}, result) + end + + test "resists null bytes and control characters" do + nasty_str = "abc\0def\n\r\t\x1b" + result = QuandleDBNif.semantic_equivalents(nasty_str) + assert match?({:ok, _}, result) or match?({:error, _}, result) + end + + test "resists invalid unicode" do + invalid_str = <<0xFFFF::utf16>> + result = QuandleDBNif.semantic_lookup(invalid_str) + assert match?({:ok, _}, result) or match?({:error, _}, result) + end +end diff --git a/beam/test/property_test.exs b/beam/test/property_test.exs new file mode 100644 index 0000000..a6a8469 --- /dev/null +++ b/beam/test/property_test.exs @@ -0,0 +1,23 @@ +defmodule QuandleDBNifPropertyTest do + use ExUnit.Case + use ExUnitProperties + + @moduledoc """ + Property-based (P2P) tests for the QuandleDB BEAM NIF boundary. + Asserts that arbitrary string inputs are handled safely without crashing the VM. + """ + + property "semantic_lookup handles arbitrary strings safely" do + check all(str <- string(:printable)) do + result = QuandleDBNif.semantic_lookup(str) + assert match?({:ok, _}, result) or match?({:error, _}, result) + end + end + + property "semantic_equivalents handles arbitrary strings safely" do + check all(str <- string(:printable)) do + result = QuandleDBNif.semantic_equivalents(str) + assert match?({:ok, _}, result) or match?({:error, _}, result) + end + end +end From 27c55193ca5939701aef3889228d4260f079665a Mon Sep 17 00:00:00 2001 From: "Jonathan D.A. Jewell" <6759885+hyperpolymath@users.noreply.github.com> Date: Sun, 26 Jul 2026 18:39:47 +0100 Subject: [PATCH 2/2] feat(server): integrate explain capabilities for main and semantic index --- server/serve.jl | 58 ++++++++++++++++++++++++++++++++----------------- 1 file changed, 38 insertions(+), 20 deletions(-) diff --git a/server/serve.jl b/server/serve.jl index 52331dd..73f7ffa 100644 --- a/server/serve.jl +++ b/server/serve.jl @@ -23,6 +23,8 @@ using SHA include("quandle_semantic.jl") using .QuandleSemantic +include("query_explain.jl") + include("Diagnostics.jl") include("krl/KRL.jl") using .KRL: parse_any, parse_krl, parse_sql, KRLParseError, KRLLexError, @@ -63,23 +65,6 @@ CREATE TABLE IF NOT EXISTS semantic_schema_info ( ); """ -const SEMANTIC_INDEX_STATEMENTS = [ - "CREATE INDEX IF NOT EXISTS idx_semantic_hash ON quandle_semantic_index(descriptor_hash)", - "CREATE INDEX IF NOT EXISTS idx_semantic_key ON quandle_semantic_index(quandle_key)", - "CREATE INDEX IF NOT EXISTS idx_semantic_crossing ON quandle_semantic_index(crossing_number)", - "CREATE INDEX IF NOT EXISTS idx_semantic_determinant ON quandle_semantic_index(determinant)", - "CREATE INDEX IF NOT EXISTS idx_semantic_signature ON quandle_semantic_index(signature)", - "CREATE INDEX IF NOT EXISTS idx_semantic_col3 ON quandle_semantic_index(colouring_count_3)", - "CREATE INDEX IF NOT EXISTS idx_semantic_col5 ON quandle_semantic_index(colouring_count_5)", - # DB-3 Phase A additions (see docs/db-3-index-strategy.md): - # writhe / genus / quandle_generator_count are accepted as filter - # parameters in GET /api/knots and GET /api/semantic but were not - # indexed; queries forced O(n) scans. B-tree per the audit. - "CREATE INDEX IF NOT EXISTS idx_semantic_writhe ON quandle_semantic_index(writhe)", - "CREATE INDEX IF NOT EXISTS idx_semantic_genus ON quandle_semantic_index(genus)", - "CREATE INDEX IF NOT EXISTS idx_semantic_gencount ON quandle_semantic_index(quandle_generator_count)", -] - const REQUIRED_SEMANTIC_COLUMNS = [ ("descriptor_version", "TEXT"), ("descriptor_hash", "TEXT"), @@ -116,8 +101,12 @@ mutable struct SemanticIndexDB end _ensure_semantic_columns!(conn) - for stmt in SEMANTIC_INDEX_STATEMENTS - DBInterface.execute(conn, stmt) + + # Create standard indices via Skein.jl (DB-3 Phase B) + for col in ["descriptor_hash", "quandle_key", "crossing_number", "determinant", + "signature", "colouring_count_3", "colouring_count_5", + "writhe", "genus", "quandle_generator_count"] + Skein.create_index!(conn, "quandle_semantic_index", col) end DBInterface.execute(conn, @@ -756,7 +745,7 @@ function handle_semantic_equivalents(db::SkeinDB, sdb::SemanticIndexDB, name::St )) end -function handle_semantic_index(sdb::SemanticIndexDB, params::Dict{String, String}) +function _build_semantic_query_sql(params::Dict{String, String}) crossing_number = parse_int_param(params, "crossing_number") determinant_val = parse_int_param(params, "determinant") signature_val = parse_int_param(params, "signature") @@ -809,10 +798,37 @@ function handle_semantic_index(sdb::SemanticIndexDB, params::Dict{String, String push!(args, limit) push!(args, offset) + return sql, args, limit, offset +end + +function handle_semantic_index(sdb::SemanticIndexDB, params::Dict{String, String}) + sql, args, limit, offset = _build_semantic_query_sql(params) rows = [semantic_to_dict(row) for row in DBInterface.execute(sdb.conn, sql, args)] json_response(Dict("semantic_index" => rows, "count" => length(rows), "limit" => limit, "offset" => offset)) end +function handle_explain(db::SkeinDB, sdb::SemanticIndexDB, params::Dict{String, String}) + endpoint = get(params, "endpoint", nothing) + if endpoint == "semantic" + sql, args, limit, offset = _build_semantic_query_sql(params) + plan = explain_query_plan(sdb.conn, sql, args) + return json_response(Dict("query_plan" => plan, "plan_summary" => plan_summary(plan))) + end + + raw_sql = get(params, "sql", nothing) + if !isnothing(raw_sql) + if !is_select_only(raw_sql) + return error_response("Mutation prohibited. EXPLAIN is restricted to SELECT/WITH queries."; status=403) + end + # Phase B: explain raw SQL using the DB connection (could be SkeinDB or SemanticIndexDB) + # Assuming the raw SQL queries the main Skein database + plan = explain_query_plan(db.conn, raw_sql, Any[]) + return json_response(Dict("query_plan" => plan, "plan_summary" => plan_summary(plan))) + end + + error_response("Missing 'endpoint=semantic' or 'sql=...' parameter"; status=400) +end + function handle_krl_query(data::DataProvider, sem::SemProvider, metrics::QueryMetrics, req::HTTP.Request) t0 = time() @@ -998,6 +1014,8 @@ function router(db::SkeinDB, sdb::SemanticIndexDB, static_dir::String, !isnothing(m_sem) && return handle_semantic_detail(db, sdb, m_sem.captures[1]) + path == "/api/explain" && return handle_explain(db, sdb, params) + m_knot = match(r"^/api/knots/(.+)$", path) !isnothing(m_knot) && return handle_knot_detail(db, sdb, m_knot.captures[1])