|
| 1 | +# SPDX-License-Identifier: MPL-2.0 |
| 2 | +# Executor router coverage — every registered LLM-facing tool exercised. |
| 3 | +# |
| 4 | +# execute_tool(name, args) is the precise layer the LLM drives: a bare string |
| 5 | +# tool name plus a loosely-typed argument Dict gets coerced into typed Julia |
| 6 | +# calls. Before this file, that string→function mapping and argument |
| 7 | +# coercion was exercised for exactly one tool ("anova"). This file |
| 8 | +# programmatically enumerates every tool registered in tools/definitions.jl |
| 9 | +# (parsed via get_tools(), never hand-copied) and calls execute_tool for |
| 10 | +# each with a table of minimal valid arguments, asserting a clean |
| 11 | +# (non-"error") Dict comes back. A small set of high-traffic tools are |
| 12 | +# additionally cross-checked against their direct Julia function call — the |
| 13 | +# same pattern already used for "anova" in reference_validation_test.jl. |
| 14 | +# |
| 15 | +# If a tool is ever added to get_tools() without a corresponding entry here |
| 16 | +# (either in ARG_TABLE or SKIP_LIST), the "every registered tool" testset |
| 17 | +# below fails. That is the point: coverage cannot silently rot. |
| 18 | + |
| 19 | +@testset "Executor Router Coverage" begin |
| 20 | + |
| 21 | + # ── Minimal valid arguments per registered tool name ────────────────── |
| 22 | + # Keep each entry the smallest input that exercises the tool's primary |
| 23 | + # code path without tripping degenerate-input guards. |
| 24 | + ARG_TABLE = Dict{String,Dict{String,Any}}( |
| 25 | + "descriptive_statistics" => Dict{String,Any}( |
| 26 | + "data" => [2.0, 4.0, 4.0, 4.0, 5.0, 5.0, 7.0, 9.0]), |
| 27 | + "frequency_analysis" => Dict{String,Any}( |
| 28 | + "data" => ["a", "b", "a", "c", "b", "a"]), |
| 29 | + "t_test" => Dict{String,Any}( |
| 30 | + "type" => "independent", |
| 31 | + "group1" => [1.0, 2.0, 3.0, 4.0, 5.0], |
| 32 | + "group2" => [2.0, 4.0, 6.0, 8.0, 10.0]), |
| 33 | + "anova" => Dict{String,Any}( |
| 34 | + "groups" => [[1.0, 2.0, 3.0], [2.0, 3.0, 4.0], [3.0, 4.0, 5.0]]), |
| 35 | + "chi_square" => Dict{String,Any}( |
| 36 | + "type" => "independence", |
| 37 | + "observed" => [[10, 20], [30, 40]]), |
| 38 | + "correlation" => Dict{String,Any}( |
| 39 | + "x" => [1.0, 2.0, 3.0, 4.0, 5.0], |
| 40 | + "y" => [1.0, 2.0, 3.0, 4.0, 6.0]), |
| 41 | + "regression" => Dict{String,Any}( |
| 42 | + "x" => [1.0, 2.0, 3.0, 4.0, 5.0], |
| 43 | + "y" => [2.0, 4.0, 5.0, 4.0, 5.0]), |
| 44 | + "nonparametric_test" => Dict{String,Any}( |
| 45 | + "type" => "mann_whitney", |
| 46 | + "group1" => [1.0, 2.0, 3.0], |
| 47 | + "group2" => [4.0, 5.0, 6.0]), |
| 48 | + "permanova" => Dict{String,Any}( |
| 49 | + "distance_matrix" => [[0.0, 1.0, 2.0, 3.0], |
| 50 | + [1.0, 0.0, 1.5, 2.5], |
| 51 | + [2.0, 1.5, 0.0, 1.0], |
| 52 | + [3.0, 2.5, 1.0, 0.0]], |
| 53 | + "group_labels" => ["A", "A", "B", "B"], |
| 54 | + "n_permutations" => 49), |
| 55 | + "effect_size_calculator" => Dict{String,Any}( |
| 56 | + "cohens_d" => 0.5, "n1" => 20, "n2" => 20), |
| 57 | + "power_analysis" => Dict{String,Any}( |
| 58 | + "effect_size" => 0.5, "n" => 30), |
| 59 | + "sample_size_calculator" => Dict{String,Any}( |
| 60 | + "design" => "means", "effect_size" => 0.5), |
| 61 | + "test_assumptions" => Dict{String,Any}( |
| 62 | + "type" => "normality", |
| 63 | + "data" => collect(1.0:15.0)), |
| 64 | + "bayesian_analysis" => Dict{String,Any}( |
| 65 | + "prior" => [0.5, 0.5], |
| 66 | + "likelihood" => [[0.8, 0.2], [0.3, 0.7]], |
| 67 | + "data_index" => 1), |
| 68 | + "bayes_factor" => Dict{String,Any}( |
| 69 | + "r_squared_full" => 0.6, "r_squared_reduced" => 0.4, |
| 70 | + "n" => 50, "p_full" => 3, "p_reduced" => 1), |
| 71 | + "credible_intervals" => Dict{String,Any}( |
| 72 | + "samples" => collect(1.0:100.0)), |
| 73 | + "fuzzy_logic_analysis" => Dict{String,Any}( |
| 74 | + "value" => 0.5, "center" => 0.5, "width" => 0.2, "operation" => "membership"), |
| 75 | + "dempster_shafer" => Dict{String,Any}( |
| 76 | + "evidence1" => Dict("A" => 0.6, "B" => 0.4), |
| 77 | + "evidence2" => Dict("A" => 0.5, "B" => 0.5)), |
| 78 | + "granger_causality" => Dict{String,Any}( |
| 79 | + "series_x" => [1.0i + 0.1 * cos(i) for i in 1:20], |
| 80 | + "series_y" => [2.0i + 0.1 * sin(i) for i in 1:20], |
| 81 | + "lag" => 1), |
| 82 | + "james_stein" => Dict{String,Any}( |
| 83 | + "observations" => [1.0, 2.5, 3.0, 4.5, 5.0, 2.0]), |
| 84 | + "diagnostic_metrics" => Dict{String,Any}( |
| 85 | + "true_positive" => 50, "false_negative" => 10, |
| 86 | + "true_negative" => 80, "false_positive" => 5), |
| 87 | + "reliability_analysis" => Dict{String,Any}( |
| 88 | + "items" => [[3.0, 4.0, 5.0], [4.0, 4.0, 4.0], [2.0, 3.0, 3.0], |
| 89 | + [5.0, 5.0, 4.0], [3.0, 4.0, 4.0], [4.0, 3.0, 5.0]]), |
| 90 | + "measurement_analysis" => Dict{String,Any}( |
| 91 | + "type" => "sem", "reliability" => 0.8, "sd" => 10.0), |
| 92 | + "validity_assessment" => Dict{String,Any}( |
| 93 | + "n_essential" => 8, "n_total" => 10), |
| 94 | + "criterion_validity_test" => Dict{String,Any}( |
| 95 | + "predictor" => [1.0, 2.0, 3.0, 4.0, 5.0], |
| 96 | + "criterion" => [2.0, 4.0, 5.0, 4.0, 6.0]), |
| 97 | + "inter_rater_reliability" => Dict{String,Any}( |
| 98 | + "type" => "cohens_kappa", |
| 99 | + "rater1" => [1, 0, 1, 1, 0, 1], |
| 100 | + "rater2" => [1, 0, 0, 1, 0, 1]), |
| 101 | + "qualitative_analysis" => Dict{String,Any}( |
| 102 | + "themes_per_interview" => [5, 3, 2, 1, 1, 0, 1]), |
| 103 | + "calculate_pre" => Dict{String,Any}( |
| 104 | + "observed" => [1.0, 2.0, 3.0, 4.0, 5.0], |
| 105 | + "predicted" => [1.1, 2.1, 2.9, 4.2, 4.8]), |
| 106 | + "sampling_design" => Dict{String,Any}( |
| 107 | + "type" => "margin_of_error", "n" => 100, |
| 108 | + "proportion" => 0.5, "confidence" => 0.95), |
| 109 | + ) |
| 110 | + |
| 111 | + # ── Skip-list: tools genuinely needing external services/files ──────── |
| 112 | + # Keep this EMPTY unless a tool truly cannot be exercised in-process. |
| 113 | + # A broken/undefined dispatch target is a bug to fix, not a skip |
| 114 | + # reason — the whole point of this file is to catch exactly that. |
| 115 | + SKIP_LIST = Dict{String,String}() |
| 116 | + |
| 117 | + registered_tools = [t["function"]["name"] for t in Statistikles.get_tools()] |
| 118 | + |
| 119 | + @testset "Registry coverage: every tool has a table entry or skip reason" begin |
| 120 | + @test !isempty(registered_tools) |
| 121 | + for name in registered_tools |
| 122 | + @test haskey(ARG_TABLE, name) || haskey(SKIP_LIST, name) |
| 123 | + end |
| 124 | + # The skip-list is an escape hatch, not a dumping ground. |
| 125 | + @test length(SKIP_LIST) <= 3 |
| 126 | + for (name, reason) in SKIP_LIST |
| 127 | + @test !isempty(reason) |
| 128 | + end |
| 129 | + end |
| 130 | + |
| 131 | + @testset "execute_tool succeeds for every non-skipped registered tool" begin |
| 132 | + for name in registered_tools |
| 133 | + haskey(SKIP_LIST, name) && continue |
| 134 | + @test haskey(ARG_TABLE, name) |
| 135 | + !haskey(ARG_TABLE, name) && continue # can't call without args |
| 136 | + result = Statistikles.execute_tool(name, ARG_TABLE[name]) |
| 137 | + @test result isa Dict |
| 138 | + @test !haskey(result, "error") |
| 139 | + end |
| 140 | + end |
| 141 | + |
| 142 | + # ── Cross-checks: router output vs. direct Julia function call ──────── |
| 143 | + # For high-traffic tools, confirm execute_tool's argument coercion |
| 144 | + # doesn't silently change the numbers — the same pattern as the |
| 145 | + # "anova" cross-check in reference_validation_test.jl. |
| 146 | + @testset "Cross-check: t_test router vs. direct call" begin |
| 147 | + g1, g2 = [1.0, 2.0, 3.0, 4.0, 5.0], [2.0, 4.0, 6.0, 8.0, 10.0] |
| 148 | + direct = t_test_independent(g1, g2) |
| 149 | + via_tool = Statistikles.execute_tool("t_test", |
| 150 | + Dict{String,Any}("type" => "independent", "group1" => g1, "group2" => g2)) |
| 151 | + @test isapprox(via_tool["t_stat"], direct["t_stat"]; atol = 1e-12) |
| 152 | + @test isapprox(via_tool["p_value"], direct["p_value"]; atol = 1e-12) |
| 153 | + @test isapprox(via_tool["cohens_d"], direct["cohens_d"]; atol = 1e-12) |
| 154 | + end |
| 155 | + |
| 156 | + @testset "Cross-check: descriptive_statistics router vs. direct call" begin |
| 157 | + data = [2.0, 4.0, 4.0, 4.0, 5.0, 5.0, 7.0, 9.0] |
| 158 | + direct = descriptive_stats(data) |
| 159 | + via_tool = Statistikles.execute_tool("descriptive_statistics", |
| 160 | + Dict{String,Any}("data" => data)) |
| 161 | + @test isapprox(via_tool["mean"], direct["mean"]; atol = 1e-12) |
| 162 | + @test isapprox(via_tool["variance"], direct["variance"]; atol = 1e-12) |
| 163 | + @test isapprox(via_tool["skewness"], direct["skewness"]; atol = 1e-12) |
| 164 | + end |
| 165 | + |
| 166 | + @testset "Cross-check: correlation router vs. direct call" begin |
| 167 | + x, y = [1.0, 2.0, 3.0, 4.0, 5.0], [1.0, 2.0, 3.0, 4.0, 6.0] |
| 168 | + direct = pearson_correlation(x, y) |
| 169 | + via_tool = Statistikles.execute_tool("correlation", |
| 170 | + Dict{String,Any}("x" => x, "y" => y, "method" => "pearson")) |
| 171 | + @test isapprox(via_tool["r"], direct["r"]; atol = 1e-12) |
| 172 | + @test isapprox(via_tool["p_value"], direct["p_value"]; atol = 1e-12) |
| 173 | + end |
| 174 | + |
| 175 | + @testset "Cross-check: regression router vs. direct call" begin |
| 176 | + x, y = [1.0, 2.0, 3.0, 4.0, 5.0], [2.0, 4.0, 5.0, 4.0, 5.0] |
| 177 | + direct = simple_linear_regression(x, y) |
| 178 | + via_tool = Statistikles.execute_tool("regression", |
| 179 | + Dict{String,Any}("x" => x, "y" => y)) |
| 180 | + @test isapprox(via_tool["slope"], direct["slope"]; atol = 1e-12) |
| 181 | + @test isapprox(via_tool["intercept"], direct["intercept"]; atol = 1e-12) |
| 182 | + @test isapprox(via_tool["r_squared"], direct["r_squared"]; atol = 1e-12) |
| 183 | + end |
| 184 | + |
| 185 | + @testset "Cross-check: mann_whitney router vs. direct call" begin |
| 186 | + g1, g2 = [1.0, 2.0, 3.0], [4.0, 5.0, 6.0] |
| 187 | + direct = mann_whitney_u(g1, g2) |
| 188 | + via_tool = Statistikles.execute_tool("nonparametric_test", |
| 189 | + Dict{String,Any}("type" => "mann_whitney", "group1" => g1, "group2" => g2)) |
| 190 | + @test isapprox(via_tool["U_statistic"], direct["U_statistic"]; atol = 1e-12) |
| 191 | + @test isapprox(via_tool["p_value"], direct["p_value"]; atol = 1e-12) |
| 192 | + end |
| 193 | + |
| 194 | +end |
0 commit comments