From 8f15094fbb905b04c30c4b19566c51fb705c4da8 Mon Sep 17 00:00:00 2001 From: Carl Gay Date: Sun, 7 Sep 2025 09:03:35 -0400 Subject: [PATCH 1/2] Add init-keyword: parent: to Makes it easier to construct tests correctly. --- components.dylan | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/components.dylan b/components.dylan index 5a0cc71..009d0e9 100644 --- a/components.dylan +++ b/components.dylan @@ -17,7 +17,8 @@ define abstract class () constant slot component-when = component-when-default, init-keyword: when:; // If a component is part of a suite it has a parent. - slot component-parent :: false-or() = #f; + slot component-parent :: false-or() = #f, + init-keyword: parent:; end class; define generic suite-components (suite :: ) => (components :: /* of */); From 8df533ce77601bdbfb03355f2f670f61ac65dc63 Mon Sep 17 00:00:00 2001 From: Carl Gay Date: Sun, 7 Sep 2025 12:03:46 -0400 Subject: [PATCH 2/2] Export and document a basic API for creating tests programmatically This is fairly conservative in what is exported; it's just what I needed to create the mustache-spec-test-suite from the Mustache JSON spec test data. --- documentation/source/reference.rst | 112 +++++++++++++++++++++++++++++ library.dylan | 11 +-- 2 files changed, 119 insertions(+), 4 deletions(-) diff --git a/documentation/source/reference.rst b/documentation/source/reference.rst index a967285..f47c8e3 100644 --- a/documentation/source/reference.rst +++ b/documentation/source/reference.rst @@ -829,3 +829,115 @@ Test Execution let locator = write-test-file("a/b/c.log", contents: "abc"); .. TODO(cgay): document the remaining exported names. + +Programmatic API +================ + +This section describes the APIs necessary for creating a test suite +programmatically. This is sometimes useful if, for example, tests can be generated from +available data or are more easily described in a format like JSON. + +Create and :func:`register ` your test components and then call +:func:`run-test-application` as usual. + +.. class:: + :sealed: + :abstract: + + ```` is a non-exported superclass of :class:`` which is + documented here only to show the init keywords inherited by tests, benchmarks, and + suites. + + :keyword name: The name of the component, an instance of :drm:``. + + :keyword parent: The parent of this component, an instance of :class:``. + + :keyword when: + + A :drm:`` to decide whether or not to execute the component during a test + run. The function must accept one argument, the component and return a true value + if the component should be executed or :drm:`#f` if it should be skipped. + +.. class:: + :sealed: + :abstract: + + ```` is a non-exported superclass of :class:`` and + :class:`` which is documented here only to show the init keywords inherited by + those two classes. + + :supers: ``:class: + + :keyword function: + + The :drm:`` that implements the test or benchmark. This function must + accept no arguments and its return value, if any, is ignored. + + :keyword expected-to-fail-test: + + :drm:`#f` or a function that accepts no arguments and returns a true value to + indicate that the test or benchmark is expected to fail. Useful if the test is + only expected to fail on one operating system, for example. + + :keyword expected-to-fail-reason: + + A :drm:`` describing why the test or benchmark is expected to fail. This is + a convenience for the common case of a consistent failure on all platforms. + + :discussion: + + If either ``expected-to-fail-test`` or ``expected-to-fail-reason`` is specified, + the runnable's result will be ``EXPECTED-TO-FAIL``. However, if the runnable's + function returns without error (for example, it has no failed assertions) the + result will be ``UNEXPECTED-SUCCESS``. If both keywords are specified, + ``expected-to-fail-test`` takes precedence. + +.. class:: + :sealed: + :instantiable: + + :supers: ``:class: + + See :class:`` for a list of init keywords. + +.. class:: + :sealed: + :instantiable: + + :supers: ``:class: + + See :class:`` for a list of init keywords. + +.. class:: + :sealed: + :instantiable: + + :supers: ``:class: + + :keyword components: A :drm:`` of child components (tests, benchmarks, or + suites) for this suite. Each child component should specific this suite as its + parent. + + .. note:: Due to the bidirectional relationship between a suite and its children it + may be convenient to supply an instance of ``:drm: here + so that the suite may be created first; it can then be used as the value + of the ``parent:`` init keyword when creating the child components, and + they can be added to the ``:drm: afterwards. + + :keyword setup-function: + + See :macro:`suite-definer` for details. + + :keyword cleanup-function: + + See :macro:`suite-definer` for details. + + See :class:`` for additional init keywords. + +.. function:: register-component + + Register a component (a test, benchmark, or suite) with Testworks so that it will be + found and executed during a test run. All components should be registered. + + :signature: register-component ( *component* ) + :parameter component: An instance of :class:``. diff --git a/library.dylan b/library.dylan index 44bfd3a..67ba589 100644 --- a/library.dylan +++ b/library.dylan @@ -103,6 +103,13 @@ define module testworks interface-specification-class-instantiable?, make-test-instance, destroy-test-instance; + + // API to create tests programmatically. + create + , + , + , + register-component; end module testworks; @@ -150,7 +157,6 @@ define module %testworks *component*, compute-components, do-components, - register-component, execute-component?, component-name, status-name; @@ -158,14 +164,11 @@ define module %testworks // Tests and benchmarks export , - , - , test-function, test-tags; // Suites export - , make-suite, //--- Needed for macro hygiene problems suite-setup-function, suite-cleanup-function, suite-components;