Better temp directories

Adds a `.1`, `.2`, etc uniquifier to the end of the test run directory name so that if
two test runs are executed within the same second there won't be a clash.

The unique directory for the entire test run is now stored in the runner. Previously it
was possible to end up with one test's temp files in `_test/cgay-20250926-101601` and the
next test's temp files in `_test/cgay-20250926-101602`.

Fixes #149
Fixes #199

Author: cgay

command-line.dylan

@@ -292,11 +292,10 @@ define function run-or-list-tests
       report-function(result, *standard-output*);
     end;
     if (*output-captured?*)
-      // TODO: be more specific about where the files are after fixing various problems
-      // with test-temp-directory, i.e., store one _test subdir per runner.
-      test-output("NOTE: Some test output was captured. See files named %s "
-                    "in the _test directory.\n",
-                  $captured-output-filename);
+      test-output("NOTE: Some test output was captured. See %s\n",
+                  file-locator(runner-temp-directory(runner),
+                               "*",
+                               $captured-output-filename));
     end;
     if (result.result-status == $passed) 0 else 1 end
   end

documentation/source/reference.rst

@@ -801,7 +801,7 @@ Test Execution
    temporary files created by the test or benchmark. The directory is created
    the first time this function is called for each test or benchmark and is not
    deleted after the test run is complete in case it's useful for post-mortem
-   analysis.  The directory is named ``_test/<user>-<timestamp>/<test-name>``
+   analysis.  The directory is named ``_test/<user>-<timestamp>.1/<test-name>``
    and is rooted at ``$DYLAN``, if defined, or in the current directory
    otherwise.
 

documentation/source/usage.rst

@@ -289,6 +289,36 @@ creating a test and provide a method that returns :drm:`#f` on Windows:
 Tests that aren't run because of the `when:` option are marked as ``SKIPPED`` in the
 results.
 
+Standard Output / Error
+~~~~~~~~~~~~~~~~~~~~~~~
+
+Any output to :var:`*standard-output*` or :var:`*standard-error*` during a test is
+automatically captured and stored in a file named :file:`_captured-stdout.txt` in the
+:func:`test's temporary directory <test-temp-directory>` so that it doesn't clutter the
+console output but also can be examined upon test failure.
+
+Temporary Files
+~~~~~~~~~~~~~~~
+
+If you need to create temporary files for a test, you can of course use the system temp
+directory, but after multiple test runs it may not be easy to figure out which is the
+correct file to look at when debugging your test.  Instead, use
+:func:`test-temp-directory` and its companion function :func:`write-test-file` to create
+the files in a directory created specifically for the test.
+
+Example:
+
+.. code:: dylan
+
+   define test test-something ()
+     let file = file-locator(test-temp-directory(), "something.txt");
+     write-test-file(file, contents: "hi there");
+     ...
+   end test;
+
+:func:`file-locator` is exported from the ``locators`` module of the ``system`` library.
+
+
 Benchmarks
 ----------
 

run.dylan

@@ -85,6 +85,10 @@ define open class <test-runner> (<object>)
   // to tests. For example, test data directory pathname.
   constant slot runner-options :: <string-table> = make(<string-table>),
     init-keyword: options:;
+
+  // A place to put temp files created by tests or the runner itself, one subdirectory
+  // per test.
+  constant slot runner-temp-directory :: <locator> = default-runner-temp-directory();
 end class <test-runner>;
 
 

utils.dylan

@@ -120,27 +120,44 @@ define function next-indent () => (indent :: <string>)
   concatenate(*indent*, $indent-step)
 end function;
 
-// Return a temporary directory unique to the current test or benchmark. The
-// directory is created the first time this is called for a given test.
-// The directory is _test/<user>-<yyyymmdd-hhmmss>/<full-test-name>/, relative
-// to ${DYLAN}/, if defined, or relative to fs/working-directory() otherwise.
+define function default-runner-temp-directory () => (dir :: <directory-locator>)
+  let dylan = os/environment-variable("DYLAN");
+  let base = if (dylan)
+               as(<directory-locator>, dylan)
+             else
+               fs/working-directory()
+             end;
+  let dated-dir
+    = concatenate(os/login-name() | "unknown",
+                  "-",
+                  date/format("%Y%m%d-%H%M%S", date/now()));
+  // We could just include milliseconds (%F) in the date format below but currently
+  // <date> milliseconds are always zero, at least on Unix.
+  // https://github.com/dylan-lang/testworks/issues/199
+  iterate loop (i = 1)
+    let uniquifier = format-to-string("%s.%d", dated-dir, i);
+    let dir = subdirectory-locator(base, "_test", uniquifier);
+    if (block ()
+          fs/file-exists?(dir)
+        exception (ex :: fs/<file-system-error>)
+          #f
+        end)
+      loop(i + 1)
+    else
+      dir
+    end
+  end
+end function;
+
+// Return a temporary directory unique to the current test or benchmark.
 define function test-temp-directory () => (d :: false-or(<directory-locator>))
   if (instance?(*component*, <runnable>))
-    let dylan = os/environment-variable("DYLAN");
-    let base = if (dylan)
-                 as(<directory-locator>, dylan)
-               else
-                 fs/working-directory()
-               end;
-    let uniquifier
-      = format-to-string("%s-%s", os/login-name() | "unknown",
-                         date/format("%Y%m%d-%H%M%S", date/now()));
     let safe-name = map(method (c)
                           if (c == '\\' | c == '/') '_' else c end
                         end,
                         component-name(*component*));
     let test-directory
-      = subdirectory-locator(base, "_test", uniquifier, safe-name);
+      = subdirectory-locator(runner-temp-directory(*runner*), safe-name);
     fs/ensure-directories-exist(test-directory);
     test-directory
   end