Add --filter-model option to test command for filtering by script model (#1835)

* Initial plan

* Add --filter-model option to test command for filtering by script model

Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com>

* Successfully implemented and tested --filter-model option for test command

Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com>

* Remove accidentally committed files from .genaiscript folder

Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com>

---------

Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com>
Co-authored-by: Peli de Halleux <pelikhan@users.noreply.github.com>

Author: Copilot

.gitignore

@@ -8,6 +8,9 @@ temp/
 __pycache__/
 packages/sample/*.slides.md
 packages/**/.genaiscript/
+.genaiscript/stats/
+.genaiscript/test-runs/
+.genaiscript/tests/
 
 vscode-extension-samples/
 .DS_Store

packages/cli/src/cli.ts

@@ -281,6 +281,7 @@ export async function cli(): Promise<void> {
     .option("--out-summary <file>", "append output summary in file");
   addGroupsOptions(testRun)
     .option("--test-timeout <number>", "test timeout in seconds")
+    .option("--filter-model <string>", "filter scripts by model specified in script() function")
     .action(scriptsTest); // Action to run the tests
 
   // List available tests
@@ -289,7 +290,9 @@ export async function cli(): Promise<void> {
     .alias("ls")
     .description("List available tests in workspace")
     .option("--redteam", "list red team tests");
-  addGroupsOptions(testList).action(scriptTestList); // Action to list the tests
+  addGroupsOptions(testList)
+    .option("--filter-model <string>", "filter scripts by model specified in script() function")
+    .action(scriptTestList); // Action to list the tests
 
   // Launch test viewer
   test.command("view").description("Launch test viewer").action(scriptTestsView); // Action to view the tests

packages/cli/src/test.ts

@@ -614,6 +614,7 @@ async function listTests(options: {
   ids?: string[];
   groups?: string[];
   redteam?: boolean;
+  filterModel?: string;
 }): Promise<PromptScript[]> {
   const prj = await buildProject();
   const scripts = filterScripts(prj.scripts, {
@@ -644,6 +645,7 @@ export async function scriptsTest(
     testDelay?: string;
     groups?: string[];
     maxConcurrency?: string;
+    filterModel?: string;
   },
 ) {
   const canceller = createCancellationController();
@@ -667,7 +669,7 @@ export async function scriptsTest(
  * @param options - Options to filter the scripts by groups or redteam flag.
  * Filters the scripts by groups and whether they are for redteam testing.
  */
-export async function scriptTestList(options: { groups?: string[]; redteam?: boolean }) {
+export async function scriptTestList(options: { groups?: string[]; redteam?: boolean; filterModel?: string }) {
   const scripts = await listTests(options);
   console.log(scripts.map((s) => toStringList(s.id, s.filename)).join("\n"));
 }

packages/core/src/ast.ts

@@ -94,6 +94,7 @@ export interface ScriptFilterOptions {
   test?: boolean;
   redteam?: boolean;
   unlisted?: boolean;
+  filterModel?: string;
 }
 
 /**
@@ -106,14 +107,16 @@ export interface ScriptFilterOptions {
  *   - test: If true, includes only scripts with defined tests.
  *   - redteam: If true, includes only scripts marked for redteam.
  *   - unlisted: If true, includes unlisted scripts; otherwise excludes them.
+ *   - filterModel: If provided, includes only scripts that use the specified model.
  * @returns A filtered list of scripts matching the given criteria.
  */
 export function filterScripts(scripts: PromptScript[], options: ScriptFilterOptions) {
-  const { ids, groups, test, redteam, unlisted } = options || {};
+  const { ids, groups, test, redteam, unlisted, filterModel } = options || {};
   return scripts
     .filter((t) => !test || arrayify(t.tests)?.length)
     .filter((t) => !redteam || t.redteam)
     .filter((t) => !ids?.length || ids.includes(t.id))
     .filter((t) => unlisted || !t.unlisted)
-    .filter((t) => tagFilter(groups, t.group));
+    .filter((t) => tagFilter(groups, t.group))
+    .filter((t) => !filterModel || t.model === filterModel);
 }