Small reqs2x improvements (#324)

ThisCakeIsALie · Zbigor · Den1552 · web-flow · commit 465ebcb28235 · 2026-03-24T15:28:58.000+01:00
Improves parsing of JSON output for llm2check (solves headaches relating
to non-JSON output on stdout). Also adds and slightly reorganizes some
CLI settings for reqs2tests and their associated VSCode settings.
Additionally adds an a new advanced LLM provider

---------

Co-authored-by: Zbigor &lt;igor.martinovic@vector.com&gt;
Co-authored-by: Den1552 &lt;denis.moslavac@vector.com&gt;
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -3,6 +3,13 @@
 All notable changes to the "vectorcastTestExplorer" extension will be documented in this file.
 
 
+## [1.0.30] - 2026-03-24
+
+### Added
+- Added setting to allow equirements-driven test generation without passing of function definitions (for blackbox-style testing)
+- Added new advanced LLM provider for use with Azure APIM
+- Reorganized LLM providers (into common and advanced/uncommon)
+
 ## [1.0.29] - 2026-02-25
 
 ### Added
diff --git a/ci/get_most_recent_reqs2tests_distribution.py b/ci/get_most_recent_reqs2tests_distribution.py
@@ -51,9 +51,15 @@ def download_file(url, filename=None):
     with open(tmp) as f:
         data = json.load(f)
 
+    def parse_date(uri):
+        try:
+            return datetime.fromisoformat(uri.rsplit("-", 1)[0][1:])
+        except ValueError:
+            return None
+
     children_urls = sorted(
-        [c["uri"] for c in data["children"] if c["uri"].rsplit("-", 1)[0][1:]],
-        key=lambda x: datetime.fromisoformat(x.rsplit("-", 1)[0][1:]),
+        [c["uri"] for c in data["children"] if parse_date(c["uri"]) is not None],
+        key=lambda x: parse_date(x),
         reverse=True,
     )
 
diff --git a/docs/reqs2x/reqs2x_documentation.md b/docs/reqs2x/reqs2x_documentation.md
@@ -10,7 +10,7 @@ The **VectorCAST Reqs2x tools** provide LLM-powered capabilities for requirement
 - **Generate VectorCAST test cases from requirements** 
 
 
-All tools require a configured LLM provider to function. The tools support multiple LLM providers including Azure OpenAI, OpenAI, Anthropic, and LiteLLM.  
+All tools require a configured LLM provider to function. The tools support multiple LLM providers including Azure OpenAI, OpenAI, Anthropic, LiteLLM, Azure APIM, and OpenAI Access Token.  
 
 This manual demonstrates Reqs2x usage workflows from inside this VS-Code extension using the `TUTORIAL_C` demo environment. Before starting, ensure you have the necessary components ready.
 
diff --git a/package.json b/package.json
@@ -2,7 +2,7 @@
   "name": "vectorcasttestexplorer",
   "displayName": "VectorCAST Test Explorer",
   "description": "VectorCAST Test Explorer for VS Code",
-  "version": "1.0.29",
+  "version": "1.0.30",
   "license": "MIT",
   "repository": {
     "type": "git",
@@ -494,15 +494,21 @@
             "default": false,
             "description": "Do not provide additional test examples to the LLM during test generation"
           },
+          "vectorcastTestExplorer.reqs2x.functionDefinitions": {
+            "type": "boolean",
+            "order": 9,
+            "default": true,
+            "description": "Supply function definitions to the language model during test generation. Disable for blackbox-style testing."
+          },
           "vectorcastTestExplorer.reqs2x.enableUutStubbing": {
             "type": "boolean",
-            "order": 8,
+            "order": 10,
             "default": true,
             "description": "Enable UUT stubbing during requirements-based test generation"
           },
           "vectorcastTestExplorer.reqs2x.generationLanguage": {
             "type": "string",
-            "order": 9,
+            "order": 11,
             "default": "en",
             "enum": [
               "af", "sq", "am", "ar", "hy", "as", "az", "be", "bn", "bs", "bg", "my", "ch", "hr", "cs", "da", "nl", "en", "et", "tl", "fi", "fr", "ka", "de", "gu", "ha", "he", "hi", "hu", "is", "ig", "id", "ga", "it", "jp", "kn", "kk", "km", "ko", "ky", "lo", "lv", "lt", "mk", "ms", "ml", "mt", "mn", "me", "ne", "no", "or", "fa", "pl", "pt", "pa", "ro", "ru", "sr", "si", "sk", "sl", "es", "sw", "sv", "tg", "ta", "te", "th", "tr", "uk", "ur", "uz", "vi", "cy", "xh", "yo", "zu"
@@ -514,119 +520,185 @@
           },
           "vectorcastTestExplorer.reqs2x.modelCompatibilityMode": {
             "type": "boolean",
-            "order": 10,
+            "order": 12,
             "default": false,
             "description": "Enable model compatibility mode (runs Reqs2X tools without structured outputs which enables compatibility with more providers)."
           },
           "vectorcastTestExplorer.reqs2x.provider": {
             "type": "string",
-            "order": 11,
+            "order": 13,
             "default": "openai",
-            "enum": ["openai", "azure_openai", "anthropic", "litellm"],
-            "enumDescriptions": ["OpenAI", "Azure OpenAI", "Anthropic", "LiteLLM"],
+            "enum": ["openai", "azure_openai", "anthropic", "litellm", "azure_apim", "openai_at"],
+            "enumDescriptions": ["OpenAI", "Azure OpenAI", "Anthropic", "LiteLLM", "Azure APIM (Advanced)", "OpenAI Access Token (Advanced)"],
             "description": "Language model provider for automatic requirements and test generation using Reqs2X"
           },
           "vectorcastTestExplorer.reqs2x.openai.apiKey": {
             "type": "string",
-            "order": 12,
+            "order": 14,
             "default": "",
             "description": "OpenAI API key"
           },
           "vectorcastTestExplorer.reqs2x.openai.modelName": {
             "type": "string",
-            "order": 13,
+            "order": 15,
             "default": "",
             "description": "OpenAI model name"
           },
           "vectorcastTestExplorer.reqs2x.openai.reasoningModelName": {
             "type": "string",
-            "order": 14,
+            "order": 16,
             "default": "",
             "description": "Optional OpenAI reasoning model name used for specialized reasoning subtasks."
           },
           "vectorcastTestExplorer.reqs2x.openai.baseUrl": {
             "type": "string",
-            "order": 15,
+            "order": 17,
             "default": "",
             "description": "OpenAI base URL (optional, set this to use OpenAI-compatible providers such as Ollama, vLLM or Bedrock)"
           },
           "vectorcastTestExplorer.reqs2x.azure.baseUrl": {
             "type": "string",
-            "order": 16,
+            "order": 18,
             "default": "",
             "description": "Azure OpenAI endpoint/base URL, e.g., https://my-example-instance.openai.azure.com"
           },
           "vectorcastTestExplorer.reqs2x.azure.apiKey": {
             "type": "string",
-            "order": 17,
+            "order": 19,
             "default": "",
             "description": "Azure OpenAI API key"
           },
           "vectorcastTestExplorer.reqs2x.azure.deployment": {
             "type": "string",
-            "order": 18,
+            "order": 20,
             "default": "",
             "description": "Azure OpenAI deployment name, e.g., my-custom-gpt-4o-deployment"
           },
           "vectorcastTestExplorer.reqs2x.azure.modelName": {
             "type": "string",
-            "order": 19,
+            "order": 21,
             "default": "",
             "description": "Azure OpenAI model name, e.g., gpt-4o"
           },
           "vectorcastTestExplorer.reqs2x.azure.reasoningModelName": {
             "type": "string",
-            "order": 20,
+            "order": 22,
             "default": "",
             "description": "Optional Azure OpenAI reasoning model name used for specialized reasoning subtasks."
           },
           "vectorcastTestExplorer.reqs2x.azure.reasoningDeployment": {
             "type": "string",
-            "order": 21,
+            "order": 23,
             "default": "",
             "description": "Azure OpenAI reasoning deployment name (required if Azure reasoning model name is provided)."
           },
           "vectorcastTestExplorer.reqs2x.azure.apiVersion": {
             "type": "string",
-            "order": 22,
+            "order": 24,
             "default": "2024-12-01-preview",
             "description": "Azure OpenAI API version"
           },
           "vectorcastTestExplorer.reqs2x.anthropic.apiKey": {
             "type": "string",
-            "order": 23,
+            "order": 25,
             "default": "",
             "description": "Anthropic API key"
           },
           "vectorcastTestExplorer.reqs2x.anthropic.modelName": {
             "type": "string",
-            "order": 24,
+            "order": 26,
             "default": "",
             "description": "Anthropic model name"
           },
           "vectorcastTestExplorer.reqs2x.anthropic.reasoningModelName": {
             "type": "string",
-            "order": 25,
+            "order": 27,
             "default": "",
             "description": "Optional Anthropic reasoning model name used for specialized reasoning subtasks."
           },
           "vectorcastTestExplorer.reqs2x.litellm.modelName": {
             "type": "string",
-            "order": 26,
+            "order": 28,
             "default": "",
             "description": "LiteLLM model name (prefix with provider used, e.g., openai/gpt-4o or azure/o3-mini)"
           },
           "vectorcastTestExplorer.reqs2x.litellm.reasoningModelName": {
             "type": "string",
-            "order": 27,
+            "order": 29,
             "default": "",
             "description": "Optional LiteLLM reasoning model name used for specialized reasoning subtasks."
           },
           "vectorcastTestExplorer.reqs2x.litellm.providerEnvVars": {
             "type": "string",
-            "order": 28,
+            "order": 30,
             "default": "",
             "description": "Environment variables to set when running LiteLLM, e.g., OPENAI_API_KEY=xxxx. Multiple variables can be separated by commas, e.g., OPENAI_API_KEY=xxxx,ANOTHER_ENV_VAR=yyyy. For a list of variables to set for a specific LiteLLM-compatible provider, see https://docs.litellm.ai/docs/providers."
+          },
+          "vectorcastTestExplorer.reqs2x.azureApim.subscriptionKey": {
+            "type": "string",
+            "order": 31,
+            "default": "",
+            "description": "[Advanced] Azure APIM subscription key (sent as the Ocp-Apim-Subscription-Key header)"
+          },
+          "vectorcastTestExplorer.reqs2x.azureApim.baseUrl": {
+            "type": "string",
+            "order": 32,
+            "default": "",
+            "description": "[Advanced] Azure APIM gateway base URL, e.g., https://some-domain.azure-api.net/v1"
+          },
+          "vectorcastTestExplorer.reqs2x.azureApim.modelName": {
+            "type": "string",
+            "order": 33,
+            "default": "",
+            "description": "[Advanced] Model name for the APIM-proxied LLM, e.g., gpt-4.1"
+          },
+          "vectorcastTestExplorer.reqs2x.azureApim.apiKey": {
+            "type": "string",
+            "order": 34,
+            "default": "",
+            "description": "[Advanced] Optional downstream Azure OpenAI API key (forwarded when the APIM policy requires it)"
+          },
+          "vectorcastTestExplorer.reqs2x.azureApim.reasoningModelName": {
+            "type": "string",
+            "order": 35,
+            "default": "",
+            "description": "[Advanced] Optional reasoning model name used for specialized reasoning subtasks (Azure APIM)."
+          },
+          "vectorcastTestExplorer.reqs2x.openaiAt.modelName": {
+            "type": "string",
+            "order": 36,
+            "default": "",
+            "description": "[Advanced] OpenAI Access Token model name"
+          },
+          "vectorcastTestExplorer.reqs2x.openaiAt.modelUrl": {
+            "type": "string",
+            "order": 37,
+            "default": "",
+            "description": "[Advanced] OpenAI Access Token model URL"
+          },
+          "vectorcastTestExplorer.reqs2x.openaiAt.authUrl": {
+            "type": "string",
+            "order": 38,
+            "default": "",
+            "description": "[Advanced] OpenAI Access Token authentication URL"
+          },
+          "vectorcastTestExplorer.reqs2x.openaiAt.appKey": {
+            "type": "string",
+            "order": 39,
+            "default": "",
+            "description": "[Advanced] OpenAI Access Token application key"
+          },
+          "vectorcastTestExplorer.reqs2x.openaiAt.appSecret": {
+            "type": "string",
+            "order": 40,
+            "default": "",
+            "description": "[Advanced] OpenAI Access Token application secret"
+          },
+          "vectorcastTestExplorer.reqs2x.openaiAt.reasoningModelName": {
+            "type": "string",
+            "order": 41,
+            "default": "",
+            "description": "[Advanced] Optional reasoning model name used for specialized reasoning subtasks (OpenAI Access Token)."
           }
         }
       }
diff --git a/src-common/commonUtilities.ts b/src-common/commonUtilities.ts
@@ -89,6 +89,77 @@ export function getVcastOptionValues(enviroPath: string): cfgOptionType {
 export const vpythonSplitString = "ACTUAL-DATA";
 export const atgAndClicastSplitString =
   "If you want to use VECTORCAST_DIR, use this syntax:";
+/**
+ * Attempt to extract and parse a JSON object or array from a string that may
+ * contain non-JSON text before or after the actual JSON (e.g. log lines,
+ * warnings). Returns the parsed value on success, or undefined if no valid
+ * JSON is found.
+ *
+ * The `escape` flag below only handles backslash-escaped quotes (`\"`) so that
+ * they don't falsely toggle the in-string state. Other escape sequences such
+ * as multi-character unicode escapes (`\uXXXX`) are not fully parsed, which is
+ * fine — we only need to track quote boundaries correctly, and `JSON.parse`
+ * handles the real validation at the end.
+ */
+export function extractJson(raw: string): any | undefined {
+  const trimmed = raw.trim();
+
+  // Fast path: the whole string is valid JSON
+  try {
+    return JSON.parse(trimmed);
+  } catch {
+    // Fall through to extraction
+  }
+
+  // Find the first '{' or '[' — whichever comes first
+  const objStart = trimmed.indexOf("{");
+  const arrStart = trimmed.indexOf("[");
+  const start =
+    objStart === -1
+      ? arrStart
+      : arrStart === -1
+        ? objStart
+        : Math.min(objStart, arrStart);
+  if (start === -1) return undefined;
+
+  let depth = 0;
+  let inString = false;
+  let escape = false;
+
+  for (let i = start; i < trimmed.length; i++) {
+    const ch = trimmed[i];
+    if (escape) {
+      escape = false;
+      continue;
+    }
+
+    // Deal with escaped quotes so that we don't accidentally toggle inString when we see a \" sequence
+    if (ch === "\\" && inString) {
+      escape = true;
+      continue;
+    }
+
+    if (ch === '"') {
+      inString = !inString;
+      continue;
+    }
+    if (inString) continue;
+    if (ch === "{" || ch === "[") depth++;
+    else if (ch === "}" || ch === "]") {
+      depth--;
+      if (depth === 0) {
+        try {
+          return JSON.parse(trimmed.substring(start, i + 1));
+        } catch {
+          return undefined;
+        }
+      }
+    }
+  }
+
+  return undefined;
+}
+
 export function cleanVectorcastOutput(outputString: string): string {
   if (outputString.includes(vpythonSplitString)) {
     const pieces = outputString.split(vpythonSplitString, 2);
diff --git a/src/requirements/requirementsOperations.ts b/src/requirements/requirementsOperations.ts
@@ -378,6 +378,7 @@ export async function generateTestsFromRequirements(
 
   const noTestExamples = config.get<boolean>("noTestExamples", false);
   const reorder = config.get<boolean>("reorder", true);
+  const funcDefs = config.get<boolean>("functionDefinitions", true);
   const allowUUTStubs = config.get<boolean>("enableUutStubbing", true);
 
   const retries = config.get<number>("retries", 2);
@@ -404,8 +405,9 @@ export async function generateTestsFromRequirements(
     "--batched",
     ...(decomposeRequirements ? [] : ["--no-requirement-decomposition"]),
     ...(noTestExamples ? ["--no-test-examples"] : []),
-    ...(!reorder ? ["--no-reorder"] : []),
-    ...(allowUUTStubs ? ["--allow-uut-stubs"] : ["--no-allow-uut-stubs"]),
+    ...(reorder ? [] : ["--no-reorder"]),
+    ...(funcDefs ? [] : ["--no-func-defs"]),
+    ...(allowUUTStubs ? [] : ["--no-allow-uut-stubs"]),
     "--allow-partial",
     "--json-events",
     ...(enableRequirementKeys ? ["--requirement-keys"] : []),
diff --git a/src/requirements/requirementsUtils.ts b/src/requirements/requirementsUtils.ts
