feat: new session per request (#68)

Author: xavidop

docs/docs/static/conversationsuite.json

@@ -26,6 +26,9 @@
 					"environmentName": {
 						"type": "string"
 					},
+					"newSessionPerTest": {
+						"type": "boolean"
+					},
 					"tests": {
 						"items": {
 							"$ref": "#/$defs/TestFile"

docs/docs/static/swagger.json

@@ -298,6 +298,10 @@
                     "type": "string",
                     "example": "Example Suite"
                 },
+                "new_session_per_test": {
+                    "type": "boolean",
+                    "example": false
+                },
                 "tests": {
                     "type": "array",
                     "items": {

docs/docs/static/swagger.yaml

@@ -98,6 +98,9 @@ definitions:
       name:
         example: Example Suite
         type: string
+      new_session_per_test:
+        example: false
+        type: boolean
       tests:
         items:
           $ref: '#/definitions/TestRequest'

docs/docs/test-platform/test-suites.md

@@ -12,6 +12,7 @@ A Test Suite is a JSON-formatted definition that contains:
 - **Test Configuration**: API keys, environment settings, and Voiceflow project details
 - **Test Cases**: Individual test scenarios with user inputs and expected agent responses
 - **Validation Rules**: Criteria for determining test success or failure
+- **Session Management**: Control whether tests share the same user session or use isolated sessions
 
 ## Creating Test Suites
 
@@ -76,6 +77,37 @@ Individual test case files referenced in the suite. You can find more details on
 
 For more details on the JSON structure, refer to the [Test Suite JSON Schema](https://docs.voiceflow.com/reference/post_api-v1-tests-execute#/).
 
+#### Session Management
+
+Test suites support session management through the `new_session_per_test` field:
+
+```json
+{
+  "name": "Example Suite",
+  "description": "Test suite with isolated sessions",
+  "environment_name": "production",
+  "new_session_per_test": true,
+  "tests": [...]
+}
+```
+
+**Session Behavior:**
+
+- **`false` (default)**: All tests share the same user session
+  - Variables persist across tests
+  - Conversation context carries over between tests
+  - Tests execute sequentially with shared state
+
+- **`true`**: Each test gets a fresh user session
+  - New user ID generated for each test
+  - Complete isolation between tests
+  - No shared context or variables
+
+**When to use isolated sessions:**
+- Testing independent scenarios that shouldn't affect each other
+- Validating conversation flows from a fresh start
+- Ensuring tests don't have hidden dependencies on execution order
+
 ## API Integration
 
 Test suites integrate with the Voiceflow API for execution:

docs/docs/tests/suites.md

@@ -13,6 +13,9 @@ name: Example Suite
 description: Suite used as an example
 # Environment name of your Voiceflow agent. It could be development, or production.
 environmentName: development
+# Optional: Create a new user session for each test (default: false)
+# When enabled, each test will run with a fresh user session instead of sharing one session across all tests
+newSessionPerTest: false
 # You can have multiple tests defined in separated files
 tests:
   # ID of the test.
@@ -23,6 +26,20 @@ tests:
 
 It has the same structure as the NLU Profiler suite.
 
+### Session Management
+
+By default, all tests within a suite share the same user session (user ID). This means that:
+
+- Variables set in one test persist to the next test
+- The conversation context carries over between tests
+- Tests are executed sequentially with the same user state
+
+If you want each test to start with a fresh session, set `newSessionPerTest: true`. This will:
+
+- Generate a new user ID for each test
+- Clear all conversation context between tests
+- Ensure tests are completely isolated from each other
+
 ## JSON Schema
 
 `voiceflow-cli` also has a [jsonschema](http://json-schema.org/draft/2020-12/json-schema-validation.html) file, which you can use to have better

examples/suite.yaml

@@ -1,6 +1,9 @@
 name: Example Conversation Profiler Suite
 description: Suite used as an example
 environmentName: production
+# Optional: Create a new user session for each test (default: false)
+# When enabled, each test will run with a fresh user session
+newSessionPerTest: true
 tests:
   - id: traditional_test
     file: ./tests/test_1.yaml

internal/types/tests/suiteTypes.go

@@ -1,11 +1,12 @@
 package tests
 
 type Suite struct {
-	Name            string        `yaml:"name" json:"name"`
-	Description     string        `yaml:"description" json:"description"`
-	EnvironmentName string        `yaml:"environmentName" json:"environmentName"`
-	Tests           []TestFile    `yaml:"tests" json:"tests"`
-	OpenAIConfig    *OpenAIConfig `yaml:"openAIConfig,omitempty" json:"openAIConfig,omitempty"`
+	Name              string        `yaml:"name" json:"name"`
+	Description       string        `yaml:"description" json:"description"`
+	EnvironmentName   string        `yaml:"environmentName" json:"environmentName"`
+	NewSessionPerTest bool          `yaml:"newSessionPerTest,omitempty" json:"newSessionPerTest,omitempty"`
+	Tests             []TestFile    `yaml:"tests" json:"tests"`
+	OpenAIConfig      *OpenAIConfig `yaml:"openAIConfig,omitempty" json:"openAIConfig,omitempty"`
 }
 
 type TestFile struct {

pkg/test/execute.go

@@ -14,6 +14,7 @@ type HTTPSuiteRequest struct {
 	Name               string              `json:"name"`
 	Description        string              `json:"description"`
 	EnvironmentName    string              `json:"environment_name"`
+	NewSessionPerTest  bool                `json:"new_session_per_test,omitempty"` // Optional flag to create a new user session for each test (default: false)
 	Tests              []HTTPTestRequest   `json:"tests"`
 	ApiKey             string              `json:"api_key,omitempty"`             // Optional token to override global.VoiceflowAPIKey
 	VoiceflowSubdomain string              `json:"voiceflow_subdomain,omitempty"` // Optional subdomain to override global.VoiceflowSubdomain
@@ -68,7 +69,7 @@ func ExecuteFromHTTPRequest(suiteReq HTTPSuiteRequest) *ExecuteSuiteResult {
 // executeHTTPSuite executes a suite from HTTP request data
 func executeHTTPSuite(suiteReq HTTPSuiteRequest, logCollector *LogCollector) error {
 	// Define the user ID
-	userID := uuid.New().String()
+	userID := "test-" + uuid.New().String()
 
 	if suiteReq.VoiceflowSubdomain != "" {
 		logCollector.AddLog("Using Voiceflow subdomain: " + suiteReq.VoiceflowSubdomain)
@@ -77,11 +78,20 @@ func executeHTTPSuite(suiteReq HTTPSuiteRequest, logCollector *LogCollector) err
 	logCollector.AddLog("Suite: " + suiteReq.Name)
 	logCollector.AddLog("Description: " + suiteReq.Description)
 	logCollector.AddLog("Environment: " + suiteReq.EnvironmentName)
-	logCollector.AddLog("User ID: " + userID)
+	if suiteReq.NewSessionPerTest {
+		logCollector.AddLog("New session per test: enabled")
+	} else {
+		logCollector.AddLog("User ID: " + userID)
+	}
 	logCollector.AddLog("Running Tests:")
 
 	// Execute each test directly from the request data
 	for _, testReq := range suiteReq.Tests {
+		// Create a new user ID for each test if newSessionPerTest is enabled
+		if suiteReq.NewSessionPerTest {
+			userID = "test-" + uuid.New().String()
+			logCollector.AddLog("User ID for test " + testReq.ID + ": " + userID)
+		}
 		logCollector.AddLog("Running Test ID: " + testReq.ID)
 		err := runTest(suiteReq.EnvironmentName, userID, testReq.Test, suiteReq.ApiKey, suiteReq.VoiceflowSubdomain, logCollector, suiteReq.OpenAIConfig)
 		if err != nil {
@@ -97,7 +107,7 @@ func executeHTTPSuite(suiteReq HTTPSuiteRequest, logCollector *LogCollector) err
 func ExecuteSuite(suitesPath string) error {
 
 	// Define the user ID
-	userID := uuid.New().String()
+	userID := "test-" + uuid.New().String()
 
 	// Load all suites from the path
 	suites, err := utils.LoadSuitesFromPath(suitesPath)
@@ -108,10 +118,19 @@ func ExecuteSuite(suitesPath string) error {
 
 	// Iterate over each suite and its tests
 	for _, suite := range suites {
-		global.Log.Infof("Suite: %s\nDescription: %s\nEnvironment: %s\nUser ID: %s", suite.Name, suite.Description, suite.EnvironmentName, userID)
+		if suite.NewSessionPerTest {
+			global.Log.Infof("Suite: %s\nDescription: %s\nEnvironment: %s\nNew session per test: enabled", suite.Name, suite.Description, suite.EnvironmentName)
+		} else {
+			global.Log.Infof("Suite: %s\nDescription: %s\nEnvironment: %s\nUser ID: %s", suite.Name, suite.Description, suite.EnvironmentName, userID)
+		}
 		global.Log.Infof("Running Tests:")
 
 		for _, testFile := range suite.Tests {
+			// Create a new user ID for each test if newSessionPerTest is enabled
+			if suite.NewSessionPerTest {
+				userID = "test-" + uuid.New().String()
+				global.Log.Infof("User ID for test %s: %s", testFile.ID, userID)
+			}
 			test, err := utils.LoadTestFromPath(testFile.File)
 			if err != nil {
 				global.Log.Errorf("Error loading test: %v", err)

server/docs/docs.go

@@ -304,6 +304,10 @@ const docTemplate = `{
                     "type": "string",
                     "example": "Example Suite"
                 },
+                "new_session_per_test": {
+                    "type": "boolean",
+                    "example": false
+                },
                 "tests": {
                     "type": "array",
                     "items": {

server/handlers/handlers.go

@@ -23,10 +23,11 @@ type TestExecutionRequest struct {
 
 // TestSuiteRequest represents a test suite configuration for HTTP requests
 type TestSuiteRequest struct {
-	Name            string        `json:"name" binding:"required" example:"Example Suite"`
-	Description     string        `json:"description" example:"Suite used as an example"`
-	EnvironmentName string        `json:"environment_name" binding:"required" example:"production"`
-	Tests           []TestRequest `json:"tests" binding:"required,dive"`
+	Name              string        `json:"name" binding:"required" example:"Example Suite"`
+	Description       string        `json:"description" example:"Suite used as an example"`
+	EnvironmentName   string        `json:"environment_name" binding:"required" example:"production"`
+	NewSessionPerTest bool          `json:"new_session_per_test" example:"false"`
+	Tests             []TestRequest `json:"tests" binding:"required,dive"`
 } // @name TestSuiteRequest
 
 // TestRequest represents a test configuration for HTTP requests
@@ -187,6 +188,7 @@ func ExecuteTestSuite(c *gin.Context) {
 			Name:               req.Suite.Name,
 			Description:        req.Suite.Description,
 			EnvironmentName:    req.Suite.EnvironmentName,
+			NewSessionPerTest:  req.Suite.NewSessionPerTest,
 			Tests:              make([]test.HTTPTestRequest, len(req.Suite.Tests)),
 			ApiKey:             req.ApiKey,             // Pass the optional ApiKey
 			VoiceflowSubdomain: req.VoiceflowSubdomain, // Pass the optional VoiceflowSubdomain