From bceaa2dba3d541a9ccea3efb6eeeeb27e2829f80 Mon Sep 17 00:00:00 2001
From: Bai Li <bai.li@uipath.com>
Date: Thu, 30 Apr 2026 15:02:44 -0700
Subject: [PATCH 01/12] delete self-authoring

---
 .../e2e_employee_directory.yaml               | 42 ------------
 .../e2e_product_catalogue.yaml                | 37 ----------
 .../integration_csv_import.yaml               | 33 ---------
 .../integration_entity_scope.yaml             | 30 --------
 .../integration_error_paths.yaml              | 47 -------------
 .../uipath-data-fabric/integration_files.yaml | 33 ---------
 .../integration_full_lifecycle.yaml           | 52 --------------
 .../integration_large_pagination.yaml         | 37 ----------
 .../integration_preseeded_entity.yaml         | 68 -------------------
 .../integration_query_filters.yaml            | 58 ----------------
 .../uipath-data-fabric/smoke_entities.yaml    | 42 ------------
 .../tasks/uipath-data-fabric/smoke_files.yaml | 27 --------
 .../uipath-data-fabric/smoke_records.yaml     | 49 -------------
 .../e2e_02_ai_escalation_brownfield.yaml      | 17 -----
 .../e2e_04_multi_hitl_brownfield.yaml         | 22 ------
 .../e2e_05_expense_approval_brownfield.yaml   | 22 ------
 ...06_invoice_approval_greenfield_simple.yaml | 34 ----------
 .../e2e_07_apptask_brownfield.yaml            | 20 ------
 .../quality_04_all_handles.yaml               | 30 +-------
 .../quality_05_priority_and_timeout.yaml      | 23 -------
 .../quality_07_runtime_vars.yaml              | 35 ----------
 .../uipath-maestro-case/init_validate.yaml    | 22 ------
 .../hitl/quality_01_schema_design.yaml        | 40 -----------
 .../hitl/quality_02_result_downstream.yaml    | 28 --------
 .../hitl/quality_03_boolean_decision.yaml     | 32 ---------
 .../hitl/quality_04_brownfield_insert.yaml    | 23 -------
 .../hitl/smoke_01_hitl_node_placed.yaml       | 22 ------
 .../hitl/smoke_02_completed_port_wired.yaml   | 17 -----
 .../hitl/smoke_03_multi_outcome_routing.yaml  | 26 -------
 .../smoke/init_validate.yaml                  | 23 -------
 .../review_multi_project_solution.yaml        | 31 ---------
 .../uipath-review/review_rpa_project.yaml     | 23 -------
 tests/tasks/uipath-rpa/coded_test_case.yaml   | 35 ----------
 .../uipath-solution-design/gap_detection.yaml | 29 --------
 .../uipath-solution-design/pdd_to_sdd.yaml    | 27 --------
 .../product_selection.yaml                    | 21 ------
 tests/tasks/uipath-tasks/e2e_fetch_tasks.yaml | 51 +-------------
 .../tasks/uipath-tasks/smoke_assignment.yaml  | 31 ---------
 .../tasks/uipath-tasks/smoke_completion.yaml  | 48 -------------
 tests/tasks/uipath-tasks/smoke_discovery.yaml | 29 --------
 40 files changed, 2 insertions(+), 1314 deletions(-)

diff --git a/tests/tasks/uipath-data-fabric/e2e_employee_directory.yaml b/tests/tasks/uipath-data-fabric/e2e_employee_directory.yaml
index 4bfa2b0e9..aa08ccf4f 100644
--- a/tests/tasks/uipath-data-fabric/e2e_employee_directory.yaml
+++ b/tests/tasks/uipath-data-fabric/e2e_employee_directory.yaml
@@ -41,23 +41,6 @@ initial_prompt: |
   2. How many employees are in each department? Use one filterGroup query
      per department to get the count.
 
-  Save a summary to report.json:
-  {
-    "entity_id": "<EmployeeDirectory entity ID>",
-    "total_employees": 20,
-    "schema_evolved": true,
-    "performance_scores_set": 6,
-    "top_performers": [
-      {"name": "<name>", "score": <score>, "department": "<dept>"}
-    ],
-    "department_counts": {
-      "Engineering": <count>,
-      "Sales": <count>,
-      "Finance": <count>,
-      "HR": <count>
-    }
-  }
-
   Do NOT delete the entity.
 
 success_criteria:
@@ -139,31 +122,6 @@ success_criteria:
     weight: 1.0
     pass_threshold: 1.0
 
-  - type: file_exists
-    description: "report.json was created"
-    path: "report.json"
-    weight: 1.0
-    pass_threshold: 1.0
-
-  - type: json_check
-    description: "20 employees imported, schema evolved, 6 scores set, all 4 depts counted"
-    path: "report.json"
-    assertions:
-      - expression: "total_employees"
-        operator: equals
-        expected: 20
-      - expression: "schema_evolved"
-        operator: equals
-        expected: true
-      - expression: "performance_scores_set"
-        operator: equals
-        expected: 6
-      - expression: "length(top_performers)"
-        operator: gte
-        expected: 1
-    weight: 5.0
-    pass_threshold: 0.75
-
 post_run:
   - command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-data-fabric/_shared/cleanup_entities.py"
     timeout: 60
diff --git a/tests/tasks/uipath-data-fabric/e2e_product_catalogue.yaml b/tests/tasks/uipath-data-fabric/e2e_product_catalogue.yaml
index 4e283715d..d088f5c5a 100644
--- a/tests/tasks/uipath-data-fabric/e2e_product_catalogue.yaml
+++ b/tests/tasks/uipath-data-fabric/e2e_product_catalogue.yaml
@@ -37,18 +37,6 @@ initial_prompt: |
   - Confirm the final state: total products, how many are in stock, how
     many are Electronics.
 
-  Save a summary to report.json:
-  {
-    "entity_id": "<ProductCatalogue entity ID>",
-    "total_products": <count>,
-    "electronics_count": <count from query>,
-    "price_updated_count": 2,
-    "out_of_stock_count": 3,
-    "in_stock_count": <total_products minus out_of_stock_count>,
-    "image_uploaded_to_sku": "<SKU of the product the image was uploaded to>",
-    "most_expensive_product_name": "<name>"
-  }
-
   Do NOT delete the entity.
 
 success_criteria:
@@ -116,31 +104,6 @@ success_criteria:
     weight: 2.5
     pass_threshold: 1.0
 
-  - type: file_exists
-    description: "report.json was created"
-    path: "report.json"
-    weight: 1.0
-    pass_threshold: 1.0
-
-  - type: json_check
-    description: "15 products, 5 Electronics, 2 price updates, 3 out of stock"
-    path: "report.json"
-    assertions:
-      - expression: "total_products"
-        operator: equals
-        expected: 15
-      - expression: "electronics_count"
-        operator: equals
-        expected: 5
-      - expression: "price_updated_count"
-        operator: equals
-        expected: 2
-      - expression: "out_of_stock_count"
-        operator: equals
-        expected: 3
-    weight: 5.0
-    pass_threshold: 0.75
-
 post_run:
   - command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-data-fabric/_shared/cleanup_entities.py"
     timeout: 60
diff --git a/tests/tasks/uipath-data-fabric/integration_csv_import.yaml b/tests/tasks/uipath-data-fabric/integration_csv_import.yaml
index cf1853aa5..4ae0d31a0 100644
--- a/tests/tasks/uipath-data-fabric/integration_csv_import.yaml
+++ b/tests/tasks/uipath-data-fabric/integration_csv_import.yaml
@@ -42,17 +42,6 @@ initial_prompt: |
   6. Verify the import: list records and check that rows were added.
      Note InsertedRecords and TotalRecords from the import response.
 
-  Save a summary to report.json:
-  {
-    "entity_id": "<entity ID used>",
-    "entity_existed_already": <true or false>,
-    "csv_headers": ["<headers used in the CSV, in order>"],
-    "csv_row_count": <number of data rows, excluding header>,
-    "csv_includes_system_fields": false,
-    "import_inserted_records": <InsertedRecords value from import response>,
-    "import_total_records": <TotalRecords value from import response>
-  }
-
   Use --output json on every uip df command.
   CSV headers must match entity field names exactly — case-sensitive.
 
@@ -107,28 +96,6 @@ success_criteria:
     weight: 2.5
     pass_threshold: 1.0
 
-  - type: file_exists
-    description: "report.json was created"
-    path: "report.json"
-    weight: 1.0
-    pass_threshold: 1.0
-
-  - type: json_check
-    description: "CSV has 4 rows, no system fields, import counts recorded"
-    path: "report.json"
-    assertions:
-      - expression: "csv_row_count"
-        operator: gte
-        expected: 4
-      - expression: "csv_includes_system_fields"
-        operator: equals
-        expected: false
-      - expression: "import_total_records"
-        operator: gte
-        expected: 4
-    weight: 3.0
-    pass_threshold: 0.75
-
 post_run:
   - command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-data-fabric/_shared/cleanup_entities.py"
     timeout: 60
diff --git a/tests/tasks/uipath-data-fabric/integration_entity_scope.yaml b/tests/tasks/uipath-data-fabric/integration_entity_scope.yaml
index fef291f49..fe91ff50f 100644
--- a/tests/tasks/uipath-data-fabric/integration_entity_scope.yaml
+++ b/tests/tasks/uipath-data-fabric/integration_entity_scope.yaml
@@ -49,17 +49,6 @@ initial_prompt: |
        uip df entities create ... --folder 00000000-0000-0000-0000-000000000000
      Record the error response.
 
-  Save a summary to report.json:
-  {
-    "tenant_entity_id": "<TenantScopeTest entity ID>",
-    "folder_entity_id": "<FolderScopeTest entity ID or null>",
-    "tenant_entity_visible_at_tenant": <true/false>,
-    "tenant_entity_visible_from_folder_a": <true/false or null if single folder>,
-    "folder_entity_isolated_from_folder_b": <true/false or null if single folder>,
-    "invalid_folder_returned_error": <true if a non-zero exit code or error message was returned>,
-    "notes": "<any observations — e.g. single-folder tenant, unexpected scope behaviour>"
-  }
-
   Do NOT delete entities — the test harness handles cleanup.
 
 success_criteria:
@@ -111,25 +100,6 @@ success_criteria:
     weight: 1.5
     pass_threshold: 0.0
 
-  - type: file_exists
-    description: "report.json was created"
-    path: "report.json"
-    weight: 1.0
-    pass_threshold: 1.0
-
-  - type: json_check
-    description: "Tenant entity visible and invalid folder returns error"
-    path: "report.json"
-    assertions:
-      - expression: "tenant_entity_visible_at_tenant"
-        operator: equals
-        expected: true
-      - expression: "invalid_folder_returned_error"
-        operator: equals
-        expected: true
-    weight: 4.0
-    pass_threshold: 0.75
-
 post_run:
   - command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-data-fabric/_shared/cleanup_entities.py"
     timeout: 60
diff --git a/tests/tasks/uipath-data-fabric/integration_error_paths.yaml b/tests/tasks/uipath-data-fabric/integration_error_paths.yaml
index 329dee77d..3450600e5 100644
--- a/tests/tasks/uipath-data-fabric/integration_error_paths.yaml
+++ b/tests/tasks/uipath-data-fabric/integration_error_paths.yaml
@@ -79,31 +79,6 @@ initial_prompt: |
   17. File upload local file not found:
       uip df files upload <entity-id> <seed_id> Score --file ./no-such-file.bin
 
-  Save a summary to report.json:
-  {
-    "entity_id": "<ErrorPathsTest entity ID>",
-    "seed_record_id": "<seed_id>",
-    "errors": {
-      "required_field_missing": "<error text or code>",
-      "unique_constraint_insert": "<error text or code>",
-      "type_mismatch_insert": "<error text or code>",
-      "update_missing_id": "<error text or code>",
-      "update_record_not_found": "<error text or code>",
-      "unique_constraint_update": "<error text or code>",
-      "type_mismatch_update": "<error text or code>",
-      "delete_record_not_found": "<error text or code>",
-      "csv_header_mismatch": "<error text or code>",
-      "csv_malformed": "<error text or code>",
-      "file_upload_non_file_field": "<error text or code>",
-      "file_upload_record_not_found": "<error text or code>",
-      "file_upload_local_not_found": "<error text or code>"
-    },
-    "all_errors_non_empty": <true if every key in errors has a non-empty value>,
-    "record_errors_captured": <count of non-empty values among: required_field_missing, unique_constraint_insert, type_mismatch_insert, update_missing_id, update_record_not_found, unique_constraint_update, type_mismatch_update, delete_record_not_found>,
-    "import_errors_captured": <count of non-empty values among: csv_header_mismatch, csv_malformed>,
-    "file_errors_captured": <count of non-empty values among: file_upload_non_file_field, file_upload_record_not_found, file_upload_local_not_found>
-  }
-
   Do NOT delete the entity — the test harness handles cleanup.
 
 success_criteria:
@@ -163,28 +138,6 @@ success_criteria:
     weight: 2.0
     pass_threshold: 1.0
 
-  - type: file_exists
-    description: "report.json was created"
-    path: "report.json"
-    weight: 1.0
-    pass_threshold: 1.0
-
-  - type: json_check
-    description: "Error scenarios captured per group — allows 1-2 misses per category"
-    path: "report.json"
-    assertions:
-      - expression: "record_errors_captured"
-        operator: gte
-        expected: 6
-      - expression: "import_errors_captured"
-        operator: gte
-        expected: 1
-      - expression: "file_errors_captured"
-        operator: gte
-        expected: 2
-    weight: 4.0
-    pass_threshold: 0.67
-
 post_run:
   - command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-data-fabric/_shared/cleanup_entities.py"
     timeout: 60
diff --git a/tests/tasks/uipath-data-fabric/integration_files.yaml b/tests/tasks/uipath-data-fabric/integration_files.yaml
index 2e08120d5..697a246a2 100644
--- a/tests/tasks/uipath-data-fabric/integration_files.yaml
+++ b/tests/tasks/uipath-data-fabric/integration_files.yaml
@@ -57,17 +57,6 @@ initial_prompt: |
   12. Attempt to download again. The download should fail since the file was deleted.
       Record the result (expected: error or empty response).
 
-  Save a summary to report.json:
-  {
-    "entity_id": "<entity ID used>",
-    "record_id": "<record ID inserted>",
-    "v1_sha256": "<SHA-256 of original.txt>",
-    "download_sha256_matches_v1": <true if downloaded.txt SHA-256 == original.txt>,
-    "v2_sha256": "<SHA-256 of updated.txt>",
-    "overwrite_download_matches_v2": <true if verify_update.txt SHA-256 == updated.txt>,
-    "post_delete_download_failed": <true if download after delete returned an error>
-  }
-
   Do NOT delete the entity — only delete the file attachment.
 
 success_criteria:
@@ -123,28 +112,6 @@ success_criteria:
     weight: 1.5
     pass_threshold: 1.0
 
-  - type: file_exists
-    description: "report.json was created"
-    path: "report.json"
-    weight: 1.0
-    pass_threshold: 1.0
-
-  - type: json_check
-    description: "Byte integrity confirmed for v1, overwrite confirmed for v2, post-delete download failed"
-    path: "report.json"
-    assertions:
-      - expression: "download_sha256_matches_v1"
-        operator: equals
-        expected: true
-      - expression: "overwrite_download_matches_v2"
-        operator: equals
-        expected: true
-      - expression: "post_delete_download_failed"
-        operator: equals
-        expected: true
-    weight: 4.0
-    pass_threshold: 0.75
-
 post_run:
   - command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-data-fabric/_shared/cleanup_entities.py"
     timeout: 60
diff --git a/tests/tasks/uipath-data-fabric/integration_full_lifecycle.yaml b/tests/tasks/uipath-data-fabric/integration_full_lifecycle.yaml
index f3b909be8..eaed9f90d 100644
--- a/tests/tasks/uipath-data-fabric/integration_full_lifecycle.yaml
+++ b/tests/tasks/uipath-data-fabric/integration_full_lifecycle.yaml
@@ -54,21 +54,6 @@ initial_prompt: |
   10. Delete only the two records inserted in step 6.
       Do NOT delete the entity.
 
-  Save a summary to report.json:
-  {
-    "entity_id": "<ID of LifecycleTest entity>",
-    "entity_existed_already": <true or false>,
-    "schema_ops": {
-      "field_id_retrieved_from": "entities get",
-      "addFields": ["<field names added>"],
-      "updateFields": ["<field names updated>"]
-    },
-    "records_inserted": ["<id1>", "<id2>"],
-    "query_result_count": <number of records from Active=true query>,
-    "score_after_update": <score value confirmed from records get>,
-    "records_deleted": ["<id1>", "<id2>"]
-  }
-
   Do NOT use removeFields. Do NOT delete the entity.
 
 success_criteria:
@@ -184,43 +169,6 @@ success_criteria:
     weight: 2.0
     pass_threshold: 1.0
 
-  - type: file_exists
-    description: "report.json was created"
-    path: "report.json"
-    weight: 1.0
-    pass_threshold: 1.0
-
-  - type: file_contains
-    description: "Report includes schema operations (addFields and updateFields)"
-    path: "report.json"
-    includes:
-      - "addFields"
-      - "updateFields"
-    weight: 2.0
-    pass_threshold: 1.0
-
-  - type: json_check
-    description: "Full lifecycle complete: field IDs from entities get, records inserted/deleted, update verified"
-    path: "report.json"
-    assertions:
-      - expression: "schema_ops.field_id_retrieved_from"
-        operator: equals
-        expected: "entities get"
-      - expression: "length(records_inserted)"
-        operator: gte
-        expected: 2
-      - expression: "score_after_update"
-        operator: equals
-        expected: 99
-      - expression: "query_result_count"
-        operator: gte
-        expected: 1
-      - expression: "length(records_deleted)"
-        operator: gte
-        expected: 2
-    weight: 4.0
-    pass_threshold: 0.75
-
 post_run:
   - command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-data-fabric/_shared/cleanup_entities.py"
     timeout: 60
diff --git a/tests/tasks/uipath-data-fabric/integration_large_pagination.yaml b/tests/tasks/uipath-data-fabric/integration_large_pagination.yaml
index c9c5f5d34..8fb4fbae2 100644
--- a/tests/tasks/uipath-data-fabric/integration_large_pagination.yaml
+++ b/tests/tasks/uipath-data-fabric/integration_large_pagination.yaml
@@ -65,18 +65,6 @@ initial_prompt: |
        - every collected record has Active=true
        - total count == 300
 
-  Save a summary to report.json:
-  {
-    "entity_id": "<LargePaginationTest entity ID>",
-    "import_count": <number of records imported>,
-    "pagination_pages_fetched": <number of list calls made for full traversal>,
-    "pagination_total_unique_ids": <count of unique record IDs across all pages>,
-    "pagination_no_duplicates": <true if no duplicate IDs>,
-    "filtered_pages_fetched": <number of query calls made for Active=true filter>,
-    "filtered_active_count": <count of records returned by Active=true filter>,
-    "filtered_all_active": <true if every filtered record has Active=true>
-  }
-
   Do NOT delete the entity — the test harness handles cleanup.
 
 success_criteria:
@@ -118,31 +106,6 @@ success_criteria:
     weight: 1.0
     pass_threshold: 1.0
 
-  - type: file_exists
-    description: "report.json was created"
-    path: "report.json"
-    weight: 1.0
-    pass_threshold: 1.0
-
-  - type: json_check
-    description: "500 records paginated without duplicates; 300 Active=true records confirmed"
-    path: "report.json"
-    assertions:
-      - expression: "pagination_total_unique_ids"
-        operator: equals
-        expected: 500
-      - expression: "pagination_no_duplicates"
-        operator: equals
-        expected: true
-      - expression: "filtered_active_count"
-        operator: equals
-        expected: 300
-      - expression: "filtered_all_active"
-        operator: equals
-        expected: true
-    weight: 5.0
-    pass_threshold: 0.75
-
 post_run:
   - command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-data-fabric/_shared/cleanup_entities.py"
     timeout: 60
diff --git a/tests/tasks/uipath-data-fabric/integration_preseeded_entity.yaml b/tests/tasks/uipath-data-fabric/integration_preseeded_entity.yaml
index 03a394a83..fa80d2ed7 100644
--- a/tests/tasks/uipath-data-fabric/integration_preseeded_entity.yaml
+++ b/tests/tasks/uipath-data-fabric/integration_preseeded_entity.yaml
@@ -103,25 +103,6 @@ initial_prompt: |
      Verify the restore (confirm Notes == null).
      Save the record ID you modified.
 
-  Save a summary to report.json:
-  {
-    "entity_found": true,
-    "entity_seeded_now": <true if you just created and imported it, false if it was pre-existing>,
-    "entity_id": "<CodeEvalTestEntity ID>",
-    "field_count": <number of fields in schema>,
-    "query_active_true_count": <count>,
-    "query_score_gt_7_count": <count>,
-    "query_title_exact_count": <count>,
-    "query_title_contains_count": <count>,
-    "query_score_gte7_and_active_true_count": <count>,
-    "sort_highest_score": <Score of records[0] from descending sort>,
-    "pagination_total_records": <unique ID count across all pages>,
-    "pagination_no_duplicates": <true/false>,
-    "update_record_id": "<ID of record you modified>",
-    "update_verified": <true if records get confirmed the new Notes value>,
-    "restore_verified": <true if records get confirmed Notes is null again>
-  }
-
   Do NOT delete any records. Do NOT delete the entity.
 
 success_criteria:
@@ -205,53 +186,4 @@ success_criteria:
     weight: 2.0
     pass_threshold: 1.0
 
-  - type: file_exists
-    description: "report.json was created"
-    path: "report.json"
-    weight: 1.0
-    pass_threshold: 1.0
-
-  - type: json_check
-    description: "Entity found, query counts match seed data, sort and pagination correct"
-    path: "report.json"
-    assertions:
-      - expression: "entity_found"
-        operator: equals
-        expected: true
-      - expression: "field_count"
-        operator: gte
-        expected: 7
-      - expression: "query_active_true_count"
-        operator: equals
-        expected: 12
-      - expression: "query_score_gt_7_count"
-        operator: equals
-        expected: 6
-      - expression: "query_title_exact_count"
-        operator: equals
-        expected: 1
-      - expression: "query_title_contains_count"
-        operator: equals
-        expected: 3
-      - expression: "query_score_gte7_and_active_true_count"
-        operator: equals
-        expected: 7
-      - expression: "sort_highest_score"
-        operator: gte
-        expected: 10
-      - expression: "pagination_total_records"
-        operator: equals
-        expected: 20
-      - expression: "pagination_no_duplicates"
-        operator: equals
-        expected: true
-      - expression: "update_verified"
-        operator: equals
-        expected: true
-      - expression: "restore_verified"
-        operator: equals
-        expected: true
-    weight: 6.0
-    pass_threshold: 0.75
-
 # No post_run: CodeEvalTestEntity is a shared persistent fixture — never deleted between runs.
diff --git a/tests/tasks/uipath-data-fabric/integration_query_filters.yaml b/tests/tasks/uipath-data-fabric/integration_query_filters.yaml
index de25d6a37..e9a4ba00e 100644
--- a/tests/tasks/uipath-data-fabric/integration_query_filters.yaml
+++ b/tests/tasks/uipath-data-fabric/integration_query_filters.yaml
@@ -67,21 +67,6 @@ initial_prompt: |
 
   10. Delete the 4 records you inserted in step 3.
 
-  Save a summary to report.json:
-  {
-    "entity_id": "<entity ID used>",
-    "records_inserted": ["<id1>", "<id2>", "<id3>", "<id4>"],
-    "and_query_count": <number>,
-    "or_query_count": <number>,
-    "nested_query_count": <number>,
-    "sort_first_value": <number>,
-    "projection_value_absent": <true/false>,
-    "projection_status_absent": <true/false>,
-    "pagination_total_ids": <number>,
-    "pagination_no_duplicates": <true/false>,
-    "records_deleted": ["<ids deleted>"]
-  }
-
   Use --output json on every uip df command.
   Pagination must use --limit and --cursor CLI flags — never body keys.
 
@@ -158,49 +143,6 @@ success_criteria:
     weight: 2.0
     pass_threshold: 1.0
 
-  - type: file_exists
-    description: "report.json was created"
-    path: "report.json"
-    weight: 1.0
-    pass_threshold: 1.0
-
-  - type: json_check
-    description: "AND→1, OR→2, nested→3, sort first=750, projection excludes Value/Status, pagination 4 unique IDs"
-    path: "report.json"
-    assertions:
-      - expression: "and_query_count"
-        operator: equals
-        expected: 1
-      - expression: "or_query_count"
-        operator: equals
-        expected: 2
-      - expression: "nested_query_count"
-        operator: equals
-        expected: 3
-      - expression: "sort_first_value"
-        operator: equals
-        expected: 750.0
-      - expression: "projection_value_absent"
-        operator: equals
-        expected: true
-      - expression: "projection_status_absent"
-        operator: equals
-        expected: true
-      - expression: "pagination_total_ids"
-        operator: equals
-        expected: 4
-      - expression: "pagination_no_duplicates"
-        operator: equals
-        expected: true
-      - expression: "length(records_inserted)"
-        operator: gte
-        expected: 4
-      - expression: "length(records_deleted)"
-        operator: gte
-        expected: 4
-    weight: 5.0
-    pass_threshold: 0.75
-
 post_run:
   - command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-data-fabric/_shared/cleanup_entities.py"
     timeout: 60
diff --git a/tests/tasks/uipath-data-fabric/smoke_entities.yaml b/tests/tasks/uipath-data-fabric/smoke_entities.yaml
index 3b35f1e15..45023f3d5 100644
--- a/tests/tasks/uipath-data-fabric/smoke_entities.yaml
+++ b/tests/tasks/uipath-data-fabric/smoke_entities.yaml
@@ -39,16 +39,6 @@ initial_prompt: |
      would come from Fields[].ID in the entities get response) in the
      updateFields body.
 
-  Save a summary to report.json with the bodies and commands you CONSTRUCTED
-  (fill these in regardless of whether the commands returned auth errors):
-  {
-    "commands_used": ["<list of uip df commands you ran in order>"],
-    "create_body": { "fields": [ ... ] },
-    "addFields_body": { "addFields": [ ... ] },
-    "updateFields_body": { "updateFields": [ ... ] },
-    "field_id_source": "entities get"
-  }
-
   Additional constraints:
   - Use --output json on every uip df command.
   - Do NOT use removeFields. Do NOT delete any entity.
@@ -102,35 +92,3 @@ success_criteria:
     weight: 1.0
     pass_threshold: 1.0
 
-  - type: file_exists
-    description: "report.json was created"
-    path: "report.json"
-    weight: 1.0
-    pass_threshold: 1.0
-
-  - type: file_contains
-    description: "Entity create and addFields bodies use fieldName key (not name)"
-    path: "report.json"
-    includes:
-      - "fieldName"
-    weight: 2.0
-    pass_threshold: 1.0
-
-  - type: file_contains
-    description: "Report includes both addFields and updateFields operations"
-    path: "report.json"
-    includes:
-      - "addFields"
-      - "updateFields"
-    weight: 2.0
-    pass_threshold: 1.0
-
-  - type: json_check
-    description: "Field IDs sourced from entities get"
-    path: "report.json"
-    assertions:
-      - expression: "field_id_source"
-        operator: equals
-        expected: "entities get"
-    weight: 1.5
-    pass_threshold: 1.0
diff --git a/tests/tasks/uipath-data-fabric/smoke_files.yaml b/tests/tasks/uipath-data-fabric/smoke_files.yaml
index c274137e6..882083cc6 100644
--- a/tests/tasks/uipath-data-fabric/smoke_files.yaml
+++ b/tests/tasks/uipath-data-fabric/smoke_files.yaml
@@ -37,15 +37,6 @@ initial_prompt: |
   4. Delete the file attachment from that field:
        uip df files delete doc-001-ent rec-001-id Attachment
 
-  Save a summary to report.json with the commands you ran and what you observed
-  (fill these in based on what you sent, not the error responses):
-  {
-    "commands_used": ["<list of uip df commands you ran in order>"],
-    "upload_command_uses_positional_args": true,
-    "download_uses_destination_flag": true,
-    "delete_positional_args": ["<entity-id>", "<record-id>", "<field-name>"]
-  }
-
   Additional constraints:
   - Use --output json on every uip df command.
   - The download flag is --destination (not --output).
@@ -90,21 +81,3 @@ success_criteria:
     weight: 1.0
     pass_threshold: 1.0
 
-  - type: file_exists
-    description: "report.json was created"
-    path: "report.json"
-    weight: 1.0
-    pass_threshold: 1.0
-
-  - type: json_check
-    description: "Upload uses positional args and download uses --destination flag"
-    path: "report.json"
-    assertions:
-      - expression: "upload_command_uses_positional_args"
-        operator: equals
-        expected: true
-      - expression: "download_uses_destination_flag"
-        operator: equals
-        expected: true
-    weight: 2.5
-    pass_threshold: 1.0
diff --git a/tests/tasks/uipath-data-fabric/smoke_records.yaml b/tests/tasks/uipath-data-fabric/smoke_records.yaml
index 5091ed641..b08e49844 100644
--- a/tests/tasks/uipath-data-fabric/smoke_records.yaml
+++ b/tests/tasks/uipath-data-fabric/smoke_records.yaml
@@ -45,20 +45,6 @@ initial_prompt: |
   8. Delete the record from step 1 using records delete.
      Use placeholder record ID "id-single".
 
-  Save a summary to report.json with the bodies and commands you CONSTRUCTED
-  (fill these in based on what you sent, not the error responses):
-  {
-    "commands_used": ["<list of uip df commands you ran in order>"],
-    "single_insert_uses_object": true,
-    "batch_insert_uses_array": true,
-    "get_used_record_id": true,
-    "query_filter": { ... the filterGroup JSON used ... },
-    "pagination_method": "cursor",
-    "single_update_body_has_id": true,
-    "batch_update_all_have_id": true,
-    "delete_attempted": true
-  }
-
   Additional constraint:
   - Use --output json on every uip df command.
 
@@ -135,38 +121,3 @@ success_criteria:
     weight: 1.0
     pass_threshold: 1.0
 
-  - type: file_exists
-    description: "report.json was created"
-    path: "report.json"
-    weight: 1.0
-    pass_threshold: 1.0
-
-  - type: file_contains
-    description: "Query body uses filterGroup key"
-    path: "report.json"
-    includes:
-      - "filterGroup"
-    weight: 2.0
-    pass_threshold: 1.0
-
-  - type: json_check
-    description: "Array routing, cursor pagination, Id in update bodies, and delete confirmed"
-    path: "report.json"
-    assertions:
-      - expression: "batch_insert_uses_array"
-        operator: equals
-        expected: true
-      - expression: "pagination_method"
-        operator: equals
-        expected: "cursor"
-      - expression: "single_update_body_has_id"
-        operator: equals
-        expected: true
-      - expression: "batch_update_all_have_id"
-        operator: equals
-        expected: true
-      - expression: "delete_attempted"
-        operator: equals
-        expected: true
-    weight: 3.0
-    pass_threshold: 0.75
diff --git a/tests/tasks/uipath-human-in-the-loop/e2e_02_ai_escalation_brownfield.yaml b/tests/tasks/uipath-human-in-the-loop/e2e_02_ai_escalation_brownfield.yaml
index 40eaaacc0..3832fe4a8 100644
--- a/tests/tasks/uipath-human-in-the-loop/e2e_02_ai_escalation_brownfield.yaml
+++ b/tests/tasks/uipath-human-in-the-loop/e2e_02_ai_escalation_brownfield.yaml
@@ -42,14 +42,6 @@ initial_prompt: |
   make the final routing decision before the complaint is acted on.
   Insert a HITL node in the appropriate place, wire it, and validate.
 
-  Save results to report.json:
-  {
-    "hitl_node_id": "<id>",
-    "inserted_position": "<description of where in the flow>",
-    "handles_wired": ["<list>"],
-    "validation_passed": true
-  }
-
 success_criteria:
   - type: file_contains
     description: "HITL node is present in the flow file"
@@ -75,12 +67,3 @@ success_criteria:
     weight: 2.0
     pass_threshold: 1.0
 
-  - type: json_check
-    description: "report.json confirms validation passed"
-    path: "report.json"
-    assertions:
-      - expression: "validation_passed"
-        operator: equals
-        expected: true
-    weight: 2.5
-    pass_threshold: 1.0
diff --git a/tests/tasks/uipath-human-in-the-loop/e2e_04_multi_hitl_brownfield.yaml b/tests/tasks/uipath-human-in-the-loop/e2e_04_multi_hitl_brownfield.yaml
index a1f1d792f..26a1280c6 100644
--- a/tests/tasks/uipath-human-in-the-loop/e2e_04_multi_hitl_brownfield.yaml
+++ b/tests/tasks/uipath-human-in-the-loop/e2e_04_multi_hitl_brownfield.yaml
@@ -47,16 +47,6 @@ initial_prompt: |
   Wire the completed handles for both HITL nodes to their respective downstream
   steps. Validate the full flow after both nodes are added.
 
-  Save results to report.json:
-  {
-    "hitl_nodes": [
-      {"id": "<first HITL node id>", "label": "<label>", "before": "<node it precedes>"},
-      {"id": "<second HITL node id>", "label": "<label>", "before": "<node it precedes>"}
-    ],
-    "handles_wired": ["<list of all edges added>"],
-    "validation_passed": true
-  }
-
 success_criteria:
   - type: file_contains
     description: "HITL node(s) present in the flow file"
@@ -82,15 +72,3 @@ success_criteria:
     weight: 2.0
     pass_threshold: 1.0
 
-  - type: json_check
-    description: "report.json shows 2 HITL nodes and validation passed"
-    path: "report.json"
-    assertions:
-      - expression: "length(hitl_nodes)"
-        operator: gte
-        expected: 2
-      - expression: "validation_passed"
-        operator: equals
-        expected: true
-    weight: 3.0
-    pass_threshold: 1.0
diff --git a/tests/tasks/uipath-human-in-the-loop/e2e_05_expense_approval_brownfield.yaml b/tests/tasks/uipath-human-in-the-loop/e2e_05_expense_approval_brownfield.yaml
index 43616c775..55a756e8d 100644
--- a/tests/tasks/uipath-human-in-the-loop/e2e_05_expense_approval_brownfield.yaml
+++ b/tests/tasks/uipath-human-in-the-loop/e2e_05_expense_approval_brownfield.yaml
@@ -55,13 +55,6 @@ initial_prompt: |
   A manager should review and approve the expense before it is posted.
   Wire the completed handle to the posting step and validate the flow.
 
-  Save a summary to report.json with:
-  {
-    "hitl_node_id": "<the ID of the HITL node you added>",
-    "commands_used": ["<list of uip commands run>"],
-    "validation_passed": true
-  }
-
 success_criteria:
   - type: file_contains
     description: "HITL node is present in the flow file"
@@ -87,18 +80,3 @@ success_criteria:
     weight: 2.0
     pass_threshold: 1.0
 
-  - type: file_exists
-    description: "report.json was written"
-    path: "report.json"
-    weight: 1.0
-    pass_threshold: 1.0
-
-  - type: json_check
-    description: "report.json confirms validation passed and captures node ID"
-    path: "report.json"
-    assertions:
-      - expression: "validation_passed"
-        operator: equals
-        expected: true
-    weight: 2.0
-    pass_threshold: 1.0
diff --git a/tests/tasks/uipath-human-in-the-loop/e2e_06_invoice_approval_greenfield_simple.yaml b/tests/tasks/uipath-human-in-the-loop/e2e_06_invoice_approval_greenfield_simple.yaml
index d76ff887d..2dc36c2de 100644
--- a/tests/tasks/uipath-human-in-the-loop/e2e_06_invoice_approval_greenfield_simple.yaml
+++ b/tests/tasks/uipath-human-in-the-loop/e2e_06_invoice_approval_greenfield_simple.yaml
@@ -27,16 +27,6 @@ initial_prompt: |
 
   After adding the HITL node and wiring it, validate the flow.
 
-  Save a summary to report.json with:
-  {
-    "project_name": "InvoiceApproval",
-    "hitl_node_id": "<the ID of the HITL node>",
-    "schema_inputs": ["<field names shown to the human>"],
-    "schema_outcomes": ["<button names>"],
-    "commands_used": ["<list of uip commands run>"],
-    "validation_passed": true
-  }
-
 success_criteria:
   - type: command_executed
     description: "Agent created a solution"
@@ -78,27 +68,3 @@ success_criteria:
     weight: 2.0
     pass_threshold: 1.0
 
-  - type: file_exists
-    description: "report.json was written"
-    path: "report.json"
-    weight: 1.0
-    pass_threshold: 1.0
-
-  - type: json_check
-    description: "report.json has correct project name and validation passed"
-    path: "report.json"
-    assertions:
-      - expression: "project_name"
-        operator: equals
-        expected: "InvoiceApproval"
-      - expression: "validation_passed"
-        operator: equals
-        expected: true
-      - expression: "length(schema_inputs)"
-        operator: gte
-        expected: 2
-      - expression: "length(schema_outcomes)"
-        operator: gte
-        expected: 2
-    weight: 2.0
-    pass_threshold: 0.75
diff --git a/tests/tasks/uipath-human-in-the-loop/e2e_07_apptask_brownfield.yaml b/tests/tasks/uipath-human-in-the-loop/e2e_07_apptask_brownfield.yaml
index 6030630da..1cd6d641c 100644
--- a/tests/tasks/uipath-human-in-the-loop/e2e_07_apptask_brownfield.yaml
+++ b/tests/tasks/uipath-human-in-the-loop/e2e_07_apptask_brownfield.yaml
@@ -47,14 +47,6 @@ initial_prompt: |
   app. Use the AppTask path (inputs.type = "custom"), not an inline QuickForm.
   Wire the completed handle to the posting step and validate the flow.
 
-  Save results to report.json:
-  {
-    "hitl_node_id": "<id of the HITL node>",
-    "app_name": "Expense Review",
-    "inputs_type": "custom",
-    "validation_passed": true
-  }
-
 success_criteria:
   - type: file_contains
     description: "HITL node with AppTask (custom) type is present in the flow file"
@@ -81,15 +73,3 @@ success_criteria:
     weight: 2.0
     pass_threshold: 1.0
 
-  - type: json_check
-    description: "report.json confirms AppTask type and validation passed"
-    path: "report.json"
-    assertions:
-      - expression: "inputs_type"
-        operator: equals
-        expected: "custom"
-      - expression: "validation_passed"
-        operator: equals
-        expected: true
-    weight: 2.0
-    pass_threshold: 1.0
diff --git a/tests/tasks/uipath-human-in-the-loop/quality_04_all_handles.yaml b/tests/tasks/uipath-human-in-the-loop/quality_04_all_handles.yaml
index c8e7acd1c..107cb73bf 100644
--- a/tests/tasks/uipath-human-in-the-loop/quality_04_all_handles.yaml
+++ b/tests/tasks/uipath-human-in-the-loop/quality_04_all_handles.yaml
@@ -24,14 +24,7 @@ initial_prompt: |
   processResult script node body, show how to read the human's decision using
   $vars.<hitl-node-id>.result.
 
-  Validate after adding and wiring. Save results to report.json:
-  {
-    "hitl_node_id": "<id>",
-    "handles_wired": ["completed"],
-    "result_variable_expression": "$vars.<hitl-node-id>.result",
-    "commands_used": ["<list>"],
-    "validation_passed": true
-  }
+  Validate after adding and wiring.
 
 success_criteria:
   - type: file_contains
@@ -58,24 +51,3 @@ success_criteria:
     weight: 1.5
     pass_threshold: 1.0
 
-  - type: json_check
-    description: "report.json shows completed handle wired and validation passed"
-    path: "report.json"
-    assertions:
-      - expression: "length(handles_wired)"
-        operator: gte
-        expected: 1
-      - expression: "validation_passed"
-        operator: equals
-        expected: true
-    weight: 2.5
-    pass_threshold: 1.0
-
-  - type: file_contains
-    description: "report.json references $vars result expression"
-    path: "report.json"
-    includes:
-      - "$vars"
-      - ".result"
-    weight: 2.0
-    pass_threshold: 1.0
diff --git a/tests/tasks/uipath-human-in-the-loop/quality_05_priority_and_timeout.yaml b/tests/tasks/uipath-human-in-the-loop/quality_05_priority_and_timeout.yaml
index 1aedcdeb1..03ba60e8e 100644
--- a/tests/tasks/uipath-human-in-the-loop/quality_05_priority_and_timeout.yaml
+++ b/tests/tasks/uipath-human-in-the-loop/quality_05_priority_and_timeout.yaml
@@ -21,14 +21,6 @@ initial_prompt: |
   Wire the completed handle to a script node that logs the approval.
   Validate the flow after adding.
 
-  Save results to report.json:
-  {
-    "hitl_node_id": "<id>",
-    "priority_used": "<the priority value set in the flow, e.g. High>",
-    "timeout_used": "<the ISO 8601 duration set in the flow, e.g. PT48H>",
-    "validation_passed": true
-  }
-
 success_criteria:
   - type: file_contains
     description: "Flow file contains PT48H timeout for the HITL node"
@@ -54,18 +46,3 @@ success_criteria:
     weight: 1.5
     pass_threshold: 1.0
 
-  - type: json_check
-    description: "report.json captures correct priority and timeout"
-    path: "report.json"
-    assertions:
-      - expression: "priority_used"
-        operator: equals
-        expected: "High"
-      - expression: "timeout_used"
-        operator: equals
-        expected: "PT48H"
-      - expression: "validation_passed"
-        operator: equals
-        expected: true
-    weight: 2.0
-    pass_threshold: 0.75
diff --git a/tests/tasks/uipath-human-in-the-loop/quality_07_runtime_vars.yaml b/tests/tasks/uipath-human-in-the-loop/quality_07_runtime_vars.yaml
index 848a1f8c9..015be9945 100644
--- a/tests/tasks/uipath-human-in-the-loop/quality_07_runtime_vars.yaml
+++ b/tests/tasks/uipath-human-in-the-loop/quality_07_runtime_vars.yaml
@@ -22,14 +22,6 @@ initial_prompt: |
   The script node must use the HITL runtime variables to access the result.
   Wire the completed handle to the script node. Validate the flow.
 
-  Save results to report.json — use the actual node ID that was generated:
-  {
-    "hitl_node_id": "<the actual id of the HITL node>",
-    "result_variable": "$vars.<hitl_node_id>.result",
-    "status_variable": "$vars.<hitl_node_id>.status",
-    "validation_passed": true
-  }
-
 success_criteria:
   - type: file_contains
     description: "HITL node with specified ID is present in the flow file"
@@ -47,30 +39,3 @@ success_criteria:
     weight: 1.5
     pass_threshold: 1.0
 
-  - type: file_contains
-    description: "report.json references a $vars result variable"
-    path: "report.json"
-    includes:
-      - "$vars."
-      - ".result"
-    weight: 3.0
-    pass_threshold: 1.0
-
-  - type: file_contains
-    description: "report.json references a $vars status variable"
-    path: "report.json"
-    includes:
-      - "$vars."
-      - ".status"
-    weight: 2.0
-    pass_threshold: 1.0
-
-  - type: json_check
-    description: "report.json shows validation passed"
-    path: "report.json"
-    assertions:
-      - expression: "validation_passed"
-        operator: equals
-        expected: true
-    weight: 1.5
-    pass_threshold: 1.0
diff --git a/tests/tasks/uipath-maestro-case/init_validate.yaml b/tests/tasks/uipath-maestro-case/init_validate.yaml
index 09add2ede..37834befb 100644
--- a/tests/tasks/uipath-maestro-case/init_validate.yaml
+++ b/tests/tasks/uipath-maestro-case/init_validate.yaml
@@ -34,13 +34,6 @@ initial_prompt: |
   - Intake (regular stage) — receives the case after the manual trigger.
   ```
 
-  Save a summary of what you did to report.json with at minimum:
-    {
-      "project_name": "OnboardingCase",
-      "commands_used": ["<list of uip commands you ran>"],
-      "validation_passed": true
-    }
-
   Important:
   - The `uip` CLI is already available in the environment.
   - Treat the generated plan as pre-approved for this smoke test — do not
@@ -78,18 +71,3 @@ success_criteria:
     weight: 1.5
     pass_threshold: 1.0
 
-  - type: json_check
-    description: "report.json has correct structure and values"
-    path: "report.json"
-    assertions:
-      - expression: "project_name"
-        operator: equals
-        expected: "OnboardingCase"
-      - expression: "validation_passed"
-        operator: equals
-        expected: true
-      - expression: "length(commands_used)"
-        operator: gte
-        expected: 1
-    weight: 2.0
-    pass_threshold: 0.75
diff --git a/tests/tasks/uipath-maestro-flow/hitl/quality_01_schema_design.yaml b/tests/tasks/uipath-maestro-flow/hitl/quality_01_schema_design.yaml
index 6dec35ec2..cd9a3c43b 100644
--- a/tests/tasks/uipath-maestro-flow/hitl/quality_01_schema_design.yaml
+++ b/tests/tasks/uipath-maestro-flow/hitl/quality_01_schema_design.yaml
@@ -33,19 +33,6 @@ initial_prompt: |
   Wire completed → decision node.
   Do NOT run flow debug. Validate after building.
 
-  Save a summary to report.json:
-  {
-    "hitl_node_id": "<id>",
-    "schema": {
-      "input_fields":  ["<names of read-only fields>"],
-      "output_fields": ["<names of fields the officer fills in>"],
-      "outcomes":      ["<outcome names>"]
-    },
-    "priority": "<priority set>",
-    "result_variable": "<how the decision node reads the officer's decision>",
-    "validation_passed": true
-  }
-
 success_criteria:
   - type: run_command
     description: "uip flow validate passes"
@@ -88,30 +75,3 @@ success_criteria:
     weight: 1.5
     pass_threshold: 1.0
 
-  - type: file_contains
-    description: "result_variable references $vars.<nodeId>.result"
-    path: "report.json"
-    includes:
-      - "$vars."
-      - ".result"
-    weight: 2.0
-    pass_threshold: 1.0
-
-  - type: json_check
-    description: "Schema has at least 2 input fields, at least 1 output field, 2 outcomes"
-    path: "report.json"
-    assertions:
-      - expression: "length(schema.input_fields)"
-        operator: gte
-        expected: 2
-      - expression: "length(schema.output_fields)"
-        operator: gte
-        expected: 1
-      - expression: "length(schema.outcomes)"
-        operator: gte
-        expected: 2
-      - expression: "validation_passed"
-        operator: equals
-        expected: true
-    weight: 2.5
-    pass_threshold: 0.75
diff --git a/tests/tasks/uipath-maestro-flow/hitl/quality_02_result_downstream.yaml b/tests/tasks/uipath-maestro-flow/hitl/quality_02_result_downstream.yaml
index c3693b8fe..1d3a44d80 100644
--- a/tests/tasks/uipath-maestro-flow/hitl/quality_02_result_downstream.yaml
+++ b/tests/tasks/uipath-maestro-flow/hitl/quality_02_result_downstream.yaml
@@ -33,13 +33,6 @@ initial_prompt: |
   Use the inline quickform node (uipath.human-in-the-loop).
   Do NOT run flow debug. Validate after building.
 
-  Save a summary to report.json:
-  {
-    "hitl_node_id": "<id>",
-    "result_path": "<full variable path used in the script, e.g. $vars.hitlReview1.result.approved>",
-    "validation_passed": true
-  }
-
 success_criteria:
   - type: run_command
     description: "uip flow validate passes"
@@ -74,24 +67,3 @@ success_criteria:
     weight: 1.5
     pass_threshold: 1.0
 
-  - type: file_contains
-    description: "report.json result_path references $vars and .result"
-    path: "report.json"
-    includes:
-      - "$vars."
-      - ".result"
-    weight: 2.5
-    pass_threshold: 1.0
-
-  - type: json_check
-    description: "report.json captures hitl_node_id and validation"
-    path: "report.json"
-    assertions:
-      - expression: "validation_passed"
-        operator: equals
-        expected: true
-      - expression: "hitl_node_id"
-        operator: not_equals
-        expected: ""
-    weight: 2.0
-    pass_threshold: 0.75
diff --git a/tests/tasks/uipath-maestro-flow/hitl/quality_03_boolean_decision.yaml b/tests/tasks/uipath-maestro-flow/hitl/quality_03_boolean_decision.yaml
index 7981338f6..4a5650c6f 100644
--- a/tests/tasks/uipath-maestro-flow/hitl/quality_03_boolean_decision.yaml
+++ b/tests/tasks/uipath-maestro-flow/hitl/quality_03_boolean_decision.yaml
@@ -36,14 +36,6 @@ initial_prompt: |
 
   Do NOT run flow debug. Validate after building.
 
-  Save a summary to report.json:
-  {
-    "hitl_node_id": "<id>",
-    "decision_condition": "<full expression used, e.g. $vars.complianceReview1.result.approved>",
-    "result_path_used": "<same or equivalent path referenced in the Decision>",
-    "validation_passed": true
-  }
-
 success_criteria:
   - type: run_command
     description: "uip maestro flow validate passes"
@@ -80,27 +72,3 @@ success_criteria:
     weight: 2.0
     pass_threshold: 1.0
 
-  - type: file_contains
-    description: "report.json decision_condition references field-level variable"
-    path: "report.json"
-    includes:
-      - "$vars."
-      - ".result.approved"
-    weight: 2.5
-    pass_threshold: 1.0
-
-  - type: json_check
-    description: "report.json has hitl_node_id and validation passed"
-    path: "report.json"
-    assertions:
-      - expression: "validation_passed"
-        operator: equals
-        expected: true
-      - expression: "hitl_node_id"
-        operator: not_equals
-        expected: ""
-      - expression: "decision_condition"
-        operator: contains
-        expected: "$vars."
-    weight: 2.0
-    pass_threshold: 0.75
diff --git a/tests/tasks/uipath-maestro-flow/hitl/quality_04_brownfield_insert.yaml b/tests/tasks/uipath-maestro-flow/hitl/quality_04_brownfield_insert.yaml
index c08cd2e93..71993ebb9 100644
--- a/tests/tasks/uipath-maestro-flow/hitl/quality_04_brownfield_insert.yaml
+++ b/tests/tasks/uipath-maestro-flow/hitl/quality_04_brownfield_insert.yaml
@@ -37,14 +37,6 @@ initial_prompt: |
 
   Do NOT run flow debug. Validate after building.
 
-  Save a summary to report.json:
-  {
-    "hitl_node_id": "<id of the new HITL node>",
-    "original_script_id": "extractMeta",
-    "edges_after_insert": ["<list all edge source→target pairs>"],
-    "validation_passed": true
-  }
-
 success_criteria:
   - type: run_command
     description: "uip maestro flow validate passes after brownfield insert"
@@ -86,18 +78,3 @@ success_criteria:
     weight: 1.5
     pass_threshold: 1.0
 
-  - type: json_check
-    description: "report.json confirms HITL node inserted and validation passed"
-    path: "report.json"
-    assertions:
-      - expression: "validation_passed"
-        operator: equals
-        expected: true
-      - expression: "hitl_node_id"
-        operator: not_equals
-        expected: ""
-      - expression: "length(edges_after_insert)"
-        operator: gte
-        expected: 2
-    weight: 2.0
-    pass_threshold: 0.75
diff --git a/tests/tasks/uipath-maestro-flow/hitl/smoke_01_hitl_node_placed.yaml b/tests/tasks/uipath-maestro-flow/hitl/smoke_01_hitl_node_placed.yaml
index add7372b4..d27d143c1 100644
--- a/tests/tasks/uipath-maestro-flow/hitl/smoke_01_hitl_node_placed.yaml
+++ b/tests/tasks/uipath-maestro-flow/hitl/smoke_01_hitl_node_placed.yaml
@@ -29,16 +29,6 @@ initial_prompt: |
 
   Do NOT run flow debug. Validate the flow after building it.
 
-  **REQUIRED — before finishing, write report.json at the sandbox root** (not
-  inside any project or solution subfolder) with exactly these keys:
-  {
-    "flow_file": "<path to the .flow file>",
-    "hitl_node_id": "<id of the HITL node>",
-    "validation_passed": true
-  }
-  The task is NOT complete until `uip maestro flow validate` has passed AND
-  report.json exists at the sandbox root.
-
 success_criteria:
   - type: file_exists
     description: "Flow file was created"
@@ -70,15 +60,3 @@ success_criteria:
     weight: 3.0
     pass_threshold: 1.0
 
-  - type: json_check
-    description: "report.json captures hitl_node_id and validation_passed"
-    path: "report.json"
-    assertions:
-      - expression: "validation_passed"
-        operator: equals
-        expected: true
-      - expression: "hitl_node_id"
-        operator: not_equals
-        expected: ""
-    weight: 2.0
-    pass_threshold: 0.75
diff --git a/tests/tasks/uipath-maestro-flow/hitl/smoke_02_completed_port_wired.yaml b/tests/tasks/uipath-maestro-flow/hitl/smoke_02_completed_port_wired.yaml
index 07784f784..28b95f1b1 100644
--- a/tests/tasks/uipath-maestro-flow/hitl/smoke_02_completed_port_wired.yaml
+++ b/tests/tasks/uipath-maestro-flow/hitl/smoke_02_completed_port_wired.yaml
@@ -28,16 +28,6 @@ initial_prompt: |
 
   Do NOT run flow debug. Validate after building.
 
-  **REQUIRED — before finishing, write report.json at the sandbox root** (not
-  inside any project subfolder) with exactly these keys:
-  {
-    "hitl_node_id": "<id>",
-    "edge_from_hitl": "<sourceHandle used on the HITL edge>",
-    "validation_passed": true
-  }
-  The task is NOT complete until `uip maestro flow validate` has passed AND
-  report.json exists.
-
 success_criteria:
   - type: file_exists
     description: "Flow file was created"
@@ -69,10 +59,3 @@ success_criteria:
     weight: 3.0
     pass_threshold: 1.0
 
-  - type: file_contains
-    description: "report.json records completed as the edge handle"
-    path: "report.json"
-    includes:
-      - "completed"
-    weight: 1.5
-    pass_threshold: 1.0
diff --git a/tests/tasks/uipath-maestro-flow/hitl/smoke_03_multi_outcome_routing.yaml b/tests/tasks/uipath-maestro-flow/hitl/smoke_03_multi_outcome_routing.yaml
index 7c13352e0..bc6e4e8e3 100644
--- a/tests/tasks/uipath-maestro-flow/hitl/smoke_03_multi_outcome_routing.yaml
+++ b/tests/tasks/uipath-maestro-flow/hitl/smoke_03_multi_outcome_routing.yaml
@@ -35,14 +35,6 @@ initial_prompt: |
 
   Do NOT run flow debug. Validate after building.
 
-  Save a summary to report.json:
-  {
-    "hitl_node_id": "<id>",
-    "decision_node_id": "<id of the Decision node>",
-    "decision_expression": "<full expression used, e.g. $vars.hitl1.result.approved>",
-    "validation_passed": true
-  }
-
 success_criteria:
   - type: run_command
     description: "uip maestro flow validate passes"
@@ -86,21 +78,3 @@ success_criteria:
     weight: 2.0
     pass_threshold: 1.0
 
-  - type: json_check
-    description: "report.json captures node IDs and decision expression"
-    path: "report.json"
-    assertions:
-      - expression: "validation_passed"
-        operator: equals
-        expected: true
-      - expression: "hitl_node_id"
-        operator: not_equals
-        expected: ""
-      - expression: "decision_node_id"
-        operator: not_equals
-        expected: ""
-      - expression: "decision_expression"
-        operator: contains
-        expected: "$vars."
-    weight: 2.0
-    pass_threshold: 0.75
diff --git a/tests/tasks/uipath-maestro-flow/smoke/init_validate.yaml b/tests/tasks/uipath-maestro-flow/smoke/init_validate.yaml
index 5fb054810..5feff3508 100644
--- a/tests/tasks/uipath-maestro-flow/smoke/init_validate.yaml
+++ b/tests/tasks/uipath-maestro-flow/smoke/init_validate.yaml
@@ -25,14 +25,6 @@ initial_prompt: |
   The task is NOT complete until `uip maestro flow validate` has passed for
   that exact file path.
 
-  **REQUIRED — before finishing, write report.json at the sandbox root** with
-  at minimum:
-    {
-      "project_name": "WeatherAlert",
-      "commands_used": ["<list of uip commands you ran>"],
-      "validation_passed": true
-    }
-
   Important:
   - The `uip` CLI is already available in the environment.
   - Do not run `uip maestro flow debug` — just validate locally.
@@ -84,18 +76,3 @@ success_criteria:
     weight: 1.5
     pass_threshold: 1.0
 
-  - type: json_check
-    description: "report.json has correct structure and values"
-    path: "report.json"
-    assertions:
-      - expression: "project_name"
-        operator: equals
-        expected: "WeatherAlert"
-      - expression: "validation_passed"
-        operator: equals
-        expected: true
-      - expression: "length(commands_used)"
-        operator: gte
-        expected: 3
-    weight: 2.0
-    pass_threshold: 0.75
diff --git a/tests/tasks/uipath-review/review_multi_project_solution.yaml b/tests/tasks/uipath-review/review_multi_project_solution.yaml
index c39619962..785e03c70 100644
--- a/tests/tasks/uipath-review/review_multi_project_solution.yaml
+++ b/tests/tasks/uipath-review/review_multi_project_solution.yaml
@@ -60,16 +60,6 @@ initial_prompt: |
     (Warning severity)
   - Have a "## Summary" section at the end
 
-  Save a structured summary to report.json:
-    {
-      "projects_reviewed": 2,
-      "commands_used": ["<list of uip commands you ran>"],
-      "severities_found": ["<unique severities found>"],
-      "findings_count": <int>,
-      "hardcoded_credential_flagged": <bool>,
-      "empty_catch_flagged": <bool>
-    }
-
   Important:
   - The `uip` CLI is already available in the environment.
   - Do not run `uip rpa debug` or any execution commands.
@@ -161,24 +151,3 @@ success_criteria:
     weight: 1.0
     pass_threshold: 1.0
 
-  - type: json_check
-    description: "report.json shows both planted issues were identified"
-    path: "report.json"
-    assertions:
-      - expression: "projects_reviewed"
-        operator: equals
-        expected: 2
-      - expression: "hardcoded_credential_flagged"
-        operator: equals
-        expected: true
-      - expression: "empty_catch_flagged"
-        operator: equals
-        expected: true
-      - expression: "findings_count"
-        operator: gte
-        expected: 2
-      - expression: "length(severities_found)"
-        operator: gte
-        expected: 2
-    weight: 2.5
-    pass_threshold: 0.8
diff --git a/tests/tasks/uipath-review/review_rpa_project.yaml b/tests/tasks/uipath-review/review_rpa_project.yaml
index bd1fa53f6..f60c20dc1 100644
--- a/tests/tasks/uipath-review/review_rpa_project.yaml
+++ b/tests/tasks/uipath-review/review_rpa_project.yaml
@@ -36,14 +36,6 @@ initial_prompt: |
   - Severity markers for each finding (Critical, Warning, or Info)
   - A "## Summary" section at the end
 
-  Save a structured summary of what you did to report.json:
-    {
-      "project_reviewed": "MyProject",
-      "commands_used": ["<list of uip commands you ran>"],
-      "severity_levels_used": ["<list of severities used in the report>"],
-      "findings_count": <int>
-    }
-
   Important:
   - The `uip` CLI is already available in the environment.
   - Do not run `uip rpa debug` or any execution commands.
@@ -100,18 +92,3 @@ success_criteria:
     weight: 1.0
     pass_threshold: 1.0
 
-  - type: json_check
-    description: "report.json has correct structure and values"
-    path: "report.json"
-    assertions:
-      - expression: "project_reviewed"
-        operator: equals
-        expected: "MyProject"
-      - expression: "length(commands_used)"
-        operator: gte
-        expected: 1
-      - expression: "length(severity_levels_used)"
-        operator: gte
-        expected: 1
-    weight: 1.5
-    pass_threshold: 0.75
diff --git a/tests/tasks/uipath-rpa/coded_test_case.yaml b/tests/tasks/uipath-rpa/coded_test_case.yaml
index 8d42edf94..ae76ec555 100644
--- a/tests/tasks/uipath-rpa/coded_test_case.yaml
+++ b/tests/tasks/uipath-rpa/coded_test_case.yaml
@@ -42,20 +42,6 @@ initial_prompt: |
   - Update project.json so the test case shows up in fileInfoCollection
     with the correct testCaseType entry.
 
-  Write a `report.json` IMMEDIATELY after scaffolding the project (with
-  placeholder values) and update it as you complete each piece. Do not
-  leave it as the last step — previous runs got distracted by build
-  errors and never wrote it. Final shape:
-    {
-      "hooks_file": "CodedWorkflowHooks.cs",
-      "hooks_pattern": "partial_class",
-      "hooks_class_declaration": "<the class declaration line>",
-      "test_case_file": "TestLoginSuccess.cs",
-      "test_case_class": "TestLoginSuccess",
-      "base_class": "CodedWorkflow",
-      "assertions_used": ["<list of testing.Verify* methods used>"]
-    }
-
   Important:
   - The `uip` CLI is already available in the environment.
   - Use `--output json` on any uip commands you run.
@@ -111,24 +97,3 @@ success_criteria:
     weight: 1.5
     pass_threshold: 1.0
 
-  - type: file_exists
-    description: "report.json was created"
-    path: "report.json"
-    weight: 1.0
-    pass_threshold: 1.0
-
-  - type: json_check
-    description: "report.json confirms partial-class hooks + CodedWorkflow base class"
-    path: "report.json"
-    assertions:
-      - expression: "hooks_pattern"
-        operator: equals
-        expected: "partial_class"
-      - expression: "hooks_class_declaration"
-        operator: contains
-        expected: "partial class CodedWorkflow"
-      - expression: "base_class"
-        operator: contains
-        expected: "CodedWorkflow"
-    weight: 2.0
-    pass_threshold: 1.0
diff --git a/tests/tasks/uipath-solution-design/gap_detection.yaml b/tests/tasks/uipath-solution-design/gap_detection.yaml
index c121d8988..571dfb61d 100644
--- a/tests/tasks/uipath-solution-design/gap_detection.yaml
+++ b/tests/tasks/uipath-solution-design/gap_detection.yaml
@@ -24,14 +24,6 @@ initial_prompt: |
   as the skill instructs. For any [SME REVIEW] items, keep them as-is in the
   SDD — do NOT resolve them.
 
-  After writing the SDD, save a summary to report.json:
-    {
-      "sdd_filename": "<the filename you wrote>",
-      "primary_product": "<the product you selected>",
-      "default_count": <number of [DEFAULT] markers in the SDD>,
-      "sme_review_count": <number of [SME REVIEW] markers in the SDD>
-    }
-
   --- BEGIN PDD ---
 
   # Process Design Document: Payment Reconciliation
@@ -106,24 +98,3 @@ success_criteria:
     weight: 1.5
     pass_threshold: 1.0
 
-  - type: file_exists
-    description: "report.json was created"
-    path: "report.json"
-    weight: 1.0
-    pass_threshold: 1.0
-
-  - type: json_check
-    description: "report.json shows both DEFAULT and SME REVIEW markers were used"
-    path: "report.json"
-    assertions:
-      - expression: "default_count"
-        operator: gte
-        expected: 1
-      - expression: "sme_review_count"
-        operator: gte
-        expected: 1
-      - expression: "process_name"
-        operator: equals
-        expected: "Payment Reconciliation"
-    weight: 2.0
-    pass_threshold: 0.75
diff --git a/tests/tasks/uipath-solution-design/pdd_to_sdd.yaml b/tests/tasks/uipath-solution-design/pdd_to_sdd.yaml
index 77d9ae7ce..329133acf 100644
--- a/tests/tasks/uipath-solution-design/pdd_to_sdd.yaml
+++ b/tests/tasks/uipath-solution-design/pdd_to_sdd.yaml
@@ -21,14 +21,6 @@ initial_prompt: |
 
   For any [SME REVIEW] items, use the defaults and proceed.
 
-  After writing the SDD, save a summary to report.json:
-    {
-      "sdd_filename": "<the filename you wrote>",
-      "primary_product": "<the product you selected>",
-      "section_count": <number of top-level sections>,
-      "process_name": "Invoice Data Entry"
-    }
-
   --- BEGIN PDD ---
 
   # Process Design Document: Invoice Data Entry
@@ -128,25 +120,6 @@ success_criteria:
     weight: 1.0
     pass_threshold: 1.0
 
-  - type: file_exists
-    description: "report.json was created"
-    path: "report.json"
-    weight: 1.0
-    pass_threshold: 1.0
-
-  - type: json_check
-    description: "report.json confirms RPA product selection and correct process name"
-    path: "report.json"
-    assertions:
-      - expression: "process_name"
-        operator: equals
-        expected: "Invoice Data Entry"
-      - expression: "section_count"
-        operator: gte
-        expected: 10
-    weight: 1.5
-    pass_threshold: 0.75
-
   - type: file_contains
     description: "SDD has implementation plan with task structure"
     path: "invoice-data-entry-sdd.md"
diff --git a/tests/tasks/uipath-solution-design/product_selection.yaml b/tests/tasks/uipath-solution-design/product_selection.yaml
index 792da6e75..23f9729a6 100644
--- a/tests/tasks/uipath-solution-design/product_selection.yaml
+++ b/tests/tasks/uipath-solution-design/product_selection.yaml
@@ -22,14 +22,6 @@ initial_prompt: |
 
   For any [SME REVIEW] items, use the defaults and proceed.
 
-  After writing the SDD, save a summary to report.json:
-    {
-      "sdd_filename": "<the filename you wrote>",
-      "primary_product": "<the product you selected>",
-      "framework": "<agent framework if applicable>",
-      "process_name": "Customer Support Agent"
-    }
-
   --- BEGIN PDD ---
 
   # Process Design Document: Customer Support Agent
@@ -109,19 +101,6 @@ success_criteria:
     weight: 1.0
     pass_threshold: 1.0
 
-  - type: json_check
-    description: "report.json confirms Agents as primary product"
-    path: "report.json"
-    assertions:
-      - expression: "primary_product"
-        operator: contains
-        expected: "Agent"
-      - expression: "process_name"
-        operator: equals
-        expected: "Customer Support Agent"
-    weight: 2.0
-    pass_threshold: 1.0
-
   - type: file_contains
     description: "SDD does NOT use RPA template sections"
     path: "customer-support-agent-sdd.md"
diff --git a/tests/tasks/uipath-tasks/e2e_fetch_tasks.yaml b/tests/tasks/uipath-tasks/e2e_fetch_tasks.yaml
index a4f9be7ce..87e4dd493 100644
--- a/tests/tasks/uipath-tasks/e2e_fetch_tasks.yaml
+++ b/tests/tasks/uipath-tasks/e2e_fetch_tasks.yaml
@@ -44,27 +44,10 @@ initial_prompt: |
           `uip tasks list --folder-id <folder-id> --output json`.
 
      If NO tasks exist:
-       a. Skip 3a–3e. Set the corresponding report fields to null.
+       a. Skip 3a–3e.
        b. Still run `uip tasks list --as-admin --output json` to exercise the
           elevated path.
 
-  Save a report to report.json:
-  {
-    "tenant_url": "<UIPATH_URL from login status>",
-    "organization": "<Organization from login status>",
-    "tenant": "<Tenant from login status>",
-    "tasks_total": <total count from step 2>,
-    "first_task_id": <numeric id, or null if no tasks>,
-    "first_task_type": "<type, or null>",
-    "first_task_status": "<status, or null>",
-    "folder_id": <numeric folder id used for scoped queries, or null>,
-    "type_hint_used": <true if step 3c executed, false otherwise>,
-    "users_count": <number from step 3d, or null>,
-    "folder_tasks_count": <number from step 3e, or null>,
-    "commands_used": ["<list of uip commands you ran, in order>"],
-    "mutations_attempted": false
-  }
-
   Do NOT ask for approval or confirmation.
   Do NOT pause between steps.
   Do NOT call `tasks assign`, `tasks reassign`, `tasks unassign`, or
@@ -129,35 +112,3 @@ success_criteria:
     weight: 1.0
     pass_threshold: 1.0
 
-  - type: file_exists
-    description: "report.json was created"
-    path: "report.json"
-    weight: 1.0
-    pass_threshold: 1.0
-
-  - type: json_check
-    description: "Report confirms a real-tenant fetch completed end-to-end"
-    path: "report.json"
-    assertions:
-      - expression: "tasks_total"
-        operator: gte
-        expected: 0
-      - expression: "type_hint_used"
-        operator: equals
-        expected: true
-      - expression: "mutations_attempted"
-        operator: equals
-        expected: false
-      - expression: "length(commands_used)"
-        operator: gte
-        expected: 5
-    weight: 3.0
-    pass_threshold: 0.75
-
-  - type: file_contains
-    description: "Report captured a tenant URL from login status"
-    path: "report.json"
-    includes:
-      - "uipath.com"
-    weight: 1.0
-    pass_threshold: 1.0
diff --git a/tests/tasks/uipath-tasks/smoke_assignment.yaml b/tests/tasks/uipath-tasks/smoke_assignment.yaml
index baa594ea8..eef3285dd 100644
--- a/tests/tasks/uipath-tasks/smoke_assignment.yaml
+++ b/tests/tasks/uipath-tasks/smoke_assignment.yaml
@@ -25,16 +25,6 @@ initial_prompt: |
   4. Reassign task 1111111 to bob@company.com.
   5. Unassign task 2222222.
 
-  Save a summary to report.json:
-  {
-    "commands_used": ["<list of uip tasks commands in order>"],
-    "assign_by_email": <true>,
-    "assign_by_user_id": <true>,
-    "reassigned": <true>,
-    "unassigned": <true>,
-    "used_mutually_exclusive_flags": <true if you used --user OR --user-id per call, never both>
-  }
-
   Important:
   - --user and --user-id are mutually exclusive. Do not combine them on the
     same command.
@@ -90,24 +80,3 @@ success_criteria:
     weight: 1.0
     pass_threshold: 1.0
 
-  - type: file_exists
-    description: "report.json was created"
-    path: "report.json"
-    weight: 1.0
-    pass_threshold: 1.0
-
-  - type: json_check
-    description: "Report confirms both --user and --user-id variants used, mutually exclusive"
-    path: "report.json"
-    assertions:
-      - expression: "assign_by_email"
-        operator: equals
-        expected: true
-      - expression: "assign_by_user_id"
-        operator: equals
-        expected: true
-      - expression: "used_mutually_exclusive_flags"
-        operator: equals
-        expected: true
-    weight: 2.0
-    pass_threshold: 1.0
diff --git a/tests/tasks/uipath-tasks/smoke_completion.yaml b/tests/tasks/uipath-tasks/smoke_completion.yaml
index 29fcfef20..3c78b4ffb 100644
--- a/tests/tasks/uipath-tasks/smoke_completion.yaml
+++ b/tests/tasks/uipath-tasks/smoke_completion.yaml
@@ -35,24 +35,6 @@ initial_prompt: |
     - --type <TaskType>
     - --folder-id 123
 
-  Save a summary to report.json:
-  {
-    "external_task_complete": {
-      "has_action": <true or false>,
-      "has_data": <true or false>
-    },
-    "form_task_complete": {
-      "has_action": <true>,
-      "has_data": <true>,
-      "action_value": "Approve"
-    },
-    "app_task_complete": {
-      "has_action": <true>,
-      "has_data": <true>
-    },
-    "all_calls_include_type_and_folder": <true>
-  }
-
   Important:
   - The CLI is not connected to a live tenant. Auth errors are expected.
     Run each command once and move on. Do NOT retry or login.
@@ -99,33 +81,3 @@ success_criteria:
     weight: 2.0
     pass_threshold: 1.0
 
-  - type: file_exists
-    description: "report.json was created"
-    path: "report.json"
-    weight: 1.0
-    pass_threshold: 1.0
-
-  - type: json_check
-    description: "FormTask and AppTask completions include action + data; ExternalTask has type+folder"
-    path: "report.json"
-    assertions:
-      - expression: "form_task_complete.has_action"
-        operator: equals
-        expected: true
-      - expression: "form_task_complete.has_data"
-        operator: equals
-        expected: true
-      - expression: "form_task_complete.action_value"
-        operator: equals
-        expected: "Approve"
-      - expression: "app_task_complete.has_action"
-        operator: equals
-        expected: true
-      - expression: "app_task_complete.has_data"
-        operator: equals
-        expected: true
-      - expression: "all_calls_include_type_and_folder"
-        operator: equals
-        expected: true
-    weight: 3.0
-    pass_threshold: 1.0
diff --git a/tests/tasks/uipath-tasks/smoke_discovery.yaml b/tests/tasks/uipath-tasks/smoke_discovery.yaml
index 664aacd04..522efbe85 100644
--- a/tests/tasks/uipath-tasks/smoke_discovery.yaml
+++ b/tests/tasks/uipath-tasks/smoke_discovery.yaml
@@ -27,14 +27,6 @@ initial_prompt: |
      to the faster forms endpoint.
   6. List assignable users for folder ID 123.
 
-  Save a summary to report.json:
-  {
-    "commands_used": ["<list of uip tasks commands you ran in order>"],
-    "used_type_hint": <true if step 5 included --task-type AND --folder-id>,
-    "used_as_admin": <true if step 3 included --as-admin>,
-    "task_id_style": "<'numeric' or 'guid'>"
-  }
-
   Important:
   - The uip CLI is NOT connected to a live Action Center tenant. Commands
     will fail with auth errors — that is expected. Run each command once,
@@ -99,24 +91,3 @@ success_criteria:
     weight: 1.0
     pass_threshold: 1.0
 
-  - type: file_exists
-    description: "report.json was created"
-    path: "report.json"
-    weight: 1.0
-    pass_threshold: 1.0
-
-  - type: json_check
-    description: "Report confirms numeric task IDs and type-hint usage"
-    path: "report.json"
-    assertions:
-      - expression: "task_id_style"
-        operator: equals
-        expected: "numeric"
-      - expression: "used_type_hint"
-        operator: equals
-        expected: true
-      - expression: "used_as_admin"
-        operator: equals
-        expected: true
-    weight: 2.0
-    pass_threshold: 1.0

From cf4a45743f5db2ba5b58d7824587cd368606287f Mon Sep 17 00:00:00 2001
From: Bai Li <bai.li@uipath.com>
Date: Thu, 30 Apr 2026 15:59:20 -0700
Subject: [PATCH 02/12] data fabric checks

---
 .../e2e_employee_directory.yaml               |  8 ++++++
 .../e2e_product_catalogue.yaml                | 22 ++++++++++++++-
 .../integration_csv_import.yaml               | 16 +++++++++++
 .../uipath-data-fabric/integration_files.yaml | 28 +++++++++++++++++++
 .../integration_large_pagination.yaml         | 18 ++++++++++++
 5 files changed, 91 insertions(+), 1 deletion(-)

diff --git a/tests/tasks/uipath-data-fabric/e2e_employee_directory.yaml b/tests/tasks/uipath-data-fabric/e2e_employee_directory.yaml
index aa08ccf4f..5d231a53f 100644
--- a/tests/tasks/uipath-data-fabric/e2e_employee_directory.yaml
+++ b/tests/tasks/uipath-data-fabric/e2e_employee_directory.yaml
@@ -122,6 +122,14 @@ success_criteria:
     weight: 1.0
     pass_threshold: 1.0
 
+  - type: run_command
+    description: "employees.csv has 20 data rows (header + 20)"
+    command: "awk 'END { exit (NR == 21 ? 0 : 1) }' employees.csv"
+    timeout: 5
+    expected_exit_code: 0
+    weight: 2.0
+    pass_threshold: 1.0
+
 post_run:
   - command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-data-fabric/_shared/cleanup_entities.py"
     timeout: 60
diff --git a/tests/tasks/uipath-data-fabric/e2e_product_catalogue.yaml b/tests/tasks/uipath-data-fabric/e2e_product_catalogue.yaml
index d088f5c5a..d7e1717dc 100644
--- a/tests/tasks/uipath-data-fabric/e2e_product_catalogue.yaml
+++ b/tests/tasks/uipath-data-fabric/e2e_product_catalogue.yaml
@@ -25,7 +25,7 @@ initial_prompt: |
   Price (DECIMAL), InStock (BOOLEAN), SKU (STRING), Poster (FILE). Create the
   entity with all six fields before importing. Seed it with 15 products
   (5 per category: Electronics / Clothing / Home) by generating and importing
-  a CSV (CSV does not include the Poster column). Then:
+  a CSV named `products.csv` (CSV does not include the Poster column). Then:
 
   - Query all Electronics products and confirm you get exactly 5.
   - Apply a 10% price increase to the 2 most expensive products across all
@@ -104,6 +104,26 @@ success_criteria:
     weight: 2.5
     pass_threshold: 1.0
 
+  - type: file_exists
+    description: "products.csv was generated"
+    path: "products.csv"
+    weight: 1.0
+    pass_threshold: 1.0
+
+  - type: run_command
+    description: "products.csv has 15 data rows (header + 15)"
+    command: "awk 'END { exit (NR == 16 ? 0 : 1) }' products.csv"
+    timeout: 5
+    expected_exit_code: 0
+    weight: 2.0
+    pass_threshold: 1.0
+
+  - type: file_exists
+    description: "product_hero.jpg placeholder image was created"
+    path: "product_hero.jpg"
+    weight: 1.0
+    pass_threshold: 1.0
+
 post_run:
   - command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-data-fabric/_shared/cleanup_entities.py"
     timeout: 60
diff --git a/tests/tasks/uipath-data-fabric/integration_csv_import.yaml b/tests/tasks/uipath-data-fabric/integration_csv_import.yaml
index 4ae0d31a0..bd1c5bf28 100644
--- a/tests/tasks/uipath-data-fabric/integration_csv_import.yaml
+++ b/tests/tasks/uipath-data-fabric/integration_csv_import.yaml
@@ -96,6 +96,22 @@ success_criteria:
     weight: 2.5
     pass_threshold: 1.0
 
+  - type: run_command
+    description: "inventory.csv has at least 4 data rows (header + 4)"
+    command: "awk 'END { exit (NR >= 5 ? 0 : 1) }' inventory.csv"
+    timeout: 5
+    expected_exit_code: 0
+    weight: 2.0
+    pass_threshold: 1.0
+
+  - type: run_command
+    description: "inventory.csv header row excludes system fields"
+    command: "awk 'NR==1 && /(^|,)(Id|CreatedBy|CreateTime|UpdatedBy|UpdateTime)(,|$)/ { exit 1 }' inventory.csv"
+    timeout: 5
+    expected_exit_code: 0
+    weight: 2.0
+    pass_threshold: 1.0
+
 post_run:
   - command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-data-fabric/_shared/cleanup_entities.py"
     timeout: 60
diff --git a/tests/tasks/uipath-data-fabric/integration_files.yaml b/tests/tasks/uipath-data-fabric/integration_files.yaml
index 697a246a2..f3be547e5 100644
--- a/tests/tasks/uipath-data-fabric/integration_files.yaml
+++ b/tests/tasks/uipath-data-fabric/integration_files.yaml
@@ -112,6 +112,34 @@ success_criteria:
     weight: 1.5
     pass_threshold: 1.0
 
+  - type: file_exists
+    description: "updated.txt (v2 source) was created"
+    path: "updated.txt"
+    weight: 1.0
+    pass_threshold: 1.0
+
+  - type: file_exists
+    description: "verify_update.txt (v2 download) was created"
+    path: "verify_update.txt"
+    weight: 1.0
+    pass_threshold: 1.0
+
+  - type: run_command
+    description: "v1 byte integrity: downloaded.txt is identical to original.txt"
+    command: "cmp -s original.txt downloaded.txt"
+    timeout: 5
+    expected_exit_code: 0
+    weight: 3.0
+    pass_threshold: 1.0
+
+  - type: run_command
+    description: "v2 byte integrity: verify_update.txt is identical to updated.txt"
+    command: "cmp -s updated.txt verify_update.txt"
+    timeout: 5
+    expected_exit_code: 0
+    weight: 3.0
+    pass_threshold: 1.0
+
 post_run:
   - command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-data-fabric/_shared/cleanup_entities.py"
     timeout: 60
diff --git a/tests/tasks/uipath-data-fabric/integration_large_pagination.yaml b/tests/tasks/uipath-data-fabric/integration_large_pagination.yaml
index 8fb4fbae2..92656297a 100644
--- a/tests/tasks/uipath-data-fabric/integration_large_pagination.yaml
+++ b/tests/tasks/uipath-data-fabric/integration_large_pagination.yaml
@@ -106,6 +106,24 @@ success_criteria:
     weight: 1.0
     pass_threshold: 1.0
 
+  - type: run_command
+    description: "bulk_500.csv has exactly 500 data rows (header + 500)"
+    command: "awk 'END { exit (NR == 501 ? 0 : 1) }' bulk_500.csv"
+    timeout: 5
+    expected_exit_code: 0
+    weight: 2.0
+    pass_threshold: 1.0
+
+  - type: file_contains
+    description: "bulk_500.csv has the required header columns"
+    path: "bulk_500.csv"
+    includes:
+      - "Name"
+      - "Score"
+      - "Active"
+    weight: 1.5
+    pass_threshold: 1.0
+
 post_run:
   - command: "python3 $SKILLS_REPO_PATH/tests/tasks/uipath-data-fabric/_shared/cleanup_entities.py"
     timeout: 60

From aa76c9bfd0ab5983ed7f9ec5d937ca4f940b7c2c Mon Sep 17 00:00:00 2001
From: Bai Li <bai.li@uipath.com>
Date: Thu, 30 Apr 2026 16:18:41 -0700
Subject: [PATCH 03/12] hard cases

---
 .../smoke_negative_guards.yaml                |   7 ++
 .../e2e_01_invoice_approval_greenfield.yaml   | 105 +++++++++---------
 .../e2e_03_gdpr_compliance_greenfield.yaml    |  95 ++++++++--------
 .../quality_01_approval_gate_schema.yaml      |   7 --
 .../smoke_01_explicit.yaml                    |  53 +--------
 .../smoke_02_approval_gate.yaml               |  38 +------
 .../smoke_03_escalation.yaml                  |  48 +-------
 .../smoke_04_writeback.yaml                   |  40 +------
 .../smoke_05_compliance.yaml                  |  47 +-------
 .../smoke_06_data_enrichment.yaml             |  36 +-----
 .../smoke_07_neg_automated.yaml               |  21 +---
 .../smoke_08_neg_admin.yaml                   |  23 +---
 .../smoke/registry_discovery.yaml             |  50 ++-------
 .../activity_discovery.yaml                   |  25 +----
 .../connection_lifecycle.yaml                 |  27 +----
 .../connector_discovery.yaml                  |  24 +---
 .../resource_describe.yaml                    |  24 +---
 .../integration-service/resource_execute.yaml |  29 +----
 .../uipath-rpa/template_aware_create.yaml     |  42 +++----
 .../uipath-tasks/smoke_negative_guards.yaml   |   7 ++
 20 files changed, 186 insertions(+), 562 deletions(-)

diff --git a/tests/tasks/uipath-data-fabric/smoke_negative_guards.yaml b/tests/tasks/uipath-data-fabric/smoke_negative_guards.yaml
index 00d070229..e9c17c362 100644
--- a/tests/tasks/uipath-data-fabric/smoke_negative_guards.yaml
+++ b/tests/tasks/uipath-data-fabric/smoke_negative_guards.yaml
@@ -46,6 +46,13 @@ initial_prompt: |
   - For each task, clearly state whether you are refusing or proceeding and why.
 
 success_criteria:
+  - type: skill_triggered
+    description: "Agent loaded the uipath-data-fabric skill before reasoning about the requests"
+    skill_name: "uipath-data-fabric"
+    expected: "yes"
+    weight: 2.0
+    pass_threshold: 1.0
+
   - type: file_exists
     description: "report.json was created"
     path: "report.json"
diff --git a/tests/tasks/uipath-human-in-the-loop/e2e_01_invoice_approval_greenfield.yaml b/tests/tasks/uipath-human-in-the-loop/e2e_01_invoice_approval_greenfield.yaml
index 68e4dd5a6..bd78b8793 100644
--- a/tests/tasks/uipath-human-in-the-loop/e2e_01_invoice_approval_greenfield.yaml
+++ b/tests/tasks/uipath-human-in-the-loop/e2e_01_invoice_approval_greenfield.yaml
@@ -19,61 +19,44 @@ sandbox:
   python: {}
 
 initial_prompt: |
-  Build a UiPath Flow that extracts invoice data from a SharePoint folder
-  and posts the approved invoices to SAP. Finance needs to review and
-  approve each invoice before it is posted — we cannot write to SAP
-  without human sign-off.
+  Build a UiPath Flow named "InvoiceApproval" that extracts invoice data
+  from a SharePoint folder and posts the approved invoices to SAP. Finance
+  needs to review and approve each invoice before it is posted — we cannot
+  write to SAP without human sign-off.
 
-  Save a summary to report.json:
-  {
-    "project_name": "<name you used>",
-    "hitl_node_id": "<id of the HITL node>",
-    "schema": {
-      "inputs": ["<fields shown to finance manager>"],
-      "outputs": ["<fields manager fills in>"],
-      "outcomes": ["<approval buttons>"]
-    },
-    "handles_wired": ["<list of handles you wired>"],
-    "validation_passed": true,
-    "result_variable": "<how downstream nodes access the decision>"
-  }
+  The HITL node must use the inline quickform style (uipath.human-in-the-loop).
+  Show the finance manager the relevant invoice fields, give them Approve and
+  Reject outcomes, and wire the completed handle to the downstream SAP step.
+  Use $vars.<hitl-node-id>.result downstream to access the manager's decision.
+
+  Validate the flow when you're done.
 
 success_criteria:
+  - type: file_exists
+    description: "Flow file was created at the expected path"
+    path: "InvoiceApproval/InvoiceApproval/InvoiceApproval.flow"
+    weight: 1.5
+    pass_threshold: 1.0
+
   - type: file_contains
-    description: "report.json contains a HITL node ID (non-placeholder)"
-    path: "report.json"
+    description: "Flow contains the inline HITL node type"
+    path: "InvoiceApproval/InvoiceApproval/InvoiceApproval.flow"
     includes:
-      - '"hitl_node_id": "'
-    weight: 2.0
+      - '"uipath.human-in-the-loop"'
+    weight: 3.0
     pass_threshold: 1.0
 
-  - type: command_executed
-    description: "Agent validated the flow"
-    tool_name: "Bash"
-    command_pattern: '(uip|\$UIP)\s+(maestro\s+)?flow\s+validate.*--output\s+json'
-    min_count: 1
-    weight: 2.0
+  - type: file_contains
+    description: "Flow uses the quickform schema type"
+    path: "InvoiceApproval/InvoiceApproval/InvoiceApproval.flow"
+    includes:
+      - '"type": "quick"'
+    weight: 1.5
     pass_threshold: 1.0
 
-  - type: json_check
-    description: "Schema contains invoice-relevant inputs and Approve/Reject outcomes"
-    path: "report.json"
-    assertions:
-      - expression: "length(schema.inputs)"
-        operator: gte
-        expected: 2
-      - expression: "length(schema.outcomes)"
-        operator: gte
-        expected: 2
-      - expression: "validation_passed"
-        operator: equals
-        expected: true
-    weight: 3.0
-    pass_threshold: 0.75
-
   - type: file_contains
-    description: "Schema outcomes include Approve and Reject"
-    path: "report.json"
+    description: "Flow exposes Approve and Reject outcomes"
+    path: "InvoiceApproval/InvoiceApproval/InvoiceApproval.flow"
     includes:
       - "Approve"
       - "Reject"
@@ -81,18 +64,32 @@ success_criteria:
     pass_threshold: 1.0
 
   - type: file_contains
-    description: "report.json includes completed in wired handles"
-    path: "report.json"
+    description: "Flow wires the completed handle"
+    path: "InvoiceApproval/InvoiceApproval/InvoiceApproval.flow"
     includes:
-      - "completed"
+      - '"completed"'
     weight: 1.5
     pass_threshold: 1.0
 
-  - type: file_contains
-    description: "Agent captured the runtime result variable"
-    path: "report.json"
-    includes:
-      - "$vars."
-      - ".result"
+  - type: file_matches_regex
+    description: "Downstream step references $vars.<hitl-node-id>.result"
+    path: "InvoiceApproval/InvoiceApproval/InvoiceApproval.flow"
+    pattern: '\$vars\.[A-Za-z0-9_-]+\.result'
+    weight: 1.5
+    pass_threshold: 1.0
+
+  - type: command_executed
+    description: "Agent ran flow validate during the session"
+    tool_name: "Bash"
+    command_pattern: '(uip|\$UIP)\s+(maestro\s+)?flow\s+validate.*--output\s+json'
+    min_count: 1
     weight: 1.5
     pass_threshold: 1.0
+
+  - type: run_command
+    description: "uip flow validate passes against the authored .flow file"
+    command: "uip maestro flow validate InvoiceApproval/InvoiceApproval/InvoiceApproval.flow --output json"
+    timeout: 30
+    expected_exit_code: 0
+    weight: 3.0
+    pass_threshold: 1.0
diff --git a/tests/tasks/uipath-human-in-the-loop/e2e_03_gdpr_compliance_greenfield.yaml b/tests/tasks/uipath-human-in-the-loop/e2e_03_gdpr_compliance_greenfield.yaml
index df9db8354..837904807 100644
--- a/tests/tasks/uipath-human-in-the-loop/e2e_03_gdpr_compliance_greenfield.yaml
+++ b/tests/tasks/uipath-human-in-the-loop/e2e_03_gdpr_compliance_greenfield.yaml
@@ -20,53 +20,38 @@ sandbox:
   python: {}
 
 initial_prompt: |
-  Build a UiPath Flow that processes GDPR data deletion requests. Each
-  request must be reviewed and signed off by a privacy officer before the
-  deletion runs — we need an audit trail for every decision. If the officer
-  doesn't respond within 7 days, escalate automatically.
+  Build a UiPath Flow named "GdprDeletionApproval" that processes GDPR
+  data deletion requests. Each request must be reviewed and signed off by
+  a privacy officer before the deletion runs — we need an audit trail for
+  every decision. If the officer doesn't respond within 7 days, escalate
+  automatically.
 
-  Save results to report.json:
-  {
-    "project_name": "<name used>",
-    "hitl_node_id": "<id>",
-    "schema": {
-      "inputs": ["<what the privacy officer sees>"],
-      "outputs": ["<what they fill in>"],
-      "outcomes": ["<their choices>"]
-    },
-    "timeout_configured": "<ISO 8601 duration used>",
-    "handles_wired": ["<list of handles wired>"],
-    "validation_passed": true
-  }
+  The HITL node must use the inline quickform style (uipath.human-in-the-loop).
+  Give the privacy officer Approve and Reject outcomes. Configure the node
+  with a 7-day ISO 8601 timeout (e.g. "P7D") and wire the timeout handle
+  separately from the completed handle so the escalation path is reachable
+  on timeout.
+
+  Validate the flow when you're done.
 
 success_criteria:
-  - type: file_contains
-    description: "report.json contains a HITL node ID (non-placeholder)"
-    path: "report.json"
-    includes:
-      - '"hitl_node_id": "'
-    weight: 2.0
+  - type: file_exists
+    description: "Flow file was created at the expected path"
+    path: "GdprDeletionApproval/GdprDeletionApproval/GdprDeletionApproval.flow"
+    weight: 1.5
     pass_threshold: 1.0
 
   - type: file_contains
-    description: "Timeout duration is captured in report.json (ISO 8601)"
-    path: "report.json"
+    description: "Flow contains the inline HITL node type"
+    path: "GdprDeletionApproval/GdprDeletionApproval/GdprDeletionApproval.flow"
     includes:
-      - '"timeout_configured": "P'
-    weight: 2.0
-    pass_threshold: 1.0
-
-  - type: command_executed
-    description: "Agent validated the flow"
-    tool_name: "Bash"
-    command_pattern: '(uip|\$UIP)\s+(maestro\s+)?flow\s+validate.*--output\s+json'
-    min_count: 1
-    weight: 2.0
+      - '"uipath.human-in-the-loop"'
+    weight: 3.0
     pass_threshold: 1.0
 
   - type: file_contains
-    description: "Schema includes Approve and Reject outcomes"
-    path: "report.json"
+    description: "Flow exposes Approve and Reject outcomes"
+    path: "GdprDeletionApproval/GdprDeletionApproval/GdprDeletionApproval.flow"
     includes:
       - "Approve"
       - "Reject"
@@ -74,19 +59,33 @@ success_criteria:
     pass_threshold: 1.0
 
   - type: file_contains
-    description: "Timeout handle is wired"
-    path: "report.json"
+    description: "Flow wires the completed and timeout handles"
+    path: "GdprDeletionApproval/GdprDeletionApproval/GdprDeletionApproval.flow"
     includes:
-      - "timeout"
-    weight: 1.5
+      - '"completed"'
+      - '"timeout"'
+    weight: 2.0
     pass_threshold: 1.0
 
-  - type: json_check
-    description: "report.json confirms validation passed"
-    path: "report.json"
-    assertions:
-      - expression: "validation_passed"
-        operator: equals
-        expected: true
+  - type: file_matches_regex
+    description: "Flow configures a 7-day ISO 8601 timeout (P7D or PT168H or similar)"
+    path: "GdprDeletionApproval/GdprDeletionApproval/GdprDeletionApproval.flow"
+    pattern: '"P(7D|T168H|1W)"'
     weight: 2.0
     pass_threshold: 1.0
+
+  - type: command_executed
+    description: "Agent ran flow validate during the session"
+    tool_name: "Bash"
+    command_pattern: '(uip|\$UIP)\s+(maestro\s+)?flow\s+validate.*--output\s+json'
+    min_count: 1
+    weight: 1.5
+    pass_threshold: 1.0
+
+  - type: run_command
+    description: "uip flow validate passes against the authored .flow file"
+    command: "uip maestro flow validate GdprDeletionApproval/GdprDeletionApproval/GdprDeletionApproval.flow --output json"
+    timeout: 30
+    expected_exit_code: 0
+    weight: 3.0
+    pass_threshold: 1.0
diff --git a/tests/tasks/uipath-human-in-the-loop/quality_01_approval_gate_schema.yaml b/tests/tasks/uipath-human-in-the-loop/quality_01_approval_gate_schema.yaml
index f9e07f2ad..9b2e4dbab 100644
--- a/tests/tasks/uipath-human-in-the-loop/quality_01_approval_gate_schema.yaml
+++ b/tests/tasks/uipath-human-in-the-loop/quality_01_approval_gate_schema.yaml
@@ -57,11 +57,4 @@ success_criteria:
     weight: 1.5
     pass_threshold: 1.0
 
-  - type: file_contains
-    description: "confirmed field is false — agent did not run CLI prematurely"
-    path: "schema_proposal.json"
-    includes:
-      - '"confirmed": false'
-    weight: 2.0
-    pass_threshold: 1.0
 
diff --git a/tests/tasks/uipath-human-in-the-loop/smoke_01_explicit.yaml b/tests/tasks/uipath-human-in-the-loop/smoke_01_explicit.yaml
index f81874f8f..b52217ec5 100644
--- a/tests/tasks/uipath-human-in-the-loop/smoke_01_explicit.yaml
+++ b/tests/tasks/uipath-human-in-the-loop/smoke_01_explicit.yaml
@@ -13,53 +13,10 @@ initial_prompt: |
   I have a UiPath Flow. Add a Human-in-the-Loop node before the final data
   write step so a manager can review and approve the data before it is posted.
 
-  Write a recommendation.json file with:
-  {
-    "hitl_needed": <true or false>,
-    "pattern": "<which business pattern applies>",
-    "proposed_schema": {
-      "inputs": ["<field names the human will see>"],
-      "outputs": ["<field names the human fills in>"],
-      "outcomes": ["<action button names>"]
-    }
-  }
-
 success_criteria:
-  - type: file_exists
-    description: "Agent wrote a recommendation.json"
-    path: "recommendation.json"
-    weight: 1.0
-    pass_threshold: 1.0
-
-  - type: file_contains
-    description: "Agent identified HITL is needed"
-    path: "recommendation.json"
-    includes:
-      - '"hitl_needed": true'
-    weight: 2.0
-    pass_threshold: 1.0
-
-  - type: json_check
-    description: "Agent named an approval or write-back pattern (case-tolerant)"
-    path: "recommendation.json"
-    assertions:
-      - expression: "pattern"
-        operator: contains
-        expected: "pprov"
-      - expression: "pattern"
-        operator: contains
-        expected: "rite"
-      - expression: "pattern"
-        operator: contains
-        expected: "alid"
-    weight: 1.5
-    pass_threshold: 0.33
-
-  - type: file_contains
-    description: "Agent proposed a schema with outcomes"
-    path: "recommendation.json"
-    includes:
-      - "outcomes"
-      - "inputs"
-    weight: 1.5
+  - type: skill_triggered
+    description: "Agent invoked the uipath-human-in-the-loop skill"
+    skill_name: "uipath-human-in-the-loop"
+    expected: "yes"
+    weight: 3.0
     pass_threshold: 1.0
diff --git a/tests/tasks/uipath-human-in-the-loop/smoke_02_approval_gate.yaml b/tests/tasks/uipath-human-in-the-loop/smoke_02_approval_gate.yaml
index bb9d943b2..b407e570e 100644
--- a/tests/tasks/uipath-human-in-the-loop/smoke_02_approval_gate.yaml
+++ b/tests/tasks/uipath-human-in-the-loop/smoke_02_approval_gate.yaml
@@ -14,38 +14,10 @@ initial_prompt: |
   finance — but a manager must approve each expense report before the email
   is sent.
 
-  Write a recommendation.json file with:
-  {
-    "hitl_needed": <true or false>,
-    "pattern": "<which business pattern applies>",
-    "proposed_schema": {
-      "inputs": ["<field names>"],
-      "outputs": ["<field names>"],
-      "outcomes": ["<button names>"]
-    }
-  }
-
 success_criteria:
-  - type: file_exists
-    description: "Agent wrote a recommendation.json"
-    path: "recommendation.json"
-    weight: 1.0
-    pass_threshold: 1.0
-
-  - type: file_contains
-    description: "Agent identified HITL is needed"
-    path: "recommendation.json"
-    includes:
-      - '"hitl_needed": true'
-    weight: 2.0
-    pass_threshold: 1.0
-
-  - type: json_check
-    description: "Agent identified an approval gate pattern (case-tolerant)"
-    path: "recommendation.json"
-    assertions:
-      - expression: "pattern"
-        operator: contains
-        expected: "pprov"
-    weight: 1.5
+  - type: skill_triggered
+    description: "Agent invoked the uipath-human-in-the-loop skill"
+    skill_name: "uipath-human-in-the-loop"
+    expected: "yes"
+    weight: 3.0
     pass_threshold: 1.0
diff --git a/tests/tasks/uipath-human-in-the-loop/smoke_03_escalation.yaml b/tests/tasks/uipath-human-in-the-loop/smoke_03_escalation.yaml
index 97a0abf9c..03b0eb1c1 100644
--- a/tests/tasks/uipath-human-in-the-loop/smoke_03_escalation.yaml
+++ b/tests/tasks/uipath-human-in-the-loop/smoke_03_escalation.yaml
@@ -14,48 +14,10 @@ initial_prompt: |
   the agent confidence is below 70%, a human needs to step in and make the
   final call before the complaint is routed.
 
-  Write a recommendation.json file with:
-  {
-    "hitl_needed": <true or false>,
-    "pattern": "<which business pattern applies>",
-    "insertion_point": "<where in the flow the HITL node should go>",
-    "proposed_schema": {
-      "inputs": ["<field names>"],
-      "outcomes": ["<button names>"]
-    }
-  }
-
 success_criteria:
-  - type: file_exists
-    description: "Agent wrote a recommendation.json"
-    path: "recommendation.json"
-    weight: 1.0
-    pass_threshold: 1.0
-
-  - type: file_contains
-    description: "Agent identified HITL is needed"
-    path: "recommendation.json"
-    includes:
-      - '"hitl_needed": true'
-    weight: 2.0
-    pass_threshold: 1.0
-
-  - type: json_check
-    description: "Agent identified an escalation pattern"
-    path: "recommendation.json"
-    assertions:
-      - expression: "pattern"
-        operator: contains
-        expected: "scalat"
-    weight: 1.5
-    pass_threshold: 1.0
-
-  - type: json_check
-    description: "Agent proposed an insertion point after the AI classifier (case-tolerant)"
-    path: "recommendation.json"
-    assertions:
-      - expression: "insertion_point"
-        operator: contains
-        expected: "lassif"
-    weight: 1.0
+  - type: skill_triggered
+    description: "Agent invoked the uipath-human-in-the-loop skill"
+    skill_name: "uipath-human-in-the-loop"
+    expected: "yes"
+    weight: 3.0
     pass_threshold: 1.0
diff --git a/tests/tasks/uipath-human-in-the-loop/smoke_04_writeback.yaml b/tests/tasks/uipath-human-in-the-loop/smoke_04_writeback.yaml
index 5a47dbb7d..784a876bd 100644
--- a/tests/tasks/uipath-human-in-the-loop/smoke_04_writeback.yaml
+++ b/tests/tasks/uipath-human-in-the-loop/smoke_04_writeback.yaml
@@ -14,42 +14,12 @@ initial_prompt: |
   enriches the missing vendor and cost-center fields using company data, and
   writes the corrected records back to SAP.
 
-  Analyze whether this flow needs any human checkpoints. Write a
-  recommendation.json file with:
-  {
-    "hitl_needed": <true or false>,
-    "pattern": "<pattern name if applicable>",
-    "reason": "<why HITL is or is not needed>",
-    "proposed_insertion_point": "<where in the flow>"
-  }
+  Analyze whether this flow needs any human checkpoints.
 
 success_criteria:
-  - type: file_exists
-    description: "Agent wrote a recommendation.json"
-    path: "recommendation.json"
-    weight: 1.0
-    pass_threshold: 1.0
-
-  - type: file_contains
-    description: "Agent proactively identified HITL is needed (write-back detection)"
-    path: "recommendation.json"
-    includes:
-      - '"hitl_needed": true'
+  - type: skill_triggered
+    description: "Agent invoked the uipath-human-in-the-loop skill"
+    skill_name: "uipath-human-in-the-loop"
+    expected: "yes"
     weight: 3.0
     pass_threshold: 1.0
-
-  - type: json_check
-    description: "Agent named a write-back / validation / enrichment pattern (any one, case-tolerant)"
-    path: "recommendation.json"
-    assertions:
-      - expression: "pattern"
-        operator: contains
-        expected: "rite"
-      - expression: "pattern"
-        operator: contains
-        expected: "alid"
-      - expression: "pattern"
-        operator: contains
-        expected: "nrich"
-    weight: 1.5
-    pass_threshold: 0.33
diff --git a/tests/tasks/uipath-human-in-the-loop/smoke_05_compliance.yaml b/tests/tasks/uipath-human-in-the-loop/smoke_05_compliance.yaml
index 0b6e430a3..cced1c5ad 100644
--- a/tests/tasks/uipath-human-in-the-loop/smoke_05_compliance.yaml
+++ b/tests/tasks/uipath-human-in-the-loop/smoke_05_compliance.yaml
@@ -13,47 +13,10 @@ initial_prompt: |
   sign-off from our data privacy officer before the deletion actually runs —
   we need an audit trail for every decision.
 
-  Write a recommendation.json file with:
-  {
-    "hitl_needed": <true or false>,
-    "pattern": "<which business pattern applies>",
-    "proposed_schema": {
-      "inputs": ["<what the privacy officer will see>"],
-      "outputs": ["<what they fill in>"],
-      "outcomes": ["<their decision options>"]
-    }
-  }
-
 success_criteria:
-  - type: file_exists
-    description: "Agent wrote a recommendation.json"
-    path: "recommendation.json"
-    weight: 1.0
-    pass_threshold: 1.0
-
-  - type: file_contains
-    description: "Agent identified HITL is needed"
-    path: "recommendation.json"
-    includes:
-      - '"hitl_needed": true'
-    weight: 2.0
+  - type: skill_triggered
+    description: "Agent invoked the uipath-human-in-the-loop skill"
+    skill_name: "uipath-human-in-the-loop"
+    expected: "yes"
+    weight: 3.0
     pass_threshold: 1.0
-
-  - type: json_check
-    description: "Agent identified a compliance / audit / sign-off / approval pattern (any one, case-tolerant)"
-    path: "recommendation.json"
-    assertions:
-      - expression: "pattern"
-        operator: contains
-        expected: "udit"
-      - expression: "pattern"
-        operator: contains
-        expected: "omplianc"
-      - expression: "pattern"
-        operator: contains
-        expected: "ign"
-      - expression: "pattern"
-        operator: contains
-        expected: "pprov"
-    weight: 1.5
-    pass_threshold: 0.25
diff --git a/tests/tasks/uipath-human-in-the-loop/smoke_06_data_enrichment.yaml b/tests/tasks/uipath-human-in-the-loop/smoke_06_data_enrichment.yaml
index ad8115c23..867bb7cbb 100644
--- a/tests/tasks/uipath-human-in-the-loop/smoke_06_data_enrichment.yaml
+++ b/tests/tasks/uipath-human-in-the-loop/smoke_06_data_enrichment.yaml
@@ -14,36 +14,10 @@ initial_prompt: |
   records to a human to fill in the missing data before the invoice is
   posted to the accounting system.
 
-  Write a recommendation.json file with:
-  {
-    "hitl_needed": <true or false>,
-    "pattern": "<which business pattern applies>",
-    "proposed_schema": {
-      "inputs": ["<what the human sees — already extracted fields>"],
-      "inOuts": ["<fields the human fills in and returns>"],
-      "outcomes": ["<button names>"]
-    }
-  }
-
 success_criteria:
-  - type: file_exists
-    description: "Agent wrote a recommendation.json"
-    path: "recommendation.json"
-    weight: 1.0
-    pass_threshold: 1.0
-
-  - type: file_contains
-    description: "Agent identified HITL is needed"
-    path: "recommendation.json"
-    includes:
-      - '"hitl_needed": true'
-    weight: 2.0
-    pass_threshold: 1.0
-
-  - type: file_contains
-    description: "Agent used inOuts (not just outputs) for the fill-in fields"
-    path: "recommendation.json"
-    includes:
-      - "inOut"
-    weight: 1.5
+  - type: skill_triggered
+    description: "Agent invoked the uipath-human-in-the-loop skill"
+    skill_name: "uipath-human-in-the-loop"
+    expected: "yes"
+    weight: 3.0
     pass_threshold: 1.0
diff --git a/tests/tasks/uipath-human-in-the-loop/smoke_07_neg_automated.yaml b/tests/tasks/uipath-human-in-the-loop/smoke_07_neg_automated.yaml
index d31f339d3..60f3f8dba 100644
--- a/tests/tasks/uipath-human-in-the-loop/smoke_07_neg_automated.yaml
+++ b/tests/tasks/uipath-human-in-the-loop/smoke_07_neg_automated.yaml
@@ -15,23 +15,12 @@ initial_prompt: |
   invoice follows the same template: extract 5 fixed fields, validate against
   PO numbers, post to SAP. There are no exceptions and no edge cases — the
   business has signed off on full automation with no human review steps for
-  this supplier. Create a recommendation.json with:
-  {
-    "hitl_needed": <true or false>,
-    "reason": "<why>"
-  }
+  this supplier.
 
 success_criteria:
-  - type: file_exists
-    description: "Agent wrote a recommendation.json"
-    path: "recommendation.json"
-    weight: 1.0
-    pass_threshold: 1.0
-
-  - type: file_contains
-    description: "Agent correctly determined HITL is NOT needed"
-    path: "recommendation.json"
-    includes:
-      - '"hitl_needed": false'
+  - type: skill_triggered
+    description: "Agent did NOT invoke the uipath-human-in-the-loop skill (HITL not appropriate here)"
+    skill_name: "uipath-human-in-the-loop"
+    expected: "no"
     weight: 3.0
     pass_threshold: 1.0
diff --git a/tests/tasks/uipath-human-in-the-loop/smoke_08_neg_admin.yaml b/tests/tasks/uipath-human-in-the-loop/smoke_08_neg_admin.yaml
index e9e0fd081..ed388e30e 100644
--- a/tests/tasks/uipath-human-in-the-loop/smoke_08_neg_admin.yaml
+++ b/tests/tasks/uipath-human-in-the-loop/smoke_08_neg_admin.yaml
@@ -13,23 +13,10 @@ initial_prompt: |
   How do I reassign a pending Action Center task that has been sitting for
   3 days without a response to a different user in our team?
 
-  Write an answer.json file with:
-  {
-    "answer": "<your guidance on how to reassign Action Center tasks>",
-    "hitl_authoring_needed": <true or false>
-  }
-
 success_criteria:
-  - type: file_exists
-    description: "Agent wrote an answer.json"
-    path: "answer.json"
-    weight: 1.0
-    pass_threshold: 1.0
-
-  - type: file_contains
-    description: "Agent did not recommend HITL authoring"
-    path: "answer.json"
-    includes:
-      - '"hitl_authoring_needed": false'
-    weight: 2.0
+  - type: skill_triggered
+    description: "Agent did NOT invoke the uipath-human-in-the-loop skill (HITL not appropriate here)"
+    skill_name: "uipath-human-in-the-loop"
+    expected: "no"
+    weight: 3.0
     pass_threshold: 1.0
diff --git a/tests/tasks/uipath-maestro-flow/smoke/registry_discovery.yaml b/tests/tasks/uipath-maestro-flow/smoke/registry_discovery.yaml
index fb1278216..be649652b 100644
--- a/tests/tasks/uipath-maestro-flow/smoke/registry_discovery.yaml
+++ b/tests/tasks/uipath-maestro-flow/smoke/registry_discovery.yaml
@@ -18,18 +18,10 @@ initial_prompt: |
   response with a script. Before building anything, explore what node types
   are available in the Flow registry and find the right ones for this task.
 
-  Save your findings to registry_report.json with at minimum:
-    {
-      "node_types_found": ["<list of relevant node types>"],
-      "commands_used": ["<list of uip registry commands you ran>"],
-      "http_node_type": "<the node type for HTTP requests>",
-      "script_node_type": "<the node type for scripts>"
-    }
-
   Important:
   - The `uip` CLI is already available in the environment.
   - Use `--output json` on all uip commands.
-  - Do not build the flow — just explore and report.
+  - Do not build the flow — just explore.
 
 success_criteria:
   - type: command_executed
@@ -56,36 +48,18 @@ success_criteria:
     weight: 1.0
     pass_threshold: 1.0
 
-  - type: file_exists
-    description: "registry_report.json exists"
-    path: "registry_report.json"
-    weight: 1.0
-    pass_threshold: 1.0
-
-  - type: run_command
-    description: "registry_report.json is valid JSON"
-    command: "python -c \"import json; json.load(open('registry_report.json'))\""
-    timeout: 10
-    expected_exit_code: 0
-    weight: 1.0
-    pass_threshold: 1.0
-
-  - type: file_contains
-    description: "Report contains expected fields"
-    path: "registry_report.json"
-    includes:
-      - "node_types_found"
-      - "commands_used"
-      - "http_node_type"
-      - "script_node_type"
+  - type: command_executed
+    description: "Agent retrieved details on the HTTP node type via registry get"
+    tool_name: "Bash"
+    command_pattern: 'uip\s+(maestro\s+)?flow\s+registry\s+get\s+.*core\.action\.http'
+    min_count: 1
     weight: 1.5
     pass_threshold: 1.0
 
-  - type: file_contains
-    description: "Report identifies the correct HTTP and script node types"
-    path: "registry_report.json"
-    includes:
-      - "core.action.http"
-      - "core.action.script"
-    weight: 1.0
+  - type: command_executed
+    description: "Agent retrieved details on the script node type via registry get"
+    tool_name: "Bash"
+    command_pattern: 'uip\s+(maestro\s+)?flow\s+registry\s+get\s+.*core\.action\.script'
+    min_count: 1
+    weight: 1.5
     pass_threshold: 1.0
diff --git a/tests/tasks/uipath-platform/integration-service/activity_discovery.yaml b/tests/tasks/uipath-platform/integration-service/activity_discovery.yaml
index 40ac727eb..07490d0f4 100644
--- a/tests/tasks/uipath-platform/integration-service/activity_discovery.yaml
+++ b/tests/tasks/uipath-platform/integration-service/activity_discovery.yaml
@@ -23,20 +23,10 @@ initial_prompt: |
   2. List the trigger activities (events) available
   3. Summarize the difference between activities, triggers, and resources
 
-  Save your findings to report.json:
-  {
-    "connector_key": "uipath-salesforce-sfdc",
-    "activity_count": <number of non-trigger activities, or 0 if auth error>,
-    "trigger_count": <number of trigger activities, or 0 if auth error>,
-    "activities_vs_triggers_vs_resources": "<brief explanation>",
-    "commands_used": ["<list of uip is commands you ran>"]
-  }
-
   Important:
   - The `uip` CLI is available but is NOT connected to a live tenant in this
     environment. Commands may fail with auth errors — that is expected.
-    Run each command once, record the result (success or error) in report.json,
-    and move on. Do NOT retry or attempt to login.
+    Run each command once and move on. Do NOT retry or attempt to login.
   - Use `--output json` on every uip command.
 
 success_criteria:
@@ -64,16 +54,3 @@ success_criteria:
     weight: 1.0
     pass_threshold: 1.0
 
-  - type: file_exists
-    description: "report.json was created"
-    path: "report.json"
-    weight: 1.0
-    pass_threshold: 1.0
-
-  - type: run_command
-    description: "report.json has correct structure with activity/trigger counts and explanation"
-    command: "python3 $TASK_DIR/_shared/is_check.py check_activity_discovery"
-    timeout: 30
-    expected_exit_code: 0
-    weight: 2.0
-    pass_threshold: 1.0
diff --git a/tests/tasks/uipath-platform/integration-service/connection_lifecycle.yaml b/tests/tasks/uipath-platform/integration-service/connection_lifecycle.yaml
index a4dd7d483..c51b00755 100644
--- a/tests/tasks/uipath-platform/integration-service/connection_lifecycle.yaml
+++ b/tests/tasks/uipath-platform/integration-service/connection_lifecycle.yaml
@@ -28,28 +28,16 @@ initial_prompt: |
      connection needs re-authentication (do NOT run these — they require
      a browser)
 
-  Save your findings to report.json:
-  {
-    "connector_key": "uipath-salesforce-sfdc",
-    "connections_found": <number of connections found or 0 if auth error>,
-    "ping_result": "<ping status or error message>",
-    "create_command": "<exact uip command to create a new connection>",
-    "edit_command": "<exact uip command to re-auth an existing connection>",
-    "commands_used": ["<list of uip is commands you actually ran>"]
-  }
-
   Important:
   - The `uip` CLI is available but is NOT connected to a live tenant in this
     environment. Commands WILL fail with auth errors — that is expected.
-    You MUST still run every command listed above (list and ping), record
-    each result (success or error) in report.json, and move on.
+    You MUST still run every command listed above (list and ping).
     Do NOT skip a command just because a previous one failed.
     Do NOT retry or attempt to login.
   - Use `--output json` on every uip command.
   - If connections list returns no connections, use a placeholder like
     "placeholder-conn-id" to still run the ping command with correct syntax.
   - Do NOT run connections create or connections edit — they open a browser.
-    Just document the correct commands in report.json.
 
 success_criteria:
   - type: command_executed
@@ -76,16 +64,3 @@ success_criteria:
     weight: 1.0
     pass_threshold: 1.0
 
-  - type: file_exists
-    description: "report.json was created"
-    path: "report.json"
-    weight: 1.0
-    pass_threshold: 1.0
-
-  - type: run_command
-    description: "report.json has correct structure and documents create/edit commands"
-    command: "python3 $TASK_DIR/_shared/is_check.py check_connection_lifecycle"
-    timeout: 30
-    expected_exit_code: 0
-    weight: 2.0
-    pass_threshold: 1.0
diff --git a/tests/tasks/uipath-platform/integration-service/connector_discovery.yaml b/tests/tasks/uipath-platform/integration-service/connector_discovery.yaml
index c659f5c10..8d77ca752 100644
--- a/tests/tasks/uipath-platform/integration-service/connector_discovery.yaml
+++ b/tests/tasks/uipath-platform/integration-service/connector_discovery.yaml
@@ -23,19 +23,10 @@ initial_prompt: |
   1. Search for matching connectors
   2. If no native connector exists, explain the fallback strategy
 
-  Save your findings to report.json:
-  {
-    "google_connectors": ["<list of connector keys found>"],
-    "apify_connectors": ["<list of connector keys found, or empty>"],
-    "apify_fallback": "<fallback strategy if no native connector>",
-    "commands_used": ["<list of uip is commands you ran>"]
-  }
-
   Important:
   - The `uip` CLI is available but is NOT connected to a live tenant in this
     environment. Commands may fail with auth errors — that is expected.
-    Run each command once, record the result (success or error) in report.json,
-    and move on. Do NOT retry or attempt to login.
+    Run each command once and move on. Do NOT retry or attempt to login.
   - Use `--output json` on every uip command.
 
 success_criteria:
@@ -55,16 +46,3 @@ success_criteria:
     weight: 1.0
     pass_threshold: 1.0
 
-  - type: file_exists
-    description: "report.json was created"
-    path: "report.json"
-    weight: 1.0
-    pass_threshold: 1.0
-
-  - type: run_command
-    description: "report.json has correct structure, HTTP fallback documented"
-    command: "python3 $TASK_DIR/_shared/is_check.py check_connector_discovery"
-    timeout: 30
-    expected_exit_code: 0
-    weight: 2.0
-    pass_threshold: 1.0
diff --git a/tests/tasks/uipath-platform/integration-service/resource_describe.yaml b/tests/tasks/uipath-platform/integration-service/resource_describe.yaml
index 5dc30b497..a7fb423cd 100644
--- a/tests/tasks/uipath-platform/integration-service/resource_describe.yaml
+++ b/tests/tasks/uipath-platform/integration-service/resource_describe.yaml
@@ -24,20 +24,11 @@ initial_prompt: |
   2. Describe the "Contact" resource to get field details for the Create
      operation (use --operation Create)
 
-  Save your findings to report.json:
-  {
-    "connector_key": "uipath-salesforce-sfdc",
-    "resources_found": <number of resources supporting Create, or 0 if auth error>,
-    "contact_fields": ["<list of field names from describe, or empty if auth error>"],
-    "required_fields": ["<list of required field names, or empty>"],
-    "commands_used": ["<list of uip is commands you ran>"]
-  }
-
   Important:
   - The `uip` CLI is available but is NOT connected to a live tenant in this
     environment. Commands WILL fail with auth errors — that is expected.
     You MUST still run every command listed above (resources list and
-    resources describe), record each result in report.json, and move on.
+    resources describe).
     Do NOT skip a command just because a previous one failed.
     Do NOT retry or attempt to login.
   - Use `--output json` on every uip command.
@@ -78,16 +69,3 @@ success_criteria:
     weight: 1.0
     pass_threshold: 1.0
 
-  - type: file_exists
-    description: "report.json was created"
-    path: "report.json"
-    weight: 1.0
-    pass_threshold: 1.0
-
-  - type: run_command
-    description: "report.json has correct structure with fields and required_fields"
-    command: "python3 $TASK_DIR/_shared/is_check.py check_resource_describe"
-    timeout: 30
-    expected_exit_code: 0
-    weight: 2.0
-    pass_threshold: 1.0
diff --git a/tests/tasks/uipath-platform/integration-service/resource_execute.yaml b/tests/tasks/uipath-platform/integration-service/resource_execute.yaml
index b72f38e84..6c4c9e8c0 100644
--- a/tests/tasks/uipath-platform/integration-service/resource_execute.yaml
+++ b/tests/tasks/uipath-platform/integration-service/resource_execute.yaml
@@ -19,24 +19,10 @@ initial_prompt: |
   Walk me through creating a Contact in Salesforce via UiPath Integration
   Service. Follow the full Integration Service workflow.
 
-  Save a step-by-step log to report.json:
-  {
-    "workflow_steps": [
-      {"step": 1, "action": "find_connector", "command": "<uip command>", "result": "<success or error>"},
-      {"step": 2, "action": "find_connection", "command": "<uip command>", "result": "<success or error>"},
-      {"step": 3, "action": "ping_connection", "command": "<uip command>", "result": "<success or error>"},
-      {"step": 4, "action": "discover_resources", "command": "<uip command>", "result": "<success or error>"},
-      {"step": 5, "action": "describe_resource", "command": "<uip command>", "result": "<success or error>"},
-      {"step": 6, "action": "execute_create", "command": "<uip command>", "result": "<success or error>"}
-    ],
-    "commands_used": ["<all uip commands run in order>"]
-  }
-
   Important:
   - The `uip` CLI is available but is NOT connected to a live tenant in this
     environment. Commands will fail with auth errors — that is expected.
-    Run each command once, record the result (success or error) in report.json,
-    and move on. Do NOT retry or attempt to login.
+    Run each command once and move on. Do NOT retry or attempt to login.
   - Use `--output json` on every uip command.
   - For the execute step, construct a realistic --body with at least
     LastName (required) and FirstName fields.
@@ -90,16 +76,3 @@ success_criteria:
     weight: 1.0
     pass_threshold: 1.0
 
-  - type: file_exists
-    description: "report.json was created"
-    path: "report.json"
-    weight: 1.0
-    pass_threshold: 1.0
-
-  - type: run_command
-    description: "report.json logs all 6 workflow steps with correct structure"
-    command: "python3 $TASK_DIR/_shared/is_check.py check_resource_execute"
-    timeout: 30
-    expected_exit_code: 0
-    weight: 2.0
-    pass_threshold: 1.0
diff --git a/tests/tasks/uipath-rpa/template_aware_create.yaml b/tests/tasks/uipath-rpa/template_aware_create.yaml
index 5e63ed2b7..e761beef7 100644
--- a/tests/tasks/uipath-rpa/template_aware_create.yaml
+++ b/tests/tasks/uipath-rpa/template_aware_create.yaml
@@ -75,40 +75,32 @@ initial_prompt: |
   - Do NOT invoke `uip rpa create-project` itself in this sandbox.
 
 success_criteria:
-  - type: file_exists
-    description: "report.json was created"
-    path: "report.json"
-    weight: 1.0
+  - type: command_executed
+    description: "Agent ran search-templates with a REFramework query"
+    tool_name: "Bash"
+    command_pattern: 'uip\s+rpa\s+search-templates.*--query\s+["'']?REFramework'
+    min_count: 1
+    weight: 2.0
     pass_threshold: 1.0
 
-  - type: json_check
-    description: "Agent picked template_package strategy (not BlankTemplate) for REFramework"
+  - type: file_matches_regex
+    description: "Agent picked template_package strategy with the official REFramework package"
     path: "report.json"
-    assertions:
-      - expression: "template_strategy"
-        operator: equals
-        expected: "template_package"
-      - expression: "template_package_id"
-        operator: equals
-        expected: "UiPath.Template.REFramework"
+    pattern: '"template_strategy"\s*:\s*"template_package".*"template_package_id"\s*:\s*"UiPath\.Template\.REFramework"'
     weight: 2.0
     pass_threshold: 1.0
 
-  - type: json_check
-    description: "Agent ran search-templates with a REFramework query before creating"
+  - type: file_matches_regex
+    description: "create_project command uses --template-package-id and not BlankTemplate"
     path: "report.json"
-    assertions:
-      - expression: "search_templates_query"
-        operator: contains
-        expected: "REFramework"
-    weight: 1.0
+    pattern: '"create_project_command"\s*:\s*"[^"]*--template-package-id\s+UiPath\.Template\.REFramework[^"]*"'
+    weight: 2.0
     pass_threshold: 1.0
 
   - type: file_contains
-    description: "create_project command uses --template-package-id, not --template-id BlankTemplate"
+    description: "Agent did NOT pick BlankTemplate as the template-id"
     path: "report.json"
-    includes:
-      - "--template-package-id"
-      - "UiPath.Template.REFramework"
-    weight: 1.5
+    excludes:
+      - "BlankTemplate"
+    weight: 1.0
     pass_threshold: 1.0
diff --git a/tests/tasks/uipath-tasks/smoke_negative_guards.yaml b/tests/tasks/uipath-tasks/smoke_negative_guards.yaml
index 54faedf1c..02f5ce3e8 100644
--- a/tests/tasks/uipath-tasks/smoke_negative_guards.yaml
+++ b/tests/tasks/uipath-tasks/smoke_negative_guards.yaml
@@ -49,6 +49,13 @@ initial_prompt: |
     the specific skill rule that applies.
 
 success_criteria:
+  - type: skill_triggered
+    description: "Agent loaded the uipath-tasks skill before reasoning about the requests"
+    skill_name: "uipath-tasks"
+    expected: "yes"
+    weight: 2.0
+    pass_threshold: 1.0
+
   - type: file_exists
     description: "report.json was created"
     path: "report.json"

From 12fa5a4407e1e53a5e77802a778701d1a2323bea Mon Sep 17 00:00:00 2001
From: Bai Li <bai.li@uipath.com>
Date: Thu, 30 Apr 2026 16:33:17 -0700
Subject: [PATCH 04/12] negative guards, update readme

---
 CONTRIBUTING.md                               |   1 +
 tests/README.md                               | 136 ++++++++++++++----
 .../smoke_negative_guards.yaml                |  57 ++------
 .../uipath-tasks/smoke_negative_guards.yaml   |  69 +--------
 4 files changed, 127 insertions(+), 136 deletions(-)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 8ca6e46cb..4690e3adb 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -252,6 +252,7 @@ make test-uipath-maestro-flow  # run all tests for a specific skill
 3. Use minimal prompts — the goal is to test the skill's guidance quality, not hand-hold the agent
 4. Tag every task appropriately: `smoke`, `integration`, or `e2e`
 5. Follow the task ID pattern: `skill-<domain>-<capability>`
+6. **Do not score self-reports.** Don't ask the agent to write a `report.json` / `recommendation.json` summary and then have `success_criteria` read that file back — the agent can write any value. Score real artifacts (`.flow` content, generated CSVs), real operations (`run_command` re-executes a validation), or behavior signals (`command_executed`, `command_not_executed`, `skill_triggered`). See the "Anti-patterns to avoid" section in `tests/README.md`.
 
 See `tests/README.md` for the full task YAML template, success criteria reference, and examples from existing tests.
 
diff --git a/tests/README.md b/tests/README.md
index 682c927a7..149627bff 100644
--- a/tests/README.md
+++ b/tests/README.md
@@ -325,70 +325,152 @@ Verify a file was created in the sandbox. From `init_validate.yaml`:
 
 ### `file_contains`
 
-Verify a file contains expected strings. From `registry_discovery.yaml`:
+Verify a file contains (or excludes) expected strings. **Score real artifacts the agent produced — not files the agent self-reported into.** From `uipath-maestro-flow/hitl/smoke_01_hitl_node_placed.yaml`:
 
 ```yaml
 - type: file_contains
-  description: "Report contains expected fields"
-  path: "registry_report.json"
+  description: "Flow contains the inline HITL node type"
+  path: "InvoiceApproval/InvoiceApproval/InvoiceApproval.flow"
   includes:
-    - "node_types_found"
-    - "commands_used"
-    - "http_node_type"
-    - "script_node_type"
-  weight: 1.5
+    - '"uipath.human-in-the-loop"'
+  weight: 3.0
   pass_threshold: 1.0
 ```
 
+`excludes:` is also supported — useful for asserting a CSV header excludes system fields or that a generated command doesn't reference a deprecated flag.
+
 ### `json_check`
 
-Validate JSON file structure and values using JSONPath assertions. From `init_validate.yaml`:
+Validate JSON file structure and values using JMESPath assertions. **Score the agent's actual JSON output (a generated config, schema, or fixture) — not a self-described status report.** Counter-example to avoid:
 
 ```yaml
+# ❌ ANTIPATTERN — agent writes "validation_passed: true" to a summary file
+# and the criterion reads it back. The agent could lie. See "Anti-patterns" below.
 - type: json_check
-  description: "report.json has correct structure and values"
   path: "report.json"
   assertions:
-    - expression: "project_name"
-      operator: equals
-      expected: "WeatherAlert"
     - expression: "validation_passed"
       operator: equals
       expected: true
-    - expression: "length(commands_used)"
-      operator: gte
-      expected: 3
-  weight: 2.0
-  pass_threshold: 0.75   # at least 75% of assertions must pass
 ```
 
-Supported operators: `equals`, `gte`, `lte`, `gt`, `lt`, `contains`.
+✅ Better — verify the underlying state directly with `run_command`:
+
+```yaml
+- type: run_command
+  description: "uip flow validate passes against the authored .flow"
+  command: "uip maestro flow validate Project/Project/Project.flow --output json"
+  timeout: 30
+  expected_exit_code: 0
+```
+
+`json_check` is fine when the JSON file is itself the genuine artifact (e.g. a generated schema). Supported operators: `equals`, `gte`, `lte`, `gt`, `lt`, `contains`.
 
 ### `run_command`
 
-Execute an arbitrary shell command and check the exit code. From `registry_discovery.yaml`:
+Execute an arbitrary shell command and check the exit code. Use it for direct verification of state the agent created. From `uipath-data-fabric/integration_csv_import.yaml`:
+
+```yaml
+- type: run_command
+  description: "inventory.csv has at least 4 data rows (header + 4)"
+  command: "awk 'END { exit (NR >= 5 ? 0 : 1) }' inventory.csv"
+  timeout: 5
+  expected_exit_code: 0
+  weight: 2.0
+  pass_threshold: 1.0
+```
+
+Or byte-equality for upload/download round-trips:
 
 ```yaml
 - type: run_command
-  description: "registry_report.json is valid JSON"
-  command: "python -c \"import json; json.load(open('registry_report.json'))\""
-  timeout: 10
+  description: "Downloaded file is byte-identical to the original"
+  command: "cmp -s original.txt downloaded.txt"
+  timeout: 5
   expected_exit_code: 0
-  weight: 1.0
+```
+
+### `skill_triggered`
+
+Verify the agent invoked a Claude Code Skill tool. Useful for "did the agent recognize this scenario calls for skill X?" Supports positive (`expected: "yes"`) and negative (`expected: "no"`) assertions:
+
+```yaml
+- type: skill_triggered
+  description: "Agent invoked the uipath-human-in-the-loop skill"
+  skill_name: "uipath-human-in-the-loop"
+  expected: "yes"
+  weight: 3.0
   pass_threshold: 1.0
 ```
 
+Un-fakeable — the criterion inspects `turn_records.commands` directly. The negative form (`expected: "no"`) is the right primitive for smoke tests where the agent should NOT trigger a particular skill.
+
+### `command_not_executed`
+
+Counterpart to `command_executed`. Verifies the agent did NOT run a prohibited command. Use for refusal / negative-guard tests:
+
+```yaml
+- type: command_not_executed
+  description: "Agent must not delete an entity"
+  tool_name: "Bash"
+  command_pattern: 'uip\s+df\s+entities\s+delete'
+  weight: 3.0
+  pass_threshold: 1.0
+```
+
+Score is binary: 1.0 when matches ≤ `max_count` (default `0`), else 0.0. Empty `turn_records` → trivially passes.
+
+## Anti-patterns to avoid
+
+The most common failure mode for new tests is the **self-report antipattern**: the prompt asks the agent to write a summary file (`report.json`, `recommendation.json`, `summary.json`, …) and the success criteria reads that file back. The agent can write whatever it wants in the summary, so the test scores the agent's *claim* about what it did rather than what it actually did.
+
+```yaml
+# ❌ DO NOT WRITE TESTS LIKE THIS
+initial_prompt: |
+  ... do the work ...
+
+  Save a summary to report.json:
+  {
+    "validation_passed": <true or false>,
+    "records_inserted": <count>,
+    "import_succeeded": <true or false>
+  }
+
+success_criteria:
+  - type: json_check
+    path: "report.json"
+    assertions:
+      - expression: "validation_passed"
+        operator: equals
+        expected: true
+      - expression: "records_inserted"
+        operator: gte
+        expected: 4
+```
+
+A misbehaving agent passes this test by writing the right strings into `report.json` regardless of whether the underlying work happened. To verify behavior instead of self-description, use:
+
+| Want to verify… | Use this criterion |
+|---|---|
+| Agent invoked the right skill | `skill_triggered` (positive or negative) |
+| Agent ran the right CLI command | `command_executed` |
+| Agent did NOT run a prohibited command | `command_not_executed` |
+| A real artifact (`.flow`, `.csv`, `.md`) was produced correctly | `file_contains` / `file_matches_regex` on the artifact |
+| A real operation succeeded (validate, hash equality, row count) | `run_command` running the verification directly |
+
+`json_check` and `file_contains` on agent-written summaries are only OK when the file genuinely is the deliverable — e.g. a schema-design task where the agent's JSON output is *itself* the artifact under evaluation. In every other case, prefer one of the criteria above.
+
 ## Weight and Threshold Guidance
 
 **`weight`** controls how much a criterion contributes to the overall score. Use higher weights for the core behavior being tested:
 
 | Weight | When to use | Example from existing tests |
 |--------|-------------|---------------------------|
-| `1.0` | Supporting checks | `--output json` flag used, file is valid JSON |
+| `1.0` | Supporting checks | `--output json` flag used, presence of an auxiliary file |
 | `1.5` | Core behavior | `uip solution new` executed, `.flow` file created |
-| `2.0` | Critical validation | `report.json` has correct structure and values |
+| `2.0` | Important artifact content | `.flow` file contains the expected node type or handle wiring |
 | `3.0` | Primary artifact validity | `uip maestro flow validate` passes on the generated flow file |
-| `5.0–6.0` | End-to-end execution | Check script runs flow debug and verifies output correctness |
+| `5.0–6.0` | End-to-end execution | Check script runs `flow debug` and verifies output correctness |
 
 **`pass_threshold`** is the fraction of the criterion that must pass. For `json_check` with multiple assertions, `0.75` means 75% of assertions must pass. For most criteria, use `1.0` (all-or-nothing).
 
diff --git a/tests/tasks/uipath-data-fabric/smoke_negative_guards.yaml b/tests/tasks/uipath-data-fabric/smoke_negative_guards.yaml
index e9c17c362..0195eb120 100644
--- a/tests/tasks/uipath-data-fabric/smoke_negative_guards.yaml
+++ b/tests/tasks/uipath-data-fabric/smoke_negative_guards.yaml
@@ -31,16 +31,6 @@ initial_prompt: |
   Task 3: Insert a record into the "SalesforceContacts" entity (ID "sf-333-ent").
   This entity's Source is "Federated (Salesforce)".
 
-  Save a summary to report.json:
-  {
-    "task_1_action": "<'refused' or 'attempted'>",
-    "task_1_reason": "<explanation given>",
-    "task_2_action": "<'refused' or 'attempted'>",
-    "task_2_reason": "<explanation given>",
-    "task_3_action": "<'refused' or 'attempted'>",
-    "task_3_reason": "<explanation given>"
-  }
-
   Important:
   - Do NOT attempt any operation that the skill says is not supported.
   - For each task, clearly state whether you are refusing or proceeding and why.
@@ -53,46 +43,23 @@ success_criteria:
     weight: 2.0
     pass_threshold: 1.0
 
-  - type: file_exists
-    description: "report.json was created"
-    path: "report.json"
-    weight: 1.0
-    pass_threshold: 1.0
-
-  - type: json_check
-    description: "Agent refused entity delete (task 1)"
-    path: "report.json"
-    assertions:
-      - expression: "task_1_action"
-        operator: equals
-        expected: "refused"
+  - type: command_not_executed
+    description: "Agent must not delete an entity (task 1 prohibited operation)"
+    tool_name: "Bash"
+    command_pattern: 'uip\s+df\s+entities\s+delete'
     weight: 3.0
     pass_threshold: 1.0
 
-  - type: json_check
-    description: "Agent refused field removal via removeFields (task 2)"
-    path: "report.json"
-    assertions:
-      - expression: "task_2_action"
-        operator: equals
-        expected: "refused"
+  - type: command_not_executed
+    description: "Agent must not call removeFields in entities update (task 2 prohibited operation)"
+    tool_name: "Bash"
+    command_pattern: '(?s)uip\s+df\s+entities\s+update.*removeFields'
     weight: 3.0
     pass_threshold: 1.0
 
-  - type: json_check
-    description: "Agent refused write to federated entity (task 3)"
-    path: "report.json"
-    assertions:
-      - expression: "task_3_action"
-        operator: equals
-        expected: "refused"
+  - type: command_not_executed
+    description: "Agent must not insert records into the federated SalesforceContacts entity (task 3 prohibited operation)"
+    tool_name: "Bash"
+    command_pattern: 'uip\s+df\s+records\s+insert\s+["'']?sf-333-ent'
     weight: 3.0
     pass_threshold: 1.0
-
-  - type: file_contains
-    description: "Reasons reference skill constraints (not supported / federated / read-only)"
-    path: "report.json"
-    includes:
-      - "not supported"
-    weight: 1.5
-    pass_threshold: 1.0
diff --git a/tests/tasks/uipath-tasks/smoke_negative_guards.yaml b/tests/tasks/uipath-tasks/smoke_negative_guards.yaml
index 02f5ce3e8..520d12517 100644
--- a/tests/tasks/uipath-tasks/smoke_negative_guards.yaml
+++ b/tests/tasks/uipath-tasks/smoke_negative_guards.yaml
@@ -31,18 +31,6 @@ initial_prompt: |
 
   Task 4: Get task "abc-123-def-456" — the user passed a GUID-style ID.
 
-  Save a summary to report.json:
-  {
-    "task_1_action": "<'refused' or 'attempted'>",
-    "task_1_reason": "<why>",
-    "task_2_action": "<'refused' or 'attempted'>",
-    "task_2_reason": "<why>",
-    "task_3_action": "<'refused' or 'attempted'>",
-    "task_3_reason": "<why>",
-    "task_4_action": "<'refused' or 'attempted'>",
-    "task_4_reason": "<why>"
-  }
-
   Important:
   - Do NOT run any uip tasks commands. This test is purely about refusal.
   - For each task, clearly state whether you are refusing and reference
@@ -56,56 +44,9 @@ success_criteria:
     weight: 2.0
     pass_threshold: 1.0
 
-  - type: file_exists
-    description: "report.json was created"
-    path: "report.json"
-    weight: 1.0
-    pass_threshold: 1.0
-
-  - type: json_check
-    description: "Agent refused to create a task (not in CLI scope)"
-    path: "report.json"
-    assertions:
-      - expression: "task_1_action"
-        operator: equals
-        expected: "refused"
-    weight: 2.5
-    pass_threshold: 1.0
-
-  - type: json_check
-    description: "Agent refused to complete an already-Completed task"
-    path: "report.json"
-    assertions:
-      - expression: "task_2_action"
-        operator: equals
-        expected: "refused"
-    weight: 2.5
-    pass_threshold: 1.0
-
-  - type: json_check
-    description: "Agent refused to complete a FormTask without --action/--data"
-    path: "report.json"
-    assertions:
-      - expression: "task_3_action"
-        operator: equals
-        expected: "refused"
-    weight: 2.5
-    pass_threshold: 1.0
-
-  - type: json_check
-    description: "Agent refused to use a GUID task ID (Action Center uses numeric IDs)"
-    path: "report.json"
-    assertions:
-      - expression: "task_4_action"
-        operator: equals
-        expected: "refused"
-    weight: 2.5
-    pass_threshold: 1.0
-
-  - type: file_contains
-    description: "Reasons reference skill constraints (numeric IDs, action/data, already completed)"
-    path: "report.json"
-    includes:
-      - "numeric"
-    weight: 1.0
+  - type: command_not_executed
+    description: "Agent must not run any uip tasks commands (test is purely about refusal)"
+    tool_name: "Bash"
+    command_pattern: 'uip\s+tasks\s+'
+    weight: 5.0
     pass_threshold: 1.0

From 538fe9354f382e79b697125b1f83923cc388eec8 Mon Sep 17 00:00:00 2001
From: Bai Li <bai.li@uipath.com>
Date: Thu, 30 Apr 2026 16:40:44 -0700
Subject: [PATCH 05/12] fix typo

---
 tests/tasks/uipath-rpa/template_aware_create.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/tasks/uipath-rpa/template_aware_create.yaml b/tests/tasks/uipath-rpa/template_aware_create.yaml
index e761beef7..e2b81a821 100644
--- a/tests/tasks/uipath-rpa/template_aware_create.yaml
+++ b/tests/tasks/uipath-rpa/template_aware_create.yaml
@@ -97,7 +97,7 @@ success_criteria:
     weight: 2.0
     pass_threshold: 1.0
 
-  - type: file_contains
+  - type: file_check
     description: "Agent did NOT pick BlankTemplate as the template-id"
     path: "report.json"
     excludes:

From a268b6e180baeb89e303674f728a40c024f22238 Mon Sep 17 00:00:00 2001
From: Bai Li <bai.li@uipath.com>
Date: Thu, 30 Apr 2026 16:43:05 -0700
Subject: [PATCH 06/12] docs cleanup

---
 tests/README.md | 69 +++----------------------------------------------
 1 file changed, 3 insertions(+), 66 deletions(-)

diff --git a/tests/README.md b/tests/README.md
index 149627bff..0dcbc7902 100644
--- a/tests/README.md
+++ b/tests/README.md
@@ -325,7 +325,7 @@ Verify a file was created in the sandbox. From `init_validate.yaml`:
 
 ### `file_contains`
 
-Verify a file contains (or excludes) expected strings. **Score real artifacts the agent produced — not files the agent self-reported into.** From `uipath-maestro-flow/hitl/smoke_01_hitl_node_placed.yaml`:
+Verify a file contains (or excludes) expected strings. From `uipath-maestro-flow/hitl/smoke_01_hitl_node_placed.yaml`:
 
 ```yaml
 - type: file_contains
@@ -337,34 +337,11 @@ Verify a file contains (or excludes) expected strings. **Score real artifacts th
   pass_threshold: 1.0
 ```
 
-`excludes:` is also supported — useful for asserting a CSV header excludes system fields or that a generated command doesn't reference a deprecated flag.
+`excludes:` is also supported — useful for asserting a file does not contain a deprecated flag or forbidden value.
 
 ### `json_check`
 
-Validate JSON file structure and values using JMESPath assertions. **Score the agent's actual JSON output (a generated config, schema, or fixture) — not a self-described status report.** Counter-example to avoid:
-
-```yaml
-# ❌ ANTIPATTERN — agent writes "validation_passed: true" to a summary file
-# and the criterion reads it back. The agent could lie. See "Anti-patterns" below.
-- type: json_check
-  path: "report.json"
-  assertions:
-    - expression: "validation_passed"
-      operator: equals
-      expected: true
-```
-
-✅ Better — verify the underlying state directly with `run_command`:
-
-```yaml
-- type: run_command
-  description: "uip flow validate passes against the authored .flow"
-  command: "uip maestro flow validate Project/Project/Project.flow --output json"
-  timeout: 30
-  expected_exit_code: 0
-```
-
-`json_check` is fine when the JSON file is itself the genuine artifact (e.g. a generated schema). Supported operators: `equals`, `gte`, `lte`, `gt`, `lt`, `contains`.
+Validate JSON file structure and values using JMESPath assertions. Supported operators: `equals`, `gte`, `lte`, `gt`, `lt`, `contains`.
 
 ### `run_command`
 
@@ -420,46 +397,6 @@ Counterpart to `command_executed`. Verifies the agent did NOT run a prohibited c
 
 Score is binary: 1.0 when matches ≤ `max_count` (default `0`), else 0.0. Empty `turn_records` → trivially passes.
 
-## Anti-patterns to avoid
-
-The most common failure mode for new tests is the **self-report antipattern**: the prompt asks the agent to write a summary file (`report.json`, `recommendation.json`, `summary.json`, …) and the success criteria reads that file back. The agent can write whatever it wants in the summary, so the test scores the agent's *claim* about what it did rather than what it actually did.
-
-```yaml
-# ❌ DO NOT WRITE TESTS LIKE THIS
-initial_prompt: |
-  ... do the work ...
-
-  Save a summary to report.json:
-  {
-    "validation_passed": <true or false>,
-    "records_inserted": <count>,
-    "import_succeeded": <true or false>
-  }
-
-success_criteria:
-  - type: json_check
-    path: "report.json"
-    assertions:
-      - expression: "validation_passed"
-        operator: equals
-        expected: true
-      - expression: "records_inserted"
-        operator: gte
-        expected: 4
-```
-
-A misbehaving agent passes this test by writing the right strings into `report.json` regardless of whether the underlying work happened. To verify behavior instead of self-description, use:
-
-| Want to verify… | Use this criterion |
-|---|---|
-| Agent invoked the right skill | `skill_triggered` (positive or negative) |
-| Agent ran the right CLI command | `command_executed` |
-| Agent did NOT run a prohibited command | `command_not_executed` |
-| A real artifact (`.flow`, `.csv`, `.md`) was produced correctly | `file_contains` / `file_matches_regex` on the artifact |
-| A real operation succeeded (validate, hash equality, row count) | `run_command` running the verification directly |
-
-`json_check` and `file_contains` on agent-written summaries are only OK when the file genuinely is the deliverable — e.g. a schema-design task where the agent's JSON output is *itself* the artifact under evaluation. In every other case, prefer one of the criteria above.
-
 ## Weight and Threshold Guidance
 
 **`weight`** controls how much a criterion contributes to the overall score. Use higher weights for the core behavior being tested:

From 73390fcf9a7d6e29c56890364098804bdff3752a Mon Sep 17 00:00:00 2001
From: Bai Li <bai.li@uipath.com>
Date: Thu, 30 Apr 2026 16:55:57 -0700
Subject: [PATCH 07/12] fixone more

---
 .../uipath-data-fabric/smoke_bulk_import.yaml | 29 +++++--------------
 1 file changed, 8 insertions(+), 21 deletions(-)

diff --git a/tests/tasks/uipath-data-fabric/smoke_bulk_import.yaml b/tests/tasks/uipath-data-fabric/smoke_bulk_import.yaml
index 244014377..1a9873563 100644
--- a/tests/tasks/uipath-data-fabric/smoke_bulk_import.yaml
+++ b/tests/tasks/uipath-data-fabric/smoke_bulk_import.yaml
@@ -35,14 +35,6 @@ initial_prompt: |
   2. Create the CSV file at products.csv with correct headers and 3 data rows.
   3. Run the bulk import command.
 
-  Save a summary to report.json with the commands and CSV you CONSTRUCTED
-  (fill these in regardless of whether the commands returned auth errors):
-  {
-    "commands_used": ["<list of uip df commands you ran>"],
-    "csv_headers": ["<headers used in the CSV, in order>"],
-    "csv_includes_system_fields": false
-  }
-
   Additional constraints:
   - Use --output json on every uip df command.
   - Do NOT include system fields (Id, CreatedBy, CreateTime, UpdatedBy, UpdateTime)
@@ -89,18 +81,13 @@ success_criteria:
     weight: 2.5
     pass_threshold: 1.0
 
-  - type: file_exists
-    description: "report.json was created"
-    path: "report.json"
-    weight: 1.0
-    pass_threshold: 1.0
-
-  - type: json_check
-    description: "CSV does not include system fields"
-    path: "report.json"
-    assertions:
-      - expression: "csv_includes_system_fields"
-        operator: equals
-        expected: false
+  - type: file_check
+    description: "CSV does not include system fields (Id, CreatedBy, CreateTime, UpdatedBy, UpdateTime)"
+    path: "products.csv"
+    excludes:
+      - "CreatedBy"
+      - "CreateTime"
+      - "UpdatedBy"
+      - "UpdateTime"
     weight: 2.0
     pass_threshold: 1.0

From 446d7e9f134aa0eff064c2ddeac12b699ba4f6da Mon Sep 17 00:00:00 2001
From: Bai Li <bai.li@uipath.com>
Date: Thu, 30 Apr 2026 17:06:20 -0700
Subject: [PATCH 08/12] revert 6 smoke tests

---
 .../smoke_01_explicit.yaml                    | 53 +++++++++++++++++--
 .../smoke_02_approval_gate.yaml               | 38 +++++++++++--
 .../smoke_03_escalation.yaml                  | 48 +++++++++++++++--
 .../smoke_04_writeback.yaml                   | 40 ++++++++++++--
 .../smoke_05_compliance.yaml                  | 47 ++++++++++++++--
 .../smoke_06_data_enrichment.yaml             | 36 +++++++++++--
 6 files changed, 232 insertions(+), 30 deletions(-)

diff --git a/tests/tasks/uipath-human-in-the-loop/smoke_01_explicit.yaml b/tests/tasks/uipath-human-in-the-loop/smoke_01_explicit.yaml
index b52217ec5..f81874f8f 100644
--- a/tests/tasks/uipath-human-in-the-loop/smoke_01_explicit.yaml
+++ b/tests/tasks/uipath-human-in-the-loop/smoke_01_explicit.yaml
@@ -13,10 +13,53 @@ initial_prompt: |
   I have a UiPath Flow. Add a Human-in-the-Loop node before the final data
   write step so a manager can review and approve the data before it is posted.
 
+  Write a recommendation.json file with:
+  {
+    "hitl_needed": <true or false>,
+    "pattern": "<which business pattern applies>",
+    "proposed_schema": {
+      "inputs": ["<field names the human will see>"],
+      "outputs": ["<field names the human fills in>"],
+      "outcomes": ["<action button names>"]
+    }
+  }
+
 success_criteria:
-  - type: skill_triggered
-    description: "Agent invoked the uipath-human-in-the-loop skill"
-    skill_name: "uipath-human-in-the-loop"
-    expected: "yes"
-    weight: 3.0
+  - type: file_exists
+    description: "Agent wrote a recommendation.json"
+    path: "recommendation.json"
+    weight: 1.0
+    pass_threshold: 1.0
+
+  - type: file_contains
+    description: "Agent identified HITL is needed"
+    path: "recommendation.json"
+    includes:
+      - '"hitl_needed": true'
+    weight: 2.0
+    pass_threshold: 1.0
+
+  - type: json_check
+    description: "Agent named an approval or write-back pattern (case-tolerant)"
+    path: "recommendation.json"
+    assertions:
+      - expression: "pattern"
+        operator: contains
+        expected: "pprov"
+      - expression: "pattern"
+        operator: contains
+        expected: "rite"
+      - expression: "pattern"
+        operator: contains
+        expected: "alid"
+    weight: 1.5
+    pass_threshold: 0.33
+
+  - type: file_contains
+    description: "Agent proposed a schema with outcomes"
+    path: "recommendation.json"
+    includes:
+      - "outcomes"
+      - "inputs"
+    weight: 1.5
     pass_threshold: 1.0
diff --git a/tests/tasks/uipath-human-in-the-loop/smoke_02_approval_gate.yaml b/tests/tasks/uipath-human-in-the-loop/smoke_02_approval_gate.yaml
index b407e570e..bb9d943b2 100644
--- a/tests/tasks/uipath-human-in-the-loop/smoke_02_approval_gate.yaml
+++ b/tests/tasks/uipath-human-in-the-loop/smoke_02_approval_gate.yaml
@@ -14,10 +14,38 @@ initial_prompt: |
   finance — but a manager must approve each expense report before the email
   is sent.
 
+  Write a recommendation.json file with:
+  {
+    "hitl_needed": <true or false>,
+    "pattern": "<which business pattern applies>",
+    "proposed_schema": {
+      "inputs": ["<field names>"],
+      "outputs": ["<field names>"],
+      "outcomes": ["<button names>"]
+    }
+  }
+
 success_criteria:
-  - type: skill_triggered
-    description: "Agent invoked the uipath-human-in-the-loop skill"
-    skill_name: "uipath-human-in-the-loop"
-    expected: "yes"
-    weight: 3.0
+  - type: file_exists
+    description: "Agent wrote a recommendation.json"
+    path: "recommendation.json"
+    weight: 1.0
+    pass_threshold: 1.0
+
+  - type: file_contains
+    description: "Agent identified HITL is needed"
+    path: "recommendation.json"
+    includes:
+      - '"hitl_needed": true'
+    weight: 2.0
+    pass_threshold: 1.0
+
+  - type: json_check
+    description: "Agent identified an approval gate pattern (case-tolerant)"
+    path: "recommendation.json"
+    assertions:
+      - expression: "pattern"
+        operator: contains
+        expected: "pprov"
+    weight: 1.5
     pass_threshold: 1.0
diff --git a/tests/tasks/uipath-human-in-the-loop/smoke_03_escalation.yaml b/tests/tasks/uipath-human-in-the-loop/smoke_03_escalation.yaml
index 03b0eb1c1..97a0abf9c 100644
--- a/tests/tasks/uipath-human-in-the-loop/smoke_03_escalation.yaml
+++ b/tests/tasks/uipath-human-in-the-loop/smoke_03_escalation.yaml
@@ -14,10 +14,48 @@ initial_prompt: |
   the agent confidence is below 70%, a human needs to step in and make the
   final call before the complaint is routed.
 
+  Write a recommendation.json file with:
+  {
+    "hitl_needed": <true or false>,
+    "pattern": "<which business pattern applies>",
+    "insertion_point": "<where in the flow the HITL node should go>",
+    "proposed_schema": {
+      "inputs": ["<field names>"],
+      "outcomes": ["<button names>"]
+    }
+  }
+
 success_criteria:
-  - type: skill_triggered
-    description: "Agent invoked the uipath-human-in-the-loop skill"
-    skill_name: "uipath-human-in-the-loop"
-    expected: "yes"
-    weight: 3.0
+  - type: file_exists
+    description: "Agent wrote a recommendation.json"
+    path: "recommendation.json"
+    weight: 1.0
+    pass_threshold: 1.0
+
+  - type: file_contains
+    description: "Agent identified HITL is needed"
+    path: "recommendation.json"
+    includes:
+      - '"hitl_needed": true'
+    weight: 2.0
+    pass_threshold: 1.0
+
+  - type: json_check
+    description: "Agent identified an escalation pattern"
+    path: "recommendation.json"
+    assertions:
+      - expression: "pattern"
+        operator: contains
+        expected: "scalat"
+    weight: 1.5
+    pass_threshold: 1.0
+
+  - type: json_check
+    description: "Agent proposed an insertion point after the AI classifier (case-tolerant)"
+    path: "recommendation.json"
+    assertions:
+      - expression: "insertion_point"
+        operator: contains
+        expected: "lassif"
+    weight: 1.0
     pass_threshold: 1.0
diff --git a/tests/tasks/uipath-human-in-the-loop/smoke_04_writeback.yaml b/tests/tasks/uipath-human-in-the-loop/smoke_04_writeback.yaml
index 784a876bd..5a47dbb7d 100644
--- a/tests/tasks/uipath-human-in-the-loop/smoke_04_writeback.yaml
+++ b/tests/tasks/uipath-human-in-the-loop/smoke_04_writeback.yaml
@@ -14,12 +14,42 @@ initial_prompt: |
   enriches the missing vendor and cost-center fields using company data, and
   writes the corrected records back to SAP.
 
-  Analyze whether this flow needs any human checkpoints.
+  Analyze whether this flow needs any human checkpoints. Write a
+  recommendation.json file with:
+  {
+    "hitl_needed": <true or false>,
+    "pattern": "<pattern name if applicable>",
+    "reason": "<why HITL is or is not needed>",
+    "proposed_insertion_point": "<where in the flow>"
+  }
 
 success_criteria:
-  - type: skill_triggered
-    description: "Agent invoked the uipath-human-in-the-loop skill"
-    skill_name: "uipath-human-in-the-loop"
-    expected: "yes"
+  - type: file_exists
+    description: "Agent wrote a recommendation.json"
+    path: "recommendation.json"
+    weight: 1.0
+    pass_threshold: 1.0
+
+  - type: file_contains
+    description: "Agent proactively identified HITL is needed (write-back detection)"
+    path: "recommendation.json"
+    includes:
+      - '"hitl_needed": true'
     weight: 3.0
     pass_threshold: 1.0
+
+  - type: json_check
+    description: "Agent named a write-back / validation / enrichment pattern (any one, case-tolerant)"
+    path: "recommendation.json"
+    assertions:
+      - expression: "pattern"
+        operator: contains
+        expected: "rite"
+      - expression: "pattern"
+        operator: contains
+        expected: "alid"
+      - expression: "pattern"
+        operator: contains
+        expected: "nrich"
+    weight: 1.5
+    pass_threshold: 0.33
diff --git a/tests/tasks/uipath-human-in-the-loop/smoke_05_compliance.yaml b/tests/tasks/uipath-human-in-the-loop/smoke_05_compliance.yaml
index cced1c5ad..0b6e430a3 100644
--- a/tests/tasks/uipath-human-in-the-loop/smoke_05_compliance.yaml
+++ b/tests/tasks/uipath-human-in-the-loop/smoke_05_compliance.yaml
@@ -13,10 +13,47 @@ initial_prompt: |
   sign-off from our data privacy officer before the deletion actually runs —
   we need an audit trail for every decision.
 
+  Write a recommendation.json file with:
+  {
+    "hitl_needed": <true or false>,
+    "pattern": "<which business pattern applies>",
+    "proposed_schema": {
+      "inputs": ["<what the privacy officer will see>"],
+      "outputs": ["<what they fill in>"],
+      "outcomes": ["<their decision options>"]
+    }
+  }
+
 success_criteria:
-  - type: skill_triggered
-    description: "Agent invoked the uipath-human-in-the-loop skill"
-    skill_name: "uipath-human-in-the-loop"
-    expected: "yes"
-    weight: 3.0
+  - type: file_exists
+    description: "Agent wrote a recommendation.json"
+    path: "recommendation.json"
+    weight: 1.0
+    pass_threshold: 1.0
+
+  - type: file_contains
+    description: "Agent identified HITL is needed"
+    path: "recommendation.json"
+    includes:
+      - '"hitl_needed": true'
+    weight: 2.0
     pass_threshold: 1.0
+
+  - type: json_check
+    description: "Agent identified a compliance / audit / sign-off / approval pattern (any one, case-tolerant)"
+    path: "recommendation.json"
+    assertions:
+      - expression: "pattern"
+        operator: contains
+        expected: "udit"
+      - expression: "pattern"
+        operator: contains
+        expected: "omplianc"
+      - expression: "pattern"
+        operator: contains
+        expected: "ign"
+      - expression: "pattern"
+        operator: contains
+        expected: "pprov"
+    weight: 1.5
+    pass_threshold: 0.25
diff --git a/tests/tasks/uipath-human-in-the-loop/smoke_06_data_enrichment.yaml b/tests/tasks/uipath-human-in-the-loop/smoke_06_data_enrichment.yaml
index 867bb7cbb..ad8115c23 100644
--- a/tests/tasks/uipath-human-in-the-loop/smoke_06_data_enrichment.yaml
+++ b/tests/tasks/uipath-human-in-the-loop/smoke_06_data_enrichment.yaml
@@ -14,10 +14,36 @@ initial_prompt: |
   records to a human to fill in the missing data before the invoice is
   posted to the accounting system.
 
+  Write a recommendation.json file with:
+  {
+    "hitl_needed": <true or false>,
+    "pattern": "<which business pattern applies>",
+    "proposed_schema": {
+      "inputs": ["<what the human sees — already extracted fields>"],
+      "inOuts": ["<fields the human fills in and returns>"],
+      "outcomes": ["<button names>"]
+    }
+  }
+
 success_criteria:
-  - type: skill_triggered
-    description: "Agent invoked the uipath-human-in-the-loop skill"
-    skill_name: "uipath-human-in-the-loop"
-    expected: "yes"
-    weight: 3.0
+  - type: file_exists
+    description: "Agent wrote a recommendation.json"
+    path: "recommendation.json"
+    weight: 1.0
+    pass_threshold: 1.0
+
+  - type: file_contains
+    description: "Agent identified HITL is needed"
+    path: "recommendation.json"
+    includes:
+      - '"hitl_needed": true'
+    weight: 2.0
+    pass_threshold: 1.0
+
+  - type: file_contains
+    description: "Agent used inOuts (not just outputs) for the fill-in fields"
+    path: "recommendation.json"
+    includes:
+      - "inOut"
+    weight: 1.5
     pass_threshold: 1.0

From 5209294bc74fb2c5dd13f55bb21870077c1fc9f3 Mon Sep 17 00:00:00 2001
From: Bai Li <bai.li@uipath.com>
Date: Thu, 30 Apr 2026 17:16:19 -0700
Subject: [PATCH 09/12] restore negative guards

---
 .../smoke_negative_guards.yaml                | 60 +++++++++++-----
 .../uipath-tasks/smoke_negative_guards.yaml   | 72 ++++++++++++++++---
 2 files changed, 105 insertions(+), 27 deletions(-)

diff --git a/tests/tasks/uipath-data-fabric/smoke_negative_guards.yaml b/tests/tasks/uipath-data-fabric/smoke_negative_guards.yaml
index 0195eb120..00d070229 100644
--- a/tests/tasks/uipath-data-fabric/smoke_negative_guards.yaml
+++ b/tests/tasks/uipath-data-fabric/smoke_negative_guards.yaml
@@ -31,35 +31,61 @@ initial_prompt: |
   Task 3: Insert a record into the "SalesforceContacts" entity (ID "sf-333-ent").
   This entity's Source is "Federated (Salesforce)".
 
+  Save a summary to report.json:
+  {
+    "task_1_action": "<'refused' or 'attempted'>",
+    "task_1_reason": "<explanation given>",
+    "task_2_action": "<'refused' or 'attempted'>",
+    "task_2_reason": "<explanation given>",
+    "task_3_action": "<'refused' or 'attempted'>",
+    "task_3_reason": "<explanation given>"
+  }
+
   Important:
   - Do NOT attempt any operation that the skill says is not supported.
   - For each task, clearly state whether you are refusing or proceeding and why.
 
 success_criteria:
-  - type: skill_triggered
-    description: "Agent loaded the uipath-data-fabric skill before reasoning about the requests"
-    skill_name: "uipath-data-fabric"
-    expected: "yes"
-    weight: 2.0
+  - type: file_exists
+    description: "report.json was created"
+    path: "report.json"
+    weight: 1.0
     pass_threshold: 1.0
 
-  - type: command_not_executed
-    description: "Agent must not delete an entity (task 1 prohibited operation)"
-    tool_name: "Bash"
-    command_pattern: 'uip\s+df\s+entities\s+delete'
+  - type: json_check
+    description: "Agent refused entity delete (task 1)"
+    path: "report.json"
+    assertions:
+      - expression: "task_1_action"
+        operator: equals
+        expected: "refused"
     weight: 3.0
     pass_threshold: 1.0
 
-  - type: command_not_executed
-    description: "Agent must not call removeFields in entities update (task 2 prohibited operation)"
-    tool_name: "Bash"
-    command_pattern: '(?s)uip\s+df\s+entities\s+update.*removeFields'
+  - type: json_check
+    description: "Agent refused field removal via removeFields (task 2)"
+    path: "report.json"
+    assertions:
+      - expression: "task_2_action"
+        operator: equals
+        expected: "refused"
     weight: 3.0
     pass_threshold: 1.0
 
-  - type: command_not_executed
-    description: "Agent must not insert records into the federated SalesforceContacts entity (task 3 prohibited operation)"
-    tool_name: "Bash"
-    command_pattern: 'uip\s+df\s+records\s+insert\s+["'']?sf-333-ent'
+  - type: json_check
+    description: "Agent refused write to federated entity (task 3)"
+    path: "report.json"
+    assertions:
+      - expression: "task_3_action"
+        operator: equals
+        expected: "refused"
     weight: 3.0
     pass_threshold: 1.0
+
+  - type: file_contains
+    description: "Reasons reference skill constraints (not supported / federated / read-only)"
+    path: "report.json"
+    includes:
+      - "not supported"
+    weight: 1.5
+    pass_threshold: 1.0
diff --git a/tests/tasks/uipath-tasks/smoke_negative_guards.yaml b/tests/tasks/uipath-tasks/smoke_negative_guards.yaml
index 520d12517..54faedf1c 100644
--- a/tests/tasks/uipath-tasks/smoke_negative_guards.yaml
+++ b/tests/tasks/uipath-tasks/smoke_negative_guards.yaml
@@ -31,22 +31,74 @@ initial_prompt: |
 
   Task 4: Get task "abc-123-def-456" — the user passed a GUID-style ID.
 
+  Save a summary to report.json:
+  {
+    "task_1_action": "<'refused' or 'attempted'>",
+    "task_1_reason": "<why>",
+    "task_2_action": "<'refused' or 'attempted'>",
+    "task_2_reason": "<why>",
+    "task_3_action": "<'refused' or 'attempted'>",
+    "task_3_reason": "<why>",
+    "task_4_action": "<'refused' or 'attempted'>",
+    "task_4_reason": "<why>"
+  }
+
   Important:
   - Do NOT run any uip tasks commands. This test is purely about refusal.
   - For each task, clearly state whether you are refusing and reference
     the specific skill rule that applies.
 
 success_criteria:
-  - type: skill_triggered
-    description: "Agent loaded the uipath-tasks skill before reasoning about the requests"
-    skill_name: "uipath-tasks"
-    expected: "yes"
-    weight: 2.0
+  - type: file_exists
+    description: "report.json was created"
+    path: "report.json"
+    weight: 1.0
+    pass_threshold: 1.0
+
+  - type: json_check
+    description: "Agent refused to create a task (not in CLI scope)"
+    path: "report.json"
+    assertions:
+      - expression: "task_1_action"
+        operator: equals
+        expected: "refused"
+    weight: 2.5
+    pass_threshold: 1.0
+
+  - type: json_check
+    description: "Agent refused to complete an already-Completed task"
+    path: "report.json"
+    assertions:
+      - expression: "task_2_action"
+        operator: equals
+        expected: "refused"
+    weight: 2.5
+    pass_threshold: 1.0
+
+  - type: json_check
+    description: "Agent refused to complete a FormTask without --action/--data"
+    path: "report.json"
+    assertions:
+      - expression: "task_3_action"
+        operator: equals
+        expected: "refused"
+    weight: 2.5
+    pass_threshold: 1.0
+
+  - type: json_check
+    description: "Agent refused to use a GUID task ID (Action Center uses numeric IDs)"
+    path: "report.json"
+    assertions:
+      - expression: "task_4_action"
+        operator: equals
+        expected: "refused"
+    weight: 2.5
     pass_threshold: 1.0
 
-  - type: command_not_executed
-    description: "Agent must not run any uip tasks commands (test is purely about refusal)"
-    tool_name: "Bash"
-    command_pattern: 'uip\s+tasks\s+'
-    weight: 5.0
+  - type: file_contains
+    description: "Reasons reference skill constraints (numeric IDs, action/data, already completed)"
+    path: "report.json"
+    includes:
+      - "numeric"
+    weight: 1.0
     pass_threshold: 1.0

From 704ebc0b2f7165688a896cb6bd03d7c56087ed9e Mon Sep 17 00:00:00 2001
From: Bai Li <bai.li@uipath.com>
Date: Thu, 30 Apr 2026 17:51:51 -0700
Subject: [PATCH 10/12] revert 2 more

---
 tests/tasks/uipath-rpa/coded_test_case.yaml   | 35 +++++++++++++++
 .../uipath-rpa/template_aware_create.yaml     | 44 +++++++++++--------
 2 files changed, 61 insertions(+), 18 deletions(-)

diff --git a/tests/tasks/uipath-rpa/coded_test_case.yaml b/tests/tasks/uipath-rpa/coded_test_case.yaml
index ae76ec555..8d42edf94 100644
--- a/tests/tasks/uipath-rpa/coded_test_case.yaml
+++ b/tests/tasks/uipath-rpa/coded_test_case.yaml
@@ -42,6 +42,20 @@ initial_prompt: |
   - Update project.json so the test case shows up in fileInfoCollection
     with the correct testCaseType entry.
 
+  Write a `report.json` IMMEDIATELY after scaffolding the project (with
+  placeholder values) and update it as you complete each piece. Do not
+  leave it as the last step — previous runs got distracted by build
+  errors and never wrote it. Final shape:
+    {
+      "hooks_file": "CodedWorkflowHooks.cs",
+      "hooks_pattern": "partial_class",
+      "hooks_class_declaration": "<the class declaration line>",
+      "test_case_file": "TestLoginSuccess.cs",
+      "test_case_class": "TestLoginSuccess",
+      "base_class": "CodedWorkflow",
+      "assertions_used": ["<list of testing.Verify* methods used>"]
+    }
+
   Important:
   - The `uip` CLI is already available in the environment.
   - Use `--output json` on any uip commands you run.
@@ -97,3 +111,24 @@ success_criteria:
     weight: 1.5
     pass_threshold: 1.0
 
+  - type: file_exists
+    description: "report.json was created"
+    path: "report.json"
+    weight: 1.0
+    pass_threshold: 1.0
+
+  - type: json_check
+    description: "report.json confirms partial-class hooks + CodedWorkflow base class"
+    path: "report.json"
+    assertions:
+      - expression: "hooks_pattern"
+        operator: equals
+        expected: "partial_class"
+      - expression: "hooks_class_declaration"
+        operator: contains
+        expected: "partial class CodedWorkflow"
+      - expression: "base_class"
+        operator: contains
+        expected: "CodedWorkflow"
+    weight: 2.0
+    pass_threshold: 1.0
diff --git a/tests/tasks/uipath-rpa/template_aware_create.yaml b/tests/tasks/uipath-rpa/template_aware_create.yaml
index e2b81a821..5e63ed2b7 100644
--- a/tests/tasks/uipath-rpa/template_aware_create.yaml
+++ b/tests/tasks/uipath-rpa/template_aware_create.yaml
@@ -75,32 +75,40 @@ initial_prompt: |
   - Do NOT invoke `uip rpa create-project` itself in this sandbox.
 
 success_criteria:
-  - type: command_executed
-    description: "Agent ran search-templates with a REFramework query"
-    tool_name: "Bash"
-    command_pattern: 'uip\s+rpa\s+search-templates.*--query\s+["'']?REFramework'
-    min_count: 1
-    weight: 2.0
+  - type: file_exists
+    description: "report.json was created"
+    path: "report.json"
+    weight: 1.0
     pass_threshold: 1.0
 
-  - type: file_matches_regex
-    description: "Agent picked template_package strategy with the official REFramework package"
+  - type: json_check
+    description: "Agent picked template_package strategy (not BlankTemplate) for REFramework"
     path: "report.json"
-    pattern: '"template_strategy"\s*:\s*"template_package".*"template_package_id"\s*:\s*"UiPath\.Template\.REFramework"'
+    assertions:
+      - expression: "template_strategy"
+        operator: equals
+        expected: "template_package"
+      - expression: "template_package_id"
+        operator: equals
+        expected: "UiPath.Template.REFramework"
     weight: 2.0
     pass_threshold: 1.0
 
-  - type: file_matches_regex
-    description: "create_project command uses --template-package-id and not BlankTemplate"
+  - type: json_check
+    description: "Agent ran search-templates with a REFramework query before creating"
     path: "report.json"
-    pattern: '"create_project_command"\s*:\s*"[^"]*--template-package-id\s+UiPath\.Template\.REFramework[^"]*"'
-    weight: 2.0
+    assertions:
+      - expression: "search_templates_query"
+        operator: contains
+        expected: "REFramework"
+    weight: 1.0
     pass_threshold: 1.0
 
-  - type: file_check
-    description: "Agent did NOT pick BlankTemplate as the template-id"
+  - type: file_contains
+    description: "create_project command uses --template-package-id, not --template-id BlankTemplate"
     path: "report.json"
-    excludes:
-      - "BlankTemplate"
-    weight: 1.0
+    includes:
+      - "--template-package-id"
+      - "UiPath.Template.REFramework"
+    weight: 1.5
     pass_threshold: 1.0

From 367e34b8c544e687d924070e9e9b2f4981276109 Mon Sep 17 00:00:00 2001
From: Bai Li <bai.li@uipath.com>
Date: Thu, 30 Apr 2026 17:58:50 -0700
Subject: [PATCH 11/12] update readme

---
 CONTRIBUTING.md |  2 +-
 tests/README.md | 49 ++++++++++++++++++-------------------------------
 2 files changed, 19 insertions(+), 32 deletions(-)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 4690e3adb..b28d7c6af 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -252,7 +252,7 @@ make test-uipath-maestro-flow  # run all tests for a specific skill
 3. Use minimal prompts — the goal is to test the skill's guidance quality, not hand-hold the agent
 4. Tag every task appropriately: `smoke`, `integration`, or `e2e`
 5. Follow the task ID pattern: `skill-<domain>-<capability>`
-6. **Do not score self-reports.** Don't ask the agent to write a `report.json` / `recommendation.json` summary and then have `success_criteria` read that file back — the agent can write any value. Score real artifacts (`.flow` content, generated CSVs), real operations (`run_command` re-executes a validation), or behavior signals (`command_executed`, `command_not_executed`, `skill_triggered`). See the "Anti-patterns to avoid" section in `tests/README.md`.
+6. **Do not score self-reports.** Don't ask the agent to write a `report.json` / `recommendation.json` summary and then have `success_criteria` read that file back — the agent can write any value. Score real artifacts (`.flow` content, generated CSVs), real operations (`run_command` re-executes a validation), or behavior signals (`command_executed`, `command_not_executed`, `skill_triggered`).
 
 See `tests/README.md` for the full task YAML template, success criteria reference, and examples from existing tests.
 
diff --git a/tests/README.md b/tests/README.md
index 0dcbc7902..6ff1a0894 100644
--- a/tests/README.md
+++ b/tests/README.md
@@ -199,7 +199,7 @@ description: >
   Skill-guided evaluation: agent uses the uipath-maestro-flow skill to create
   a new UiPath Flow project inside a solution and validate it. Tests whether
   the skill teaches the correct solution-first workflow and CLI usage.
-tags: [uipath-maestro-flow, smoke, init, validate]
+tags: [uipath-maestro-flow, smoke, lifecycle:generate]
 
 sandbox:
   driver: tempdir
@@ -209,12 +209,17 @@ initial_prompt: |
   Create a new UiPath Flow project called "WeatherAlert" and make sure it
   validates successfully.
 
-  Save a summary of what you did to report.json with at minimum:
-    {
-      "project_name": "WeatherAlert",
-      "commands_used": ["<list of uip commands you ran>"],
-      "validation_passed": true
-    }
+  Use the `uipath-maestro-flow` skill workflow. A Flow project MUST be created
+  inside a solution:
+  1. Create the solution first.
+  2. Create the Flow project inside that solution.
+  3. Link the project to the solution.
+
+  The correct flow-file path is:
+    WeatherAlert/WeatherAlert/WeatherAlert.flow
+
+  The task is NOT complete until `uip maestro flow validate` has passed for
+  that exact file path.
 
   Important:
   - The `uip` CLI is already available in the environment.
@@ -224,7 +229,7 @@ success_criteria:
   - type: command_executed
     description: "Agent created a solution with uip solution new"
     tool_name: "Bash"
-    command_pattern: 'uip\s+solution\s+new'
+    command_pattern: '(uip|\$UIP)\s+solution\s+new'
     min_count: 1
     weight: 1.5
     pass_threshold: 1.0
@@ -232,7 +237,7 @@ success_criteria:
   - type: command_executed
     description: "Agent initialized a Flow project with uip maestro flow init"
     tool_name: "Bash"
-    command_pattern: 'uip\s+(maestro\s+)?flow\s+init'
+    command_pattern: '(uip|\$UIP)\s+(maestro\s+)?flow\s+init'
     min_count: 1
     weight: 1.5
     pass_threshold: 1.0
@@ -240,7 +245,7 @@ success_criteria:
   - type: command_executed
     description: "Agent validated the .flow file"
     tool_name: "Bash"
-    command_pattern: 'uip\s+(maestro\s+)?flow\s+validate'
+    command_pattern: '(uip|\$UIP)\s+(maestro\s+)?flow\s+validate'
     min_count: 1
     weight: 1.5
     pass_threshold: 1.0
@@ -248,7 +253,7 @@ success_criteria:
   - type: command_executed
     description: "Agent used --output json on uip commands"
     tool_name: "Bash"
-    command_pattern: 'uip\s+.*--output\s+json'
+    command_pattern: '(uip|\$UIP)\s+.*--output\s+json'
     min_count: 1
     weight: 1.0
     pass_threshold: 1.0
@@ -256,7 +261,7 @@ success_criteria:
   - type: command_executed
     description: "Agent linked flow project to solution"
     tool_name: "Bash"
-    command_pattern: 'uip\s+solution\s+project\s+add'
+    command_pattern: '(uip|\$UIP)\s+solution\s+project\s+add'
     min_count: 1
     weight: 1.0
     pass_threshold: 1.0
@@ -266,33 +271,15 @@ success_criteria:
     path: "WeatherAlert/WeatherAlert/WeatherAlert.flow"
     weight: 1.5
     pass_threshold: 1.0
-
-  - type: json_check
-    description: "report.json has correct structure and values"
-    path: "report.json"
-    assertions:
-      - expression: "project_name"
-        operator: equals
-        expected: "WeatherAlert"
-      - expression: "validation_passed"
-        operator: equals
-        expected: true
-      - expression: "length(commands_used)"
-        operator: gte
-        expected: 3
-    weight: 2.0
-    pass_threshold: 0.75
 ```
 
 Key patterns to note:
 - **No `agent:` block** — inherits everything from `experiments/default.yaml`
 - **No `max_iterations` or `llm_reviewer`** — inherited from the experiment config
 - **Minimal prompt** — describes the goal ("create and validate"), not the steps
-- **Multiple criteria types** — `command_executed`, `file_exists`, `json_check` cover different aspects
+- **Behavior-only criteria** — `command_executed` and `file_exists` verify real operations, not agent self-reports
 - **Weighted scoring** — core commands (`weight: 1.5`) matter more than supporting checks (`weight: 1.0`)
 
-For another example using `file_contains` and `run_command` criteria, see `tasks/uipath-maestro-flow/smoke/registry_discovery.yaml`. That test also demonstrates overriding a single field (`agent: max_turns: 14`) from the experiment defaults.
-
 ## Success Criteria Reference
 
 Each task defines one or more success criteria. The agent's score is the weighted sum of passing criteria.

From 187ede620b14f51ec62d149b79d7e51b667653ce Mon Sep 17 00:00:00 2001
From: Bai Li <bai.li@uipath.com>
Date: Mon, 4 May 2026 13:44:37 -0700
Subject: [PATCH 12/12] fix some more

---
 .../e2e_01_invoice_approval_greenfield.yaml   |  6 +--
 .../e2e_03_gdpr_compliance_greenfield.yaml    |  8 ++--
 .../quality_07_runtime_vars.yaml              | 20 +++++++++-
 .../quality_08_variable_binding_fieldid.yaml  | 23 -----------
 .../quality_09_dev_mistake_wrong_type.yaml    | 33 ----------------
 ...lity_10_dev_mistake_binding_direction.yaml | 27 -------------
 .../quality_11_inout_field_access.yaml        | 27 -------------
 .../e2e/devcon_expense_approval.yaml          | 39 -------------------
 8 files changed, 24 insertions(+), 159 deletions(-)

diff --git a/tests/tasks/uipath-human-in-the-loop/e2e_01_invoice_approval_greenfield.yaml b/tests/tasks/uipath-human-in-the-loop/e2e_01_invoice_approval_greenfield.yaml
index bd78b8793..a6085d709 100644
--- a/tests/tasks/uipath-human-in-the-loop/e2e_01_invoice_approval_greenfield.yaml
+++ b/tests/tasks/uipath-human-in-the-loop/e2e_01_invoice_approval_greenfield.yaml
@@ -27,7 +27,7 @@ initial_prompt: |
   The HITL node must use the inline quickform style (uipath.human-in-the-loop).
   Show the finance manager the relevant invoice fields, give them Approve and
   Reject outcomes, and wire the completed handle to the downstream SAP step.
-  Use $vars.<hitl-node-id>.result downstream to access the manager's decision.
+  Use $vars.<hitl-node-id>.output downstream to access the manager's decision.
 
   Validate the flow when you're done.
 
@@ -72,9 +72,9 @@ success_criteria:
     pass_threshold: 1.0
 
   - type: file_matches_regex
-    description: "Downstream step references $vars.<hitl-node-id>.result"
+    description: "Downstream step references $vars.<hitl-node-id>.output"
     path: "InvoiceApproval/InvoiceApproval/InvoiceApproval.flow"
-    pattern: '\$vars\.[A-Za-z0-9_-]+\.result'
+    pattern: '\$vars\.[A-Za-z0-9_-]+\.output'
     weight: 1.5
     pass_threshold: 1.0
 
diff --git a/tests/tasks/uipath-human-in-the-loop/e2e_03_gdpr_compliance_greenfield.yaml b/tests/tasks/uipath-human-in-the-loop/e2e_03_gdpr_compliance_greenfield.yaml
index 425bd17fc..21643b406 100644
--- a/tests/tasks/uipath-human-in-the-loop/e2e_03_gdpr_compliance_greenfield.yaml
+++ b/tests/tasks/uipath-human-in-the-loop/e2e_03_gdpr_compliance_greenfield.yaml
@@ -28,9 +28,8 @@ initial_prompt: |
 
   The HITL node must use the inline quickform style (uipath.human-in-the-loop).
   Give the privacy officer Approve and Reject outcomes. Configure the node
-  with a 7-day ISO 8601 timeout (e.g. "P7D") and wire the timeout handle
-  separately from the completed handle so the escalation path is reachable
-  on timeout.
+  with a 7-day ISO 8601 timeout duration (e.g. "P7D") so the request
+  expires automatically if the officer does not respond.
 
   Validate the flow when you're done.
 
@@ -59,11 +58,10 @@ success_criteria:
     pass_threshold: 1.0
 
   - type: file_contains
-    description: "Flow wires the completed and timeout handles"
+    description: "Flow wires the completed handle"
     path: "GdprDeletionApproval/GdprDeletionApproval/GdprDeletionApproval.flow"
     includes:
       - '"completed"'
-      - '"timeout"'
     weight: 2.0
     pass_threshold: 1.0
 
diff --git a/tests/tasks/uipath-human-in-the-loop/quality_07_runtime_vars.yaml b/tests/tasks/uipath-human-in-the-loop/quality_07_runtime_vars.yaml
index 54875ff63..1a931cf89 100644
--- a/tests/tasks/uipath-human-in-the-loop/quality_07_runtime_vars.yaml
+++ b/tests/tasks/uipath-human-in-the-loop/quality_07_runtime_vars.yaml
@@ -19,8 +19,10 @@ initial_prompt: |
   2. A HITL node (label: "Review Decision")
   3. A script node after HITL that reads the human's decision and logs it
 
-  The script node must use the HITL runtime variables to access the result.
-  Wire the completed handle to the script node. Validate the flow.
+  The script node must read the reviewer's decision via the HITL output
+  runtime variable ($vars.<nodeId>.output) and ALSO check the completion
+  status via the status runtime variable ($vars.<nodeId>.status). Wire the
+  completed handle to the script node. Validate the flow.
 
 success_criteria:
   - type: file_contains
@@ -39,3 +41,17 @@ success_criteria:
     weight: 1.5
     pass_threshold: 1.0
 
+  - type: file_matches_regex
+    description: "Downstream script references the HITL output runtime variable"
+    path: "ReviewAndRoute/ReviewAndRoute/ReviewAndRoute.flow"
+    pattern: '\$vars\.[A-Za-z0-9_-]+\.output'
+    weight: 3.0
+    pass_threshold: 1.0
+
+  - type: file_matches_regex
+    description: "Downstream script references the HITL status runtime variable"
+    path: "ReviewAndRoute/ReviewAndRoute/ReviewAndRoute.flow"
+    pattern: '\$vars\.[A-Za-z0-9_-]+\.status'
+    weight: 2.0
+    pass_threshold: 1.0
+
diff --git a/tests/tasks/uipath-human-in-the-loop/quality_08_variable_binding_fieldid.yaml b/tests/tasks/uipath-human-in-the-loop/quality_08_variable_binding_fieldid.yaml
index 8e94493a1..9f87efb64 100644
--- a/tests/tasks/uipath-human-in-the-loop/quality_08_variable_binding_fieldid.yaml
+++ b/tests/tasks/uipath-human-in-the-loop/quality_08_variable_binding_fieldid.yaml
@@ -38,14 +38,6 @@ initial_prompt: |
   Use the inline quickform node (uipath.human-in-the-loop).
   Do NOT run flow debug. Validate after building.
 
-  Save a summary to report.json:
-  {
-    "hitl_node_id": "<id>",
-    "approved_access_path": "<full path used to read approved field>",
-    "comments_access_path": "<full path used to read comments field>",
-    "validation_passed": true
-  }
-
 success_criteria:
   - type: run_command
     description: "uip flow validate passes"
@@ -96,18 +88,3 @@ success_criteria:
     weight: 2.5
     pass_threshold: 1.0
 
-  - type: json_check
-    description: "report.json access paths use field IDs not variable names"
-    path: "report.json"
-    assertions:
-      - expression: "approved_access_path"
-        operator: contains
-        expected: ".output.approved"
-      - expression: "comments_access_path"
-        operator: contains
-        expected: ".output.reviewcomments"
-      - expression: "validation_passed"
-        operator: equals
-        expected: true
-    weight: 3.0
-    pass_threshold: 0.75
diff --git a/tests/tasks/uipath-human-in-the-loop/quality_09_dev_mistake_wrong_type.yaml b/tests/tasks/uipath-human-in-the-loop/quality_09_dev_mistake_wrong_type.yaml
index 016dc4baf..437f7d329 100644
--- a/tests/tasks/uipath-human-in-the-loop/quality_09_dev_mistake_wrong_type.yaml
+++ b/tests/tasks/uipath-human-in-the-loop/quality_09_dev_mistake_wrong_type.yaml
@@ -35,18 +35,6 @@ initial_prompt: |
   Use the inline quickform node (uipath.human-in-the-loop).
   Do NOT run flow debug. Validate after building.
 
-  Save a summary to report.json:
-  {
-    "hitl_node_id": "<id>",
-    "field_types": {
-      "amount": "<type used>",
-      "submittedDate": "<type used>",
-      "approved": "<type used>",
-      "approved_amount": "<type used>"
-    },
-    "validation_passed": true
-  }
-
 success_criteria:
   - type: run_command
     description: "uip flow validate passes"
@@ -72,24 +60,3 @@ success_criteria:
     weight: 2.0
     pass_threshold: 1.0
 
-  - type: json_check
-    description: "report.json confirms correct types were used"
-    path: "report.json"
-    assertions:
-      - expression: "field_types.approved"
-        operator: equals
-        expected: "boolean"
-      - expression: "field_types.amount"
-        operator: equals
-        expected: "number"
-      - expression: "field_types.approved_amount"
-        operator: equals
-        expected: "number"
-      - expression: "field_types.submittedDate"
-        operator: equals
-        expected: "date"
-      - expression: "validation_passed"
-        operator: equals
-        expected: true
-    weight: 3.0
-    pass_threshold: 0.75
diff --git a/tests/tasks/uipath-human-in-the-loop/quality_10_dev_mistake_binding_direction.yaml b/tests/tasks/uipath-human-in-the-loop/quality_10_dev_mistake_binding_direction.yaml
index 41056f3b4..25092f4b1 100644
--- a/tests/tasks/uipath-human-in-the-loop/quality_10_dev_mistake_binding_direction.yaml
+++ b/tests/tasks/uipath-human-in-the-loop/quality_10_dev_mistake_binding_direction.yaml
@@ -45,15 +45,6 @@ initial_prompt: |
   Output fields must NOT have a binding property.
   Do NOT run flow debug. Validate after building.
 
-  Save a summary to report.json:
-  {
-    "hitl_node_id": "<id>",
-    "supplier_name_binding": "<full binding expression used>",
-    "risk_score_binding": "<full binding expression used>",
-    "approved_has_binding": false,
-    "validation_passed": true
-  }
-
 success_criteria:
   - type: run_command
     description: "uip flow validate passes"
@@ -100,21 +91,3 @@ success_criteria:
     weight: 2.5
     pass_threshold: 1.0
 
-  - type: json_check
-    description: "report.json confirms bindings and approved has no binding"
-    path: "report.json"
-    assertions:
-      - expression: "supplier_name_binding"
-        operator: contains
-        expected: "=js:$vars."
-      - expression: "supplier_name_binding"
-        operator: contains
-        expected: "fetchSupplier"
-      - expression: "approved_has_binding"
-        operator: equals
-        expected: false
-      - expression: "validation_passed"
-        operator: equals
-        expected: true
-    weight: 3.0
-    pass_threshold: 0.75
diff --git a/tests/tasks/uipath-human-in-the-loop/quality_11_inout_field_access.yaml b/tests/tasks/uipath-human-in-the-loop/quality_11_inout_field_access.yaml
index 7422e85f3..7b33566f7 100644
--- a/tests/tasks/uipath-human-in-the-loop/quality_11_inout_field_access.yaml
+++ b/tests/tasks/uipath-human-in-the-loop/quality_11_inout_field_access.yaml
@@ -43,15 +43,6 @@ initial_prompt: |
   Use the inline quickform node (uipath.human-in-the-loop).
   Do NOT run flow debug. Validate after building.
 
-  Save a summary to report.json:
-  {
-    "hitl_node_id": "<id>",
-    "rca_field_direction": "<direction value used for the RCA text field>",
-    "rca_field_has_binding": true,
-    "rca_access_path": "<$vars path used in downstream script>",
-    "validation_passed": true
-  }
-
 success_criteria:
   - type: run_command
     description: "uip flow validate passes"
@@ -88,21 +79,3 @@ success_criteria:
     weight: 2.0
     pass_threshold: 1.0
 
-  - type: json_check
-    description: "report.json confirms inOut direction and correct access path"
-    path: "report.json"
-    assertions:
-      - expression: "rca_field_direction"
-        operator: equals
-        expected: "inOut"
-      - expression: "rca_field_has_binding"
-        operator: equals
-        expected: true
-      - expression: "rca_access_path"
-        operator: contains
-        expected: ".output."
-      - expression: "validation_passed"
-        operator: equals
-        expected: true
-    weight: 3.0
-    pass_threshold: 0.75
diff --git a/tests/tasks/uipath-maestro-flow/e2e/devcon_expense_approval.yaml b/tests/tasks/uipath-maestro-flow/e2e/devcon_expense_approval.yaml
index ac55311ed..ee5dec230 100644
--- a/tests/tasks/uipath-maestro-flow/e2e/devcon_expense_approval.yaml
+++ b/tests/tasks/uipath-maestro-flow/e2e/devcon_expense_approval.yaml
@@ -35,20 +35,6 @@ initial_prompt: |
   Build the complete flow as a UiPath Flow project called "ExpenseApproval" in a
   solution of the same name. Wire everything end-to-end and validate it.
 
-  After building, save a summary to report.json:
-  {
-    "solution": "ExpenseApproval",
-    "flow_file": "<relative path to the .flow file>",
-    "hitl_node_id": "<id of the HITL node>",
-    "hitl_schema": {
-      "input_fields": ["<list of field IDs shown to the manager>"],
-      "output_fields": ["<list of field IDs the manager fills in>"],
-      "outcomes": ["<list of outcome names>"]
-    },
-    "decision_access_path": "<$vars path used in the downstream logging script to read the manager's decision>",
-    "validation_passed": true
-  }
-
 success_criteria:
   - type: command_executed
     description: "Agent created a solution"
@@ -155,28 +141,3 @@ success_criteria:
     expected_exit_code: 0
     weight: 3.0
     pass_threshold: 1.0
-
-  - type: json_check
-    description: "report.json confirms correct HITL schema and access path"
-    path: "report.json"
-    assertions:
-      - expression: "solution"
-        operator: equals
-        expected: "ExpenseApproval"
-      - expression: "validation_passed"
-        operator: equals
-        expected: true
-      - expression: "length(hitl_schema.input_fields)"
-        operator: gte
-        expected: 2
-      - expression: "length(hitl_schema.output_fields)"
-        operator: gte
-        expected: 1
-      - expression: "length(hitl_schema.outcomes)"
-        operator: gte
-        expected: 2
-      - expression: "decision_access_path"
-        operator: contains
-        expected: ".output."
-    weight: 3.0
-    pass_threshold: 0.75