fix: resolve e2e import test concurrency races (#1067)

* fix: resolve e2e import test concurrency races

Fix two independent concurrency issues causing flaky e2e import tests:

1. TOCTOU race in evaluator import (import-evaluator.ts):
   The beforeConfigWrite hook lists all online eval configs then fetches
   details for each with Promise.all. If a config is deleted between the
   list and get calls, the API throws 'Online evaluation configuration
   not found' and the entire import fails. Fixed by using
   Promise.allSettled and filtering out disappeared configs.

2. Resource name collisions across parallel CI shards (setup_*.py):
   Python setup scripts generated resource names using int(time.time())
   (second-level precision). Parallel CI shards starting in the same
   second would collide with ConflictException. The test already passes
   a unique RESOURCE_SUFFIX env var but scripts ignored it for naming.
   Added NAME_SUFFIX to common.py that prefers RESOURCE_SUFFIX when set,
   and updated all setup scripts to use it.

* chore: remove unused time imports from setup scripts

Author: Hweinstock

e2e-tests/fixtures/import/common.py

@@ -2,13 +2,16 @@
 import json
 import os
 import time
+import uuid
 import zipfile
 import tempfile
 
 import boto3
 
 REGION = os.environ.get("AWS_REGION") or os.environ.get("AWS_DEFAULT_REGION") or "us-east-1"
 RESOURCE_SUFFIX = os.environ.get("RESOURCE_SUFFIX", "")
+# Unique suffix for resource names — avoids collisions across parallel CI shards.
+NAME_SUFFIX = RESOURCE_SUFFIX or uuid.uuid4().hex[:12]
 SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
 APP_DIR = os.path.join(SCRIPT_DIR, "app")
 _resources_name = f"bugbash-resources-{RESOURCE_SUFFIX}.json" if RESOURCE_SUFFIX else "bugbash-resources.json"

e2e-tests/fixtures/import/setup_evaluator.py

@@ -8,19 +8,18 @@
 import os
 sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
 
-import time
 from common import (
     get_control_client, save_resource, tag_resource,
     wait_for_evaluator, print_import_command,
+    NAME_SUFFIX,
 )
 
 DEFAULT_EVALUATOR_MODEL = os.environ.get("DEFAULT_EVALUATOR_MODEL", "us.anthropic.claude-sonnet-4-5-20250929-v1:0")
 
 
 def main():
     client = get_control_client()
-    ts = int(time.time())
-    evaluator_name = f"bugbash_eval_{ts}"
+    evaluator_name = f"bugbash_eval_{NAME_SUFFIX}"
 
     print(f"Creating evaluator: {evaluator_name}")
     resp = client.create_evaluator(

e2e-tests/fixtures/import/setup_gateway.py

@@ -12,18 +12,17 @@
 import os
 sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
 
-import time
 from common import (
     REGION, get_control_client, ensure_role, save_resource,
     tag_resource, wait_for_gateway, wait_for_gateway_target,
+    NAME_SUFFIX,
 )
 
 
 def main():
     role_arn = ensure_role()
     client = get_control_client()
-    ts = int(time.time())
-    gateway_name = f"bugbashGw{ts}"
+    gateway_name = f"bugbashGw{NAME_SUFFIX}"
 
     # ------------------------------------------------------------------
     # 1. Create gateway

e2e-tests/fixtures/import/setup_memory_full.py

@@ -8,22 +8,22 @@
 import os
 sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
 
-import time
 from common import (
     ensure_role, get_control_client, wait_for_memory,
     save_resource, print_import_command, tag_resource,
+    NAME_SUFFIX,
 )
 
 
 def main():
     role_arn = ensure_role()
     client = get_control_client()
-    memory_name = f"bugbash_memory_{int(time.time())}"
+    memory_name = f"bugbash_memory_{NAME_SUFFIX}"
 
     print(f"Creating memory: {memory_name}")
     resp = client.create_memory(
         name=memory_name,
-        clientToken=f"bugbash-{int(time.time())}",
+        clientToken=f"bugbash-{NAME_SUFFIX}",
         eventExpiryDuration=30,
         memoryExecutionRoleArn=role_arn,
         memoryStrategies=[

e2e-tests/fixtures/import/setup_runtime_basic.py

@@ -7,20 +7,19 @@
 import os
 sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
 
-import time
 from common import (
     ensure_role, get_control_client, wait_for_runtime,
     save_resource, print_import_command, upload_code,
+    NAME_SUFFIX,
 )
 
 
 def main():
     role_arn = ensure_role()
     client = get_control_client()
-    ts = int(time.time())
-    runtime_name = f"bugbash_basic_{ts}"
+    runtime_name = f"bugbash_basic_{NAME_SUFFIX}"
 
-    bucket, s3_key = upload_code(f"bugbash-basic-{ts}")
+    bucket, s3_key = upload_code(f"bugbash-basic-{NAME_SUFFIX}")
 
     print(f"Creating basic runtime: {runtime_name}")
     resp = client.create_agent_runtime(

src/cli/commands/import/import-evaluator.ts

@@ -10,6 +10,7 @@ import { ANSI } from './constants';
 import { failResult, parseAndValidateArn } from './import-utils';
 import { executeResourceImport } from './resource-import';
 import type { ImportResourceOptions, ImportResourceResult, ResourceImportDescriptor } from './types';
+import { ResourceNotFoundException } from '@aws-sdk/client-bedrock-agentcore-control';
 import type { Command } from '@commander-js/extra-typings';
 
 /**
@@ -92,11 +93,18 @@ const evaluatorDescriptor: ResourceImportDescriptor<GetEvaluatorResult, Evaluato
 
     const oecSummaries = await listAllOnlineEvaluationConfigs({ region: target.region });
     if (oecSummaries.length > 0) {
-      const oecDetails = await Promise.all(
-        oecSummaries.map(s =>
-          getOnlineEvaluationConfig({ region: target.region, configId: s.onlineEvaluationConfigId })
+      // Configs can be deleted between list and get (TOCTOU race).
+      // Skip ResourceNotFoundException — a deleted config can't be locking our evaluator.
+      const oecDetails = (
+        await Promise.all(
+          oecSummaries.map(s =>
+            getOnlineEvaluationConfig({ region: target.region, configId: s.onlineEvaluationConfigId }).catch(err => {
+              if (err instanceof ResourceNotFoundException) return null;
+              throw err;
+            })
+          )
         )
-      );
+      ).filter(r => r !== null);
 
       const referencingOec = oecDetails.find(oec => oec.evaluatorIds?.includes(detail.evaluatorId));