fix: resolve e2e import test concurrency races

Fix two independent concurrency issues causing flaky e2e import tests:

1. TOCTOU race in evaluator import (import-evaluator.ts):
   The beforeConfigWrite hook lists all online eval configs then fetches
   details for each with Promise.all. If a config is deleted between the
   list and get calls, the API throws 'Online evaluation configuration
   not found' and the entire import fails. Fixed by using
   Promise.allSettled and filtering out disappeared configs.

2. Resource name collisions across parallel CI shards (setup_*.py):
   Python setup scripts generated resource names using int(time.time())
   (second-level precision). Parallel CI shards starting in the same
   second would collide with ConflictException. The test already passes
   a unique RESOURCE_SUFFIX env var but scripts ignored it for naming.
   Added NAME_SUFFIX to common.py that prefers RESOURCE_SUFFIX when set,
   and updated all setup scripts to use it.

Author: Hweinstock

e2e-tests/fixtures/import/common.py

@@ -2,13 +2,16 @@
 import json
 import os
 import time
+import uuid
 import zipfile
 import tempfile
 
 import boto3
 
 REGION = os.environ.get("AWS_REGION") or os.environ.get("AWS_DEFAULT_REGION") or "us-east-1"
 RESOURCE_SUFFIX = os.environ.get("RESOURCE_SUFFIX", "")
+# Unique suffix for resource names — avoids collisions across parallel CI shards.
+NAME_SUFFIX = RESOURCE_SUFFIX or uuid.uuid4().hex[:12]
 SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
 APP_DIR = os.path.join(SCRIPT_DIR, "app")
 _resources_name = f"bugbash-resources-{RESOURCE_SUFFIX}.json" if RESOURCE_SUFFIX else "bugbash-resources.json"

e2e-tests/fixtures/import/setup_evaluator.py

@@ -12,15 +12,15 @@
 from common import (
     get_control_client, save_resource, tag_resource,
     wait_for_evaluator, print_import_command,
+    NAME_SUFFIX,
 )
 
 DEFAULT_EVALUATOR_MODEL = os.environ.get("DEFAULT_EVALUATOR_MODEL", "us.anthropic.claude-sonnet-4-5-20250929-v1:0")
 
 
 def main():
     client = get_control_client()
-    ts = int(time.time())
-    evaluator_name = f"bugbash_eval_{ts}"
+    evaluator_name = f"bugbash_eval_{NAME_SUFFIX}"
 
     print(f"Creating evaluator: {evaluator_name}")
     resp = client.create_evaluator(

e2e-tests/fixtures/import/setup_gateway.py

@@ -16,14 +16,14 @@
 from common import (
     REGION, get_control_client, ensure_role, save_resource,
     tag_resource, wait_for_gateway, wait_for_gateway_target,
+    NAME_SUFFIX,
 )
 
 
 def main():
     role_arn = ensure_role()
     client = get_control_client()
-    ts = int(time.time())
-    gateway_name = f"bugbashGw{ts}"
+    gateway_name = f"bugbashGw{NAME_SUFFIX}"
 
     # ------------------------------------------------------------------
     # 1. Create gateway

e2e-tests/fixtures/import/setup_memory_full.py

@@ -12,18 +12,19 @@
 from common import (
     ensure_role, get_control_client, wait_for_memory,
     save_resource, print_import_command, tag_resource,
+    NAME_SUFFIX,
 )
 
 
 def main():
     role_arn = ensure_role()
     client = get_control_client()
-    memory_name = f"bugbash_memory_{int(time.time())}"
+    memory_name = f"bugbash_memory_{NAME_SUFFIX}"
 
     print(f"Creating memory: {memory_name}")
     resp = client.create_memory(
         name=memory_name,
-        clientToken=f"bugbash-{int(time.time())}",
+        clientToken=f"bugbash-{NAME_SUFFIX}",
         eventExpiryDuration=30,
         memoryExecutionRoleArn=role_arn,
         memoryStrategies=[

e2e-tests/fixtures/import/setup_runtime_basic.py

@@ -11,16 +11,16 @@
 from common import (
     ensure_role, get_control_client, wait_for_runtime,
     save_resource, print_import_command, upload_code,
+    NAME_SUFFIX,
 )
 
 
 def main():
     role_arn = ensure_role()
     client = get_control_client()
-    ts = int(time.time())
-    runtime_name = f"bugbash_basic_{ts}"
+    runtime_name = f"bugbash_basic_{NAME_SUFFIX}"
 
-    bucket, s3_key = upload_code(f"bugbash-basic-{ts}")
+    bucket, s3_key = upload_code(f"bugbash-basic-{NAME_SUFFIX}")
 
     print(f"Creating basic runtime: {runtime_name}")
     resp = client.create_agent_runtime(

src/cli/commands/import/import-evaluator.ts

@@ -1,5 +1,5 @@
 import type { Evaluator } from '../../../schema';
-import type { EvaluatorSummary, GetEvaluatorResult } from '../../aws/agentcore-control';
+import type { EvaluatorSummary, GetEvaluatorResult, GetOnlineEvalConfigResult } from '../../aws/agentcore-control';
 import {
   getEvaluator,
   getOnlineEvaluationConfig,
@@ -92,11 +92,15 @@ const evaluatorDescriptor: ResourceImportDescriptor<GetEvaluatorResult, Evaluato
 
     const oecSummaries = await listAllOnlineEvaluationConfigs({ region: target.region });
     if (oecSummaries.length > 0) {
-      const oecDetails = await Promise.all(
+      // Use allSettled to tolerate configs deleted between list and get (TOCTOU race).
+      const settled = await Promise.allSettled(
         oecSummaries.map(s =>
           getOnlineEvaluationConfig({ region: target.region, configId: s.onlineEvaluationConfigId })
         )
       );
+      const oecDetails = settled
+        .filter((r): r is PromiseFulfilledResult<GetOnlineEvalConfigResult> => r.status === 'fulfilled')
+        .map(r => r.value);
 
       const referencingOec = oecDetails.find(oec => oec.evaluatorIds?.includes(detail.evaluatorId));