Merge branch 'main' into k8s_test_5

Kolleida · Kolleida · commit 3c97f294fb88 · 2026-01-12T12:53:01.000-05:00
diff --git a/README.md b/README.md
@@ -34,7 +34,7 @@ Edit the `[[participants]]` section:
 ```toml
 [[participants]]
 agentbeats_id = "your-agent-id-here"
-name = "routing_operator"
+name = "route_operator"
 env = { AZURE_API_KEY = "${AZURE_API_KEY}", AZURE_API_BASE = "${AZURE_API_BASE}" }
 ```
 
@@ -60,4 +60,4 @@ For each benchmark, agents are primarily evaluated on the following metrics:
 
 3. Latency: How long does it take for an agent to produce a answer. Can be measured in seconds or number of calls (iterations) to that agent.
 
-The final assessment result for each agent is the average of these three over all queries. Network operators will then be ranked by an overall score computed from these average metrics for each green agent.
+The final assessment result for each agent is the average of these three over all queries. Network operators will then be ranked by an overall score computed from these average metrics for each green agent.
diff --git a/generate_compose.py b/generate_compose.py
@@ -74,7 +74,7 @@ def fetch_agent_info(agentbeats_id: str) -> dict:
     depends_on:{green_depends}
     networks:
       - agent-network
-    {k8s_service_options}
+    {extra_options}
 
 {participant_services}
   agentbeats-client:
@@ -184,11 +184,17 @@ def generate_docker_compose(scenario: dict[str, Any], app: str) -> str:
     participant_names = [p["name"] for p in participants]
 
     # Expose kubeconfig and localhost for k8s app to allow communication with kind cluster
-    k8s_service_options = """extra_hosts:
-      - "host.docker.internal:host-gateway"
-    volumes:
+    extra_options = ""
+    if app == "k8s":
+        extra_options = """volumes:
       - ./kubeconfig:/root/.kube/:ro
-"""
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+        """
+    elif app == "route":
+        extra_options = """volumes:
+      - /lib/modules:/lib/modules
+    """
 
     participant_services = "\n".join([
         PARTICIPANT_TEMPLATE.format(
@@ -215,7 +221,7 @@ def generate_docker_compose(scenario: dict[str, Any], app: str) -> str:
         participant_services=participant_services,
         client_depends=format_depends_on(all_services),
         privileged=str(app == "route").lower(),
-        k8s_service_options=k8s_service_options if app == "k8s" else ""
+        extra_options=extra_options
     )
 
 
diff --git a/results/Froot-NetSys-20260111-020722.json b/results/Froot-NetSys-20260111-020722.json
diff --git a/results/lesleychou-20260111-170921.json b/results/lesleychou-20260111-170921.json
diff --git a/results/lesleychou-20260111-173431.json b/results/lesleychou-20260111-173431.json
diff --git a/route_scenario.toml b/route_scenario.toml
@@ -4,7 +4,7 @@ env = { LOG_LEVEL = "INFO" }
 
 [[participants]]
 agentbeats_id = ""
-name = "routing_operator"
+name = "route_operator"
 env = { SECRET = "${GITHUB_SECRET_NAME}" }
 
 [config]
diff --git a/submissions/Froot-NetSys-20260111-020722.provenance.json b/submissions/Froot-NetSys-20260111-020722.provenance.json
@@ -0,0 +1,8 @@
+{
+  "image_digests": {
+    "green-agent": "ghcr.io/froot-netsys/route_agent@sha256:7772f5975f8bddd6adf5a0fc94adf25dc8e933b88c9654153dd8faa073bce3e3",
+    "route_operator": "ghcr.io/froot-netsys/a2a_llm@sha256:4dd2c986b5ee439c71d4b758b1fb8b201e940fffa022b87e1c8c375a57abfba9",
+    "agentbeats-client": "ghcr.io/agentbeats/agentbeats-client@sha256:13dfe3ef4e583a80e7ce2fe3becd0ce3b879841368a7f4fa40b6ebbabeeb014e"
+  },
+  "timestamp": "2026-01-11T02:07:22Z"
+}
diff --git a/submissions/Froot-NetSys-20260111-020722.toml b/submissions/Froot-NetSys-20260111-020722.toml
@@ -0,0 +1,28 @@
+[green_agent]
+agentbeats_id = "019ba8d8-c1d1-7923-b6c7-c5020e1c6cbe"
+env = { LOG_LEVEL = "INFO" }
+
+[[participants]]
+agentbeats_id = "019ba44f-9d1f-7ff3-8fbe-35c4ecfa40b2"
+name = "route_operator"
+
+[participants.env]
+    AZURE_API_KEY = "${AZURE_API_KEY}"
+    AZURE_API_BASE = "${AZURE_API_BASE}"
+    AZURE_API_VERSION = "2024-12-01-preview"
+    MODEL_NAME = "azure/gpt-4.1"
+
+[config]
+# Prompt strategy for the agent. Options: "zeroshot_base", "zeroshot_cot", "fewshot_base", "fewshot_cot"
+prompt_type = "zeroshot_base"
+# Number of queries to generate for each error type (15 error types total).
+num_queries = 1
+# Maximum number of iterations the agent can take to resolve a given routing issue.
+max_iterations = 10
+
+# DO NOT MODIFY BELOW THIS LINE.
+benchmark_path = "assessment_error_config.json"
+regenerate_benchmark = true
+output_dir = "dump"
+num_switches = 2
+num_hosts_per_subnet = 1
diff --git a/submissions/lesleychou-20260111-170921.provenance.json b/submissions/lesleychou-20260111-170921.provenance.json
@@ -0,0 +1,8 @@
+{
+  "image_digests": {
+    "green-agent": "ghcr.io/froot-netsys/route_agent@sha256:7772f5975f8bddd6adf5a0fc94adf25dc8e933b88c9654153dd8faa073bce3e3",
+    "route_operator": "ghcr.io/froot-netsys/a2a_llm@sha256:4dd2c986b5ee439c71d4b758b1fb8b201e940fffa022b87e1c8c375a57abfba9",
+    "agentbeats-client": "ghcr.io/agentbeats/agentbeats-client@sha256:13dfe3ef4e583a80e7ce2fe3becd0ce3b879841368a7f4fa40b6ebbabeeb014e"
+  },
+  "timestamp": "2026-01-11T17:09:21Z"
+}
diff --git a/submissions/lesleychou-20260111-170921.toml b/submissions/lesleychou-20260111-170921.toml
@@ -0,0 +1,23 @@
+[green_agent]
+agentbeats_id = "019ba8d8-c1d1-7923-b6c7-c5020e1c6cbe"
+env = { LOG_LEVEL = "INFO" }
+
+[[participants]]
+agentbeats_id = "019ba44f-9d1f-7ff3-8fbe-35c4ecfa40b2"
+name = "route_operator"
+env = { AZURE_API_KEY = "${AZURE_API_KEY}", AZURE_API_BASE = "${AZURE_API_BASE}", AZURE_API_VERSION = "${AZURE_API_VERSION}", MODEL_NAME = "${MODEL_NAME}" }
+
+[config]
+# Prompt strategy for the agent. Options: "zeroshot_base", "zeroshot_cot", "fewshot_base", "fewshot_cot"
+prompt_type = "zeroshot_base"
+# Number of queries to generate for each error type (15 error types total).
+num_queries = 2
+# Maximum number of iterations the agent can take to resolve a given routing issue.
+max_iterations = 10
+
+# DO NOT MODIFY BELOW THIS LINE.
+benchmark_path = "assessment_error_config.json"
+regenerate_benchmark = true
+output_dir = "dump"
+num_switches = 2
+num_hosts_per_subnet = 1
diff --git a/submissions/lesleychou-20260111-173431.provenance.json b/submissions/lesleychou-20260111-173431.provenance.json
@@ -0,0 +1,8 @@
+{
+  "image_digests": {
+    "green-agent": "ghcr.io/froot-netsys/k8s_agent@sha256:f92db8d6764a57040ef4e44d04190eff1eaaa3f163ff97e1beb9e1bfdcc86b91",
+    "k8s_operator": "ghcr.io/froot-netsys/a2a_llm@sha256:4dd2c986b5ee439c71d4b758b1fb8b201e940fffa022b87e1c8c375a57abfba9",
+    "agentbeats-client": "ghcr.io/agentbeats/agentbeats-client@sha256:13dfe3ef4e583a80e7ce2fe3becd0ce3b879841368a7f4fa40b6ebbabeeb014e"
+  },
+  "timestamp": "2026-01-11T17:34:31Z"
+}
diff --git a/submissions/lesleychou-20260111-173431.toml b/submissions/lesleychou-20260111-173431.toml
@@ -0,0 +1,24 @@
+[green_agent]
+agentbeats_id = "019ba4cb-2f71-7583-a69b-07e1c4329428"
+env = { LOG_LEVEL = "INFO" } # to trigger test again
+
+[[participants]]
+agentbeats_id = "019ba44f-9d1f-7ff3-8fbe-35c4ecfa40b2"
+name = "k8s_operator"
+env = { AZURE_API_KEY = "${AZURE_API_KEY}", AZURE_API_BASE = "${AZURE_API_BASE}", AZURE_API_VERSION = "${AZURE_API_VERSION}", MODEL_NAME = "${MODEL_NAME}" }
+
+[config]
+# Prompt strategy for the agent. Options: "zeroshot_base", "zeroshot_cot", "fewshot_base", "fewshot_cot"
+prompt_type = "zeroshot_base"
+
+# Number of queries to generate for each error type (15 error types total)
+num_queries = 2
+
+# Maximum number of iterations the agent can take to resolve a given K8s network policy issue.
+max_iterations = 15
+
+# DO NOT MODIFY BELOW THIS LINE.
+benchmark_path = "assessment_error_config.json"
+output_dir = "dump"
+microservice_dir = "/microservices-demo"
+regenerate_config = true
