feat: create multiple generators at once. Remove all of them. Save info about created generators in a local file

Author: kudroma404

.gitignore

@@ -210,4 +210,5 @@ __marimo__/
 prompts.txt
 results/
 images*/
-duels.json
+duels.json
+generators.csv

README.md

@@ -164,26 +164,60 @@ Avoid these after submitting — they create new commits with different SHAs:
 
 ### Start Generator
 
-The `start-generator` command deploys and starts a generator container on Targon. It deploys the container and outputs the container URL which can then be used with the `generate` command.
+The `start-generator` command deploys and starts one or more generator containers on Targon concurrently. It can create multiple containers at once and saves their names and URLs to a CSV file. The command displays tables showing successful and failed container deployments.
 
 **Options:**
 - `--image-url` (required): URL of the Docker image to deploy
 - `--targon-api-key` (required): Targon API key for authentication
+- `--hf-token` (optional): HuggingFace token to pass as `HF_TOKEN` environment variable to containers
+- `--name` (optional): Custom container name prefix. Containers will be named `generator_{name}_{index}`. If not provided, uses default naming.
+- `--count` (optional, default: 1): Number of containers to create concurrently
+- `--output-file` (optional, default: "generators.csv"): Path to the CSV file where container names and URLs will be saved
 
 **Example:**
 ```bash
 404-cli start-generator \
   --image-url docker.io/username/model-generator:v1.0.0 \
-  --targon-api-key your-targon-api-key-here
+  --targon-api-key your-targon-api-key-here \
+  --name my-model \
+  --count 4 \
+  --hf-token hf_xxxxxxxxxxxxx \
+  --output-file my-generators.csv
 ```
 
+This will create 4 containers named `generator_my-model_0`, `generator_my-model_1`, `generator_my-model_2`, and `generator_my-model_3`.
+
 **Output:**
-On success, outputs JSON with the container URL:
+The command displays two tables:
+1. **Successful Containers**: Shows ID, container name, and URL for each successfully deployed container
+2. **Failed Containers**: Shows ID, container name, and error message for any failed deployments
+
+On success, outputs JSON:
+```json
+{"success": true, "output_file": "generators.csv", "created": 4}
+```
+
+If some containers fail:
 ```json
-{"success": true, "container_url": "https://generator-abc123.targon.io"}
+{
+  "success": true,
+  "output_file": "generators.csv",
+  "created": 3,
+  "failed": 1,
+  "failed_containers": [
+    {"name": "generator_my-model_2", "error": "Container deployment timeout"}
+  ]
+}
 ```
 
-The container URL is also displayed on stderr and should be used as the `--endpoint` parameter for the `generate` command.
+**CSV Output:**
+The CSV file contains two columns: `container_name` and `url`. Only successfully created containers are written to the CSV file.
+
+**Notes:**
+- Containers are created concurrently for faster deployment
+- Container URLs can be used as the `--endpoint` parameter for the `generate` command
+- If some containers fail to deploy, successful ones are still saved to the CSV file
+- Progress messages and container status are displayed on stderr, while JSON results go to stdout
 
 ### Generate Models
 

commit.py

@@ -1,4 +1,5 @@
 import asyncio
+import csv
 import json
 import re
 import sys
@@ -527,6 +528,59 @@ async def _list(round_number: int, schedule: Schedule, current_round: int, exclu
     # Always display as table
     _display_commitments_table(results, round_number)
 
+    
+def _display_containers_table(
+    successful_containers: list[tuple[str, str]], failed_containers: list[tuple[str, str]]
+) -> None:
+    """Display successful and failed containers in formatted tables."""
+    # Display successful containers
+    if successful_containers:
+        click.echo(f"\n{'='*120}", err=True)
+        click.echo(f"Successful Containers ({len(successful_containers)})", err=True)
+        click.echo(f"{'='*120}", err=True)
+        
+        # Calculate column widths
+        max_name_len = max(len(name) for name, _ in successful_containers) if successful_containers else 0
+        max_url_len = max(len(url) for _, url in successful_containers) if successful_containers else 0
+        name_width = max(max_name_len, len("Container Name"))
+        url_width = max(max_url_len, len("URL"))
+        
+        # Table header
+        header = f"{'ID':<4} {'Container Name':<{name_width}} {'URL':<{url_width}}"
+        click.echo(header, err=True)
+        click.echo("-" * 120, err=True)
+        
+        # Table rows
+        for idx, (name, url) in enumerate(successful_containers, 1):
+            row = f"{idx:<4} {name:<{name_width}} {url:<{url_width}}"
+            click.echo(row, err=True)
+        
+        click.echo(f"{'='*120}\n", err=True)
+    
+    # Display failed containers
+    if failed_containers:
+        click.echo(f"{'='*120}", err=True)
+        click.echo(f"Failed Containers ({len(failed_containers)})", err=True)
+        click.echo(f"{'='*120}", err=True)
+        
+        # Calculate column widths
+        max_name_len = max(len(name) for name, _ in failed_containers) if failed_containers else 0
+        max_error_len = max(len(error) for _, error in failed_containers) if failed_containers else 0
+        name_width = max(max_name_len, len("Container Name"))
+        error_width = max(max_error_len, len("Error"))
+        
+        # Table header
+        header = f"{'ID':<4} {'Container Name':<{name_width}} {'Error':<{error_width}}"
+        click.echo(header, err=True)
+        click.echo("-" * 120, err=True)
+        
+        # Table rows
+        for idx, (name, error) in enumerate(failed_containers, 1):
+            row = f"{idx:<4} {name:<{name_width}} {error:<{error_width}}"
+            click.echo(row, err=True)
+        
+        click.echo(f"{'='*120}\n", err=True)
+
 
 def _display_commitments_table(results: list[dict], round_number: int) -> None:
     """Display commitments in a formatted table."""
@@ -638,7 +692,9 @@ def _parse_commitments(commitments: dict, round_number: int, schedule: Schedule,
     help="HuggingFace token to pass as HF_TOKEN environment variable",
 )
 @click.option("--name", "container_name", default=None, help="Custom container name (default: generator)")
-def start_generator_cmd(image_url: str, targon_api_key: str, hf_token: str | None, container_name: str | None) -> None:
+@click.option("--count", default=1, help="Number of models to generate (default: 1)")
+@click.option("--output-file", default="generators.csv", help="Path to the CSV file where container names and URLs will be saved (default: generators.csv)")
+def start_generator_cmd(image_url: str, targon_api_key: str, hf_token: str | None, container_name: str | None, count: float, output_file: str) -> None:
     """Start the generator container."""
     click.echo(f"Starting generator: {image_url}", err=True)
 
@@ -648,24 +704,78 @@ def start_generator_cmd(image_url: str, targon_api_key: str, hf_token: str | Non
             env = {"HF_TOKEN": hf_token}
 
         # Format container name: "generator_{name}" if name provided, otherwise use default
-        if container_name:
-            name = f"generator_{container_name}"
-        else:
-            name = _GENERATOR_POD_NAME
-
-        container_url = asyncio.run(
-            _create_container(
-                image_url=image_url,
-                container_name=name,
-                targon_api_key=targon_api_key,
-                resource_name="h200-small",
-                port=_GENERATOR_PORT,
-                health_check_path=_GENERATOR_HEALTH_CHECK_PATH,
-                echo=lambda msg: click.echo(msg, err=True),
-                env=env,
+        container_names: list[str] = []
+        for idx in range(int(count)):
+            if container_name:
+                name = f"generator_{container_name}_{idx}"
+            else:
+                name = f"{_GENERATOR_POD_NAME}_{idx}"
+            container_names.append(name)
+
+        # Create async function to run tasks
+        async def _create_all_containers() -> list[str | Exception]:
+            """Create all containers concurrently using tasks."""
+            tasks: list[asyncio.Task[str]] = []
+            for name in container_names:
+                task = asyncio.create_task(
+                    _create_container(
+                        image_url=image_url,
+                        container_name=name,
+                        targon_api_key=targon_api_key,
+                        resource_name="h200-small",
+                        port=_GENERATOR_PORT,
+                        health_check_path=_GENERATOR_HEALTH_CHECK_PATH,
+                        echo=lambda msg: click.echo(msg, err=True),
+                        env=env,
+                    )
+                )
+                tasks.append(task)
+            
+            # Gather results, allowing some tasks to fail
+            return await asyncio.gather(*tasks, return_exceptions=True)
+
+        # Run the async function
+        results = asyncio.run(_create_all_containers())
+        
+        # Separate successful and failed containers
+        successful_containers: list[tuple[str, str]] = []
+        failed_containers: list[tuple[str, str]] = []
+        
+        for name, result in zip(container_names, results):
+            if isinstance(result, Exception):
+                error_msg = str(result)
+                failed_containers.append((name, error_msg))
+                logger.error(f"Failed to create container {name}: {error_msg}")
+            else:
+                successful_containers.append((name, result))
+        
+        # Display results in tables
+        _display_containers_table(successful_containers, failed_containers)
+        
+        # Write successful containers to CSV file
+        with Path(output_file).open("w", newline="") as f:
+            writer = csv.writer(f)
+            writer.writerow(["container_name", "url"])
+            for name, url in successful_containers:
+                writer.writerow([name, url])
+        
+        # Report results
+        if failed_containers:
+            click.echo(
+                json.dumps(
+                    {
+                        "success": len(successful_containers) > 0,
+                        "output_file": output_file,
+                        "created": len(successful_containers),
+                        "failed": len(failed_containers),
+                        "failed_containers": [
+                            {"name": name, "error": error} for name, error in failed_containers
+                        ],
+                    }
+                )
             )
-        )
-        click.echo(json.dumps({"success": True, "container_url": container_url}))
+        else:
+            click.echo(json.dumps({"success": True, "output_file": output_file, "created": len(successful_containers)}))
     except KeyboardInterrupt:
         logger.warning("Generator start interrupted by user")
         click.echo(json.dumps({"success": False, "error": "Interrupted by user"}))
@@ -800,21 +910,51 @@ def judge_cmd(
 
 @cli.command("stop-pods")
 @click.option("--targon-api-key", required=True, help="Targon API key.")
-def stop_pods_cmd(targon_api_key: str) -> None:
-    """Stop the generator, render and judge pods."""
+@click.option("--file", "csv_file", default="generators.csv", help="Path to CSV file with container names and URLs (default: generators.csv)")
+def stop_pods_cmd(targon_api_key: str, csv_file: str) -> None:
+    """Stop generator containers listed in a CSV file, plus render and judge pods."""
     click.echo("Stopping pods...", err=True)
+    
+    # Read container names from CSV file
+    container_names_from_csv: list[str] = []
+    try:
+        with Path(csv_file).open("r", newline="") as f:
+            reader = csv.DictReader(f)
+            for row in reader:
+                container_name = row.get("container_name", "").strip()
+                if container_name:
+                    container_names_from_csv.append(container_name)
+        
+        if container_names_from_csv:
+            click.echo(f"Found {len(container_names_from_csv)} container(s) in {csv_file}", err=True)
+    except FileNotFoundError:
+        click.echo(f"CSV file {csv_file} not found, will only stop render and judge pods", err=True)
+    except Exception as e:
+        click.echo(f"Failed to read CSV file {csv_file}: {e}, will only stop render and judge pods", err=True)
 
     async def _stop() -> None:
         async with TargonClient(api_key=targon_api_key) as targon:
             containers = await targon.list_containers()
+            # Create a mapping of container names to UIDs
+            name_to_uid: dict[str, str] = {c.name: c.uid for c in containers}
+            stopped: list[str] = []
+            not_found: list[str] = []
+            # Stop containers from CSV file
+            for container_name in container_names_from_csv:
+                if container_name in name_to_uid:
+                    click.echo(f"Stopping container {container_name} ({name_to_uid[container_name]})", err=True)
+                    await targon.delete_container(name_to_uid[container_name])
+                    stopped.append(container_name)
+                else:
+                    click.echo(f"Container {container_name} not found", err=True)
+                    not_found.append(container_name)
+            # Also stop render and judge pods (legacy behavior)
             for c in containers:
-                # Stop all containers that start with "generator_", plus render and judge pods
-                if c.name.startswith("generator_") or c.name in [
-                    _RENDER_POD_NAME,
-                    _JUDGE_POD_NAME,
-                ]:
-                    click.echo(f"Stopping container {c.name} ({c.uid})", err=True)
-                    await targon.delete_container(c.uid)
+                if c.name in [_RENDER_POD_NAME, _JUDGE_POD_NAME]:
+                    if c.name not in stopped:  # Avoid stopping twice if it was in CSV
+                        click.echo(f"Stopping container {c.name} ({c.uid})", err=True)
+                        await targon.delete_container(c.uid)
+                        stopped.append(c.name)
 
     try:
         asyncio.run(_stop())
@@ -926,21 +1066,21 @@ async def _create_container(
                 echo=echo,
             )
             if container:
-                echo(f"Container deployed successfully. UID: {container.uid}")
-                echo(f"Container URL: {container.url}")
+                echo(f"{container_name}: Container deployed successfully. UID: {container.uid}")
+                echo(f"{container_name}: Container URL: {container.url}")
                 url: str = str(container.url)
                 return url
             else:
-                raise RuntimeError("Failed to deploy and start container")
+                raise RuntimeError(f"{container_name}: Failed to deploy and start container")
     except (KeyboardInterrupt, asyncio.CancelledError):
-        echo("\nInterrupted by user. Cleaning up...")
+        echo(f"{container_name}: \nInterrupted by user. Cleaning up...")
         if container:
             try:
                 async with TargonClient(api_key=targon_api_key) as targon:
                     await targon.delete_container(container.uid)
-                    echo("Container deleted successfully")
+                    echo(f"{container_name}: Container deleted successfully")
             except Exception as cleanup_error:
-                echo(f"Error during cleanup: {cleanup_error}")
+                echo(f"{container_name}: Error during cleanup: {cleanup_error}")
         raise
 
 

targon_utils.py

@@ -53,6 +53,7 @@ async def wait_for_healthy(
     timeout: float,  # noqa: ASYNC109
     check_interval: float = 5.0,
     health_check_path: str = "/health",
+    name: str | None = None,
     echo: Callable[[str], None] | None = None,
 ) -> bool:
     """Wait for the container health endpoint to return 200 (stage 2)."""
@@ -62,6 +63,7 @@ async def wait_for_healthy(
     else:
         health_url = f"{url}{health_check_path}"
 
+    container_label = f"{name}: " if name else ""
     async with httpx.AsyncClient(timeout=30.0) as http:
         start = asyncio.get_running_loop().time()
         time_elapsed = asyncio.get_running_loop().time() - start
@@ -70,13 +72,13 @@ async def wait_for_healthy(
                 response = await http.get(health_url)
                 response.raise_for_status()
                 if response.status_code == 200:
-                    _log(f"Container at {url} healthy", echo, "info")
+                    _log(f"{container_label}Container at {url} healthy", echo, "info")
                     return True
             except Exception:
-                _log(f"Container not ready yet: {time_elapsed:.1f}/{timeout:.1f}s", echo, "info")
+                _log(f"{container_label}Container not ready yet: {time_elapsed:.1f}/{timeout:.1f}s", echo, "info")
                 await asyncio.sleep(check_interval)
                 time_elapsed = asyncio.get_running_loop().time() - start
-    _log(f"Container at {url} not healthy within {timeout}s. Timeout reached.", echo, "error")
+    _log(f"{container_label}Container at {url} not healthy within {timeout}s. Timeout reached.", echo, "error")
     return False
 
 
@@ -132,6 +134,7 @@ async def ensure_running_container(
         timeout=warmup_timeout,
         check_interval=check_interval,
         health_check_path=health_check_path,
+        name=name,
         echo=echo,
     ):
         _log("Container failed health check, deleting", echo, "error")