fix(operator): dead doc link, VPC Link race hardening, Cloud Map teardown retry

Round-8 review:
- F1 (blocking): README and code comments linked to
  docs/refarch/running-telegram-line-on-aws.md, which does not exist on
  main or this branch (its PR #1274 is still open). Repointed the README
  to the existing docs/refarch/telegram-cloudflare-tunnel.md as an
  interim cross-reference with a note that a dedicated AWS-native doc is
  tracked in #1274; ingress.rs/manifest.rs doc comments now point at
  operator/README.md instead of the nonexistent file.
- F2: VPC Link names are not unique to the apigatewayv2 API (confirmed:
  AWS does not error on a duplicate name), so two 'oabctl apply'
  processes racing to create the same per-VPC link in a brand-new VPC
  could end up with two links. Within a single apply run this can't
  happen (manifests are processed sequentially), but the reconciler
  hardens against the cross-process race anyway: ensure_vpc_link now
  collects ALL matching links, deterministically picks the
  lexicographically-first ID (stable regardless of list ordering) rather
  than an arbitrary one, and warns with cleanup commands if more than one
  exists.
- F3: Cloud Map service teardown gave up on the first 'still has
  registered instances' error, permanently orphaning the service if the
  caller didn't manually retry. ECS deregisters the instance
  asynchronously on scale-to-0/delete, so this is usually just a timing
  window, not a real conflict — teardown now retries delete_service up to
  6 times over ~25s before falling back to a warning with the exact
  manual cleanup command.

Verified: build, clippy --all-targets -D warnings, cargo test (13 passed).

Author: chaodu-agent

operator/README.md

@@ -215,9 +215,12 @@ Discord bots are outbound-only and need no ingress. Webhook platforms (Telegram,
 LINE, ...) POST *into* the task, so they need a public HTTPS endpoint. Adding an
 optional `spec.ingress` block makes `oabctl apply` provision the cheapest
 AWS-native path in one shot — API Gateway HTTP API → VPC Link → Cloud Map → the
-task — instead of running ~7 manual `aws` commands. See
-[`docs/refarch/running-telegram-line-on-aws.md`](../docs/refarch/running-telegram-line-on-aws.md)
-(Option 1) for the architecture and cost breakdown.
+task — instead of running ~7 manual `aws` commands, replacing the manual steps
+implemented here. For a Kubernetes/Cloudflare-Tunnel alternative, see
+[`docs/refarch/telegram-cloudflare-tunnel.md`](../docs/refarch/telegram-cloudflare-tunnel.md).
+A dedicated AWS-native refarch doc covering this path in depth is tracked in
+[#1274](https://github.com/openabdev/openab/pull/1274); once merged it will be
+linked here.
 
 ```yaml
 spec:

operator/src/ingress.rs

@@ -1,7 +1,11 @@
 //! Ingress reconciliation for webhook-based platforms (Telegram, LINE, ...).
 //!
-//! Implements Option 1 of `docs/refarch/running-telegram-line-on-aws.md`:
-//! API Gateway HTTP API → VPC Link → Cloud Map → ECS Fargate task on `:port`.
+//! Implements the API Gateway HTTP API → VPC Link → Cloud Map → ECS Fargate
+//! path for inbound webhook ingress (Telegram, LINE, ...), replacing ~7 manual
+//! `aws apigatewayv2`/`servicediscovery`/`ecs` CLI steps. See
+//! `operator/README.md` ("Ingress — inbound webhooks") for the manifest schema
+//! and operational notes; a dedicated AWS reference architecture doc for this
+//! path is tracked in openabdev/openab#1274.
 //!
 //! All operations are idempotent — resources are looked up by name and reused,
 //! so repeated `oabctl apply` runs converge instead of duplicating. The shared
@@ -180,11 +184,31 @@ pub async fn teardown(config: &aws_config::SdkConfig, namespace: &str, name: &st
         }
     }
     if let Some(service_id) = service_id {
-        match sd.delete_service().id(&service_id).send().await {
-            Ok(_) => eprintln!("  ✓ Deleted Cloud Map service: {service_name}"),
-            Err(e) => eprintln!(
-                "  ⚠ Cloud Map service '{service_name}' not deleted — it may still have\n    registered instances; retry once the ECS tasks have fully drained ({e})"
-            ),
+        // ECS deregisters the task's Cloud Map instance asynchronously when a
+        // service scales to 0 / is deleted, so `delete_service` can fail with
+        // "still has registered instances" for a short window even though the
+        // task is already gone. Retry briefly instead of giving up on the
+        // first attempt — this is the common case, not an edge case.
+        let mut last_err = None;
+        let mut deleted = false;
+        for attempt in 0..6 {
+            if attempt > 0 {
+                tokio::time::sleep(std::time::Duration::from_secs(5)).await;
+            }
+            match sd.delete_service().id(&service_id).send().await {
+                Ok(_) => {
+                    eprintln!("  ✓ Deleted Cloud Map service: {service_name}");
+                    deleted = true;
+                    break;
+                }
+                Err(e) => last_err = Some(e),
+            }
+        }
+        if !deleted {
+            eprintln!(
+                "  ⚠ Cloud Map service '{service_name}' not deleted after retrying — it still\n    has registered instances. It will be orphaned until manually removed:\n      aws servicediscovery delete-service --id {service_id}\n    ({})",
+                last_err.map(|e| e.to_string()).unwrap_or_default()
+            );
         }
     }
 
@@ -399,35 +423,52 @@ async fn ensure_vpc_link(
     use aws_sdk_apigatewayv2::types::VpcLinkStatus;
     let link_name = vpc_link_name(vpc_id);
 
-    // Reuse an existing, non-failed VPC Link with our per-VPC name.
-    let mut found: Option<(String, Option<VpcLinkStatus>)> = None;
+    // Reuse an existing, non-failed VPC Link with our per-VPC name. VPC Link
+    // names are NOT unique to the API — if two `oabctl apply` invocations race
+    // to create the same-named link in a brand-new VPC (e.g. a fleet's agents
+    // applied via separate concurrent processes), AWS will happily create two.
+    // We can't prevent that race across processes, but we can make reuse
+    // deterministic afterward: collect all matches and always pick the one
+    // with the lexicographically smallest ID (stable across repeated calls,
+    // regardless of list ordering), warning if more than one exists so an
+    // operator can clean up the duplicate.
+    let mut candidates: Vec<(String, Option<VpcLinkStatus>)> = Vec::new();
     let mut next: Option<String> = None;
-    'outer: loop {
+    loop {
         let mut req = api.get_vpc_links();
         if let Some(t) = &next {
             req = req.next_token(t);
         }
         let resp = req.send().await.context("failed to list VPC Links")?;
         for link in resp.items() {
-            if link.name() == Some(link_name.as_str()) {
-                if matches!(
+            if link.name() == Some(link_name.as_str())
+                && !matches!(
                     link.vpc_link_status(),
                     Some(VpcLinkStatus::Failed) | Some(VpcLinkStatus::Deleting)
-                ) {
-                    continue;
-                }
-                found = Some((
+                )
+            {
+                candidates.push((
                     link.vpc_link_id().unwrap_or_default().to_string(),
                     link.vpc_link_status().cloned(),
                 ));
-                break 'outer;
             }
         }
         match resp.next_token() {
             Some(t) => next = Some(t.to_string()),
             None => break,
         }
     }
+    candidates.sort_by(|(a, _), (b, _)| a.cmp(b));
+    if candidates.len() > 1 {
+        eprintln!(
+            "  ⚠ Found {} VPC Links named '{link_name}' (a race between concurrent\n    `apply` runs can create duplicates — AWS does not enforce name\n    uniqueness). Using the lexicographically first; consider deleting the extras:",
+            candidates.len()
+        );
+        for (id, _) in &candidates[1..] {
+            eprintln!("      aws apigatewayv2 delete-vpc-link --vpc-link-id {id}");
+        }
+    }
+    let found = candidates.into_iter().next();
 
     let link_id = if let Some((id, _status)) = found {
         eprintln!("  ✓ VPC Link exists: {link_name} ({id})");

operator/src/manifest.rs

@@ -167,8 +167,9 @@ pub struct Spec {
 /// Inbound HTTPS ingress for webhook-based platforms (Telegram, LINE, ...).
 ///
 /// Provisions the cheapest AWS-native path: API Gateway HTTP API → VPC Link →
-/// Cloud Map → the ECS task on `containerPort`. See
-/// `docs/refarch/running-telegram-line-on-aws.md` (Option 1).
+/// Cloud Map → the ECS task on `containerPort`. See `operator/README.md`
+/// ("Ingress — inbound webhooks") for the manifest schema and operational
+/// notes.
 #[derive(Debug, Deserialize, Serialize, Clone)]
 #[serde(rename_all = "camelCase")]
 pub struct Ingress {