add s3 smoke test

Author: you-rgb

.github/workflows/infinia-deploy-aws.yml

@@ -129,8 +129,9 @@ jobs:
           AWS_REGION: us-east-1
         run: |
           python3 scripts/s3_smoke_test.py \
-            --region "${{ inputs.aws_region }}" \
+            --region "us-east-1" \
             --admin-password "${{ secrets.INFINIA_ADMIN_PASSWORD }}" \
             --realm-tag-key "Role" --realm-tag-value "realm" \
             --client-tag-key "Role" --client-tag-value "client" \
-            --endpoint-port 8111
+            --endpoint-port 8111 \
+            --no-verify-ssl

.github/workflows/prechecks-aws.yml

@@ -55,11 +55,11 @@ jobs:
       infinia_version: ${{ needs.prepare.outputs.infinia_version }}
     secrets: inherit
 
-  infinia-destroy:
-    name: Destroy Infinia Infrastructure
-    needs: infinia-deploy
-    if: always()
-    uses: ./.github/workflows/infinia-destroy-aws.yml
-    with:
-      aws_region: us-east-1
-    secrets: inherit
+  # infinia-destroy:
+  #   name: Destroy Infinia Infrastructure
+  #   needs: infinia-deploy
+  #   if: always()
+  #   uses: ./.github/workflows/infinia-destroy-aws.yml
+  #   with:
+  #     aws_region: us-east-1
+  #   secrets: inherit

scripts/s3_smoke_test.py scripts/s3_smoke_test_aws.pyscripts/s3_smoke_test.py renamed to scripts/s3_smoke_test_aws.py

@@ -5,67 +5,67 @@
 End-to-end S3 API smoke test for Infinia, driven via AWS SSM.
 
 Flow:
-1) Find the realm EC2 by tag (e.g., Role=realm). Get its instance ID and public IP.
-2) On the realm instance (via SSM), run redcli to create s3_test_user and mint S3 access keys.
-3) Parse S3_KEY and S3_SECRET from the SSM output.
+1) Find the realm EC2 by tag (e.g., Role=realm). Get instance ID + IP.
+2) On the realm (via SSM), run redcli to mint S3 access keys for s3_test_user.
+3) Parse S3_KEY / S3_SECRET from the output (no secrets are printed to runner logs).
 4) Find all client EC2 instances by tag (e.g., Role=client).
-5) On each client (via SSM), write ~/.aws/credentials with the generated keys, then:
-   - create a unique bucket (ignore “already exists”),
-   - upload a timestamped file via aws s3 cp to the Infinia endpoint,
-   - list bucket objects and verify the file appears.
-
-Exit codes:
-  0 on success; non-zero on failure.
-
-Requirements on the runner:
-  - AWS credentials set (GitHub action `configure-aws-credentials` is fine)
-  - boto3, botocore installed (your pipeline already installs boto3)
-
-Assumptions:
-  - Clients already have AWS CLI v2 installed (as per your note).
-  - redcli is installed on the realm instance.
-  - The S3 endpoint is https://<realm-public-ip>:<endpoint_port>.
+5) On each client (via SSM), write ~/.aws/credentials with the generated keys,
+   create a unique bucket, upload a test file, list objects, and confirm success.
+
+Robustness:
+- Uses public IP if available, else falls back to private IP automatically.
+- You can force private IP with --use-private-endpoint.
+- You can override the endpoint host with --endpoint-host (e.g. internal NLB DNS).
+- Optional --no-verify-ssl to skip TLS verification against the endpoint.
+
+Exit codes: 0 = success; non-zero = failure.
 """
 
 import argparse
-import os
+import datetime
+import random
 import re
 import sys
 import time
-import json
-import random
-import datetime
+from typing import Optional, Dict, List
 
 import boto3
-from botocore.exceptions import ClientError
 
 
 def eprint(*args, **kwargs):
     print(*args, file=sys.stderr, **kwargs)
 
 
-def find_one_instance_id_by_tag(ec2, key, value):
+# ----------------------------- EC2 helpers ----------------------------- #
+
+def find_one_instance_id_by_tag(ec2, key: str, value: str) -> Optional[str]:
     resp = ec2.describe_instances(
         Filters=[
             {"Name": f"tag:{key}", "Values": [value]},
             {"Name": "instance-state-name", "Values": ["running"]},
         ],
-        MaxResults=50,
+        MaxResults=100,
     )
     for r in resp.get("Reservations", []):
         for i in r.get("Instances", []):
             return i["InstanceId"]
     return None
 
 
-def get_public_ip_by_instance_id(ec2, instance_id):
+def get_public_ip_by_instance_id(ec2, instance_id: str) -> Optional[str]:
     resp = ec2.describe_instances(InstanceIds=[instance_id])
     inst = resp["Reservations"][0]["Instances"][0]
     return inst.get("PublicIpAddress")
 
 
-def list_instance_ids_by_tag(ec2, key, value):
-    ids = []
+def get_private_ip_by_instance_id(ec2, instance_id: str) -> Optional[str]:
+    resp = ec2.describe_instances(InstanceIds=[instance_id])
+    inst = resp["Reservations"][0]["Instances"][0]
+    return inst.get("PrivateIpAddress")
+
+
+def list_instance_ids_by_tag(ec2, key: str, value: str) -> List[str]:
+    ids: List[str] = []
     paginator = ec2.get_paginator("describe_instances")
     for page in paginator.paginate(
         Filters=[
@@ -79,10 +79,14 @@ def list_instance_ids_by_tag(ec2, key, value):
     return ids
 
 
-def ssm_send_and_wait(ssm, instance_ids, commands, comment, timeout_sec=600, poll_sec=5):
-    """Send AWS-RunShellScript to one or many instances and wait for completion.
+# ----------------------------- SSM helpers ----------------------------- #
 
-    Returns dict: {instance_id: {"Status": str, "StdOut": str, "StdErr": str}}
+def ssm_send_and_wait(ssm, instance_ids, commands, comment, timeout_sec=900, poll_sec=5) -> Dict[str, Dict[str, str]]:
+    """
+    Send AWS-RunShellScript to one or many instances and wait for completion.
+
+    Returns dict keyed by instance_id:
+      {instance_id: {"Status": str, "StdOut": str, "StdErr": str}}
     """
     if isinstance(instance_ids, str):
         instance_ids = [instance_ids]
@@ -96,7 +100,7 @@ def ssm_send_and_wait(ssm, instance_ids, commands, comment, timeout_sec=600, pol
     cmd_id = resp["Command"]["CommandId"]
 
     deadline = time.time() + timeout_sec
-    results = {iid: None for iid in instance_ids}
+    results: Dict[str, Optional[Dict[str, str]]] = {iid: None for iid in instance_ids}
 
     while time.time() < deadline:
         done = True
@@ -123,15 +127,17 @@ def ssm_send_and_wait(ssm, instance_ids, commands, comment, timeout_sec=600, pol
             break
         time.sleep(poll_sec)
 
-    # Fill any missing as timeout
+    # Mark any missing as timeout
     for iid in instance_ids:
         if results[iid] is None:
             results[iid] = {"Status": "TimedOut", "StdOut": "", "StdErr": ""}
 
-    return results
+    return results  # type: ignore[return-value]
+
 
+# ----------------------------- Parsing helpers ----------------------------- #
 
-def parse_redcli_creds_from_text(text):
+def parse_redcli_creds_from_text(text: str):
     """
     Extract S3_KEY and S3_SECRET from redcli pretty-table-like output.
 
@@ -146,49 +152,76 @@ def parse_redcli_creds_from_text(text):
     return s3_key.strip(), s3_secret.strip()
 
 
-def main():
+# ----------------------------- Main ----------------------------- #
+
+def main() -> int:
     ap = argparse.ArgumentParser(description="Infinia S3 API smoke test via SSM")
     ap.add_argument("--region", required=True, help="AWS region (e.g. us-east-1)")
     ap.add_argument("--admin-password", required=True, help="Infinia realm_admin password")
     ap.add_argument("--realm-tag-key", default="Role")
     ap.add_argument("--realm-tag-value", default="realm")
     ap.add_argument("--client-tag-key", default="Role")
     ap.add_argument("--client-tag-value", default="client")
-    ap.add_argument("--endpoint-port", type=int, default=8111)
-    ap.add_argument("--no-verify-ssl", action="store_true", default=True,
-                    help="Use --no-verify-ssl against the endpoint")
-    ap.add_argument("--timeout-sec", type=int, default=900)
+
+    ap.add_argument("--endpoint-port", type=int, default=8111, help="Infinia S3 endpoint port (default: 8111)")
+    ap.add_argument("--use-private-endpoint", action="store_true",
+                    help="Force using the realm's private IP for the endpoint URL")
+    ap.add_argument("--endpoint-host", default="",
+                    help="Override endpoint host entirely (e.g., internal NLB DNS). If set, IP detection is skipped.")
+    ap.add_argument("--no-verify-ssl", action="store_true",
+                    help="Pass --no-verify-ssl to aws CLI when calling the endpoint")
+
+    ap.add_argument("--timeout-sec", type=int, default=900, help="Overall SSM per-command timeout (default: 900s)")
+
     args = ap.parse_args()
 
     session = boto3.Session(region_name=args.region)
     ec2 = session.client("ec2")
     ssm = session.client("ssm")
 
-    # 1) Realm
+    # 1) Realm instance
     realm_id = find_one_instance_id_by_tag(ec2, args.realm_tag_key, args.realm_tag_value)
     if not realm_id:
         eprint(f"❌ No running realm instance found with tag {args.realm_tag_key}={args.realm_tag_value}")
         return 2
     eprint(f"Realm instance: {realm_id}")
 
-    realm_ip = get_public_ip_by_instance_id(ec2, realm_id)
-    if not realm_ip:
-        eprint("❌ Realm public IP not found")
-        return 2
-    endpoint_url = f"https://{realm_ip}:{args.endpoint_port}"
-    eprint(f"Realm endpoint: {endpoint_url}")
+    # Endpoint host resolution
+    if args.endpoint_host:
+        target_host = args.endpoint_host
+        reason = "custom host override"
+    else:
+        pub_ip = get_public_ip_by_instance_id(ec2, realm_id)
+        priv_ip = get_private_ip_by_instance_id(ec2, realm_id)
+
+        if args.use_private_endpoint:
+            if not priv_ip:
+                eprint("❌ --use-private-endpoint set, but no PrivateIpAddress on realm")
+                return 2
+            target_host, reason = priv_ip, "private (forced)"
+        else:
+            if pub_ip and pub_ip != "None":
+                target_host, reason = pub_ip, "public"
+            elif priv_ip:
+                target_host, reason = priv_ip, "private (fallback)"
+            else:
+                eprint("❌ Neither PublicIpAddress nor PrivateIpAddress found on realm")
+                return 2
+
+    endpoint_url = f"https://{target_host}:{args.endpoint_port}"
+    eprint(f"Realm endpoint ({reason}): {endpoint_url}")
 
     # 2) Create S3 creds on realm (SSM → redcli)
     redcli_cmds = [
-        'set -euo pipefail',
+        "set -euo pipefail",
         f'redcli user login realm_admin -p "{args.admin_password}"',
         'redcli user add s3_test_user -p Adminpassword -r red -t red || true',
         'redcli user grant s3_test_user red/red -t red || true',
         'OUT="$(redcli s3 access add s3_test_user -t red -r red/red/redobj -e 10y)"',
         'echo "===BEGIN-OUT==="',
         'echo "$OUT"',
         'echo "===END-OUT==="',
-        # Friendly parse lines for easier regex later
+        # Emit parse-friendly lines:
         'echo "S3_KEY=$(echo "$OUT" | sed -n \'s/.*S3_KEY[^A-Za-z0-9_-]*\\([A-Za-z0-9_-]\\+\\).*/\\1/p\' | head -n1)"',
         'echo "S3_SECRET=$(echo "$OUT" | sed -n \'s/.*S3_SECRET[^A-Za-z0-9+\\/=_-]*\\([A-Za-z0-9+\\/=_-]\\+\\).*/\\1/p\' | head -n1)"',
     ]
@@ -208,11 +241,10 @@ def main():
         eprint("STDOUT:\n" + realm_result["StdOut"])
         return 4
 
-    # Do not print secrets in logs
     eprint(f"S3_KEY parsed: {s3_key[:4]}… (masked)")
     eprint("S3_SECRET parsed: **** (masked)")
 
-    # 3) Find clients
+    # 3) Client instances
     client_ids = list_instance_ids_by_tag(ec2, args.client_tag_key, args.client_tag_value)
     if not client_ids:
         eprint(f"❌ No running client instances found with tag {args.client_tag_key}={args.client_tag_value}")
@@ -222,38 +254,43 @@ def main():
     # 4) S3 test on each client
     now = datetime.datetime.utcnow().strftime("%Y%m%d%H%M%S")
     bucket = f"infinia-smoke-{now}-{random.randint(1000,9999)}"
-    test_file_cmd = r'TS=$(date +%Y%m%dT%H%M%S); echo "S3 test file created at $TS" > /home/ubuntu/test-$TS.txt'
 
-    create_list_upload_cmds = [
+    no_verify = "--no-verify-ssl" if args.no_verify_ssl else ""
+    # All logic happens on the client; avoid jq dependency and keep it pure AWS CLI
+    client_cmds = [
         "set -euo pipefail",
         "mkdir -p /home/ubuntu/.aws && chmod 700 /home/ubuntu/.aws",
-        # write credentials (no echo in logs)
+        # Write credentials (won't echo contents)
         f"cat > /home/ubuntu/.aws/credentials <<'EOF'\n[default]\naws_access_key_id={s3_key}\naws_secret_access_key={s3_secret}\nEOF",
-        test_file_cmd,
+        "TS=$(date +%Y%m%dT%H%M%S)",
+        "FNAME=test-${TS}.txt",
+        'echo "S3 test file created at $TS" > "/home/ubuntu/$FNAME"',
         "export AWS_SHARED_CREDENTIALS_FILE=/home/ubuntu/.aws/credentials",
-        "aws --version || true",
-        f"aws {'--no-verify-ssl' if args.no_verify_ssl else ''} --endpoint-url={endpoint_url} s3api create-bucket --bucket {bucket} || true",
-        r"FILE=$(ls -1t /home/ubuntu/test-*.txt | head -n1)",
-        f"aws {'--no-verify-ssl' if args.no_verify_ssl else ''} --endpoint-url={endpoint_url} s3 cp \"$FILE\" s3://{bucket}/",
-        f"aws {'--no-verify-ssl' if args.no_verify_ssl else ''} --endpoint-url={endpoint_url} s3api list-objects-v2 --bucket {bucket} --output json --query 'Contents[].Key'",
-        "echo OK",
+        # Create bucket (ignore exists)
+        f'aws {no_verify} --endpoint-url="{endpoint_url}" s3api create-bucket --bucket "{bucket}" || true',
+        # Upload
+        f'aws {no_verify} --endpoint-url="{endpoint_url}" s3 cp "/home/ubuntu/$FNAME" "s3://{bucket}/"',
+        # List and verify presence using --query (prints JSON array or "None")
+        f'LIST=$(aws {no_verify} --endpoint-url="{endpoint_url}" s3api list-objects-v2 --bucket "{bucket}" --query "Contents[].Key" --output text || true)',
+        'echo "LIST:$LIST"',
+        'echo "$LIST" | grep -q "$FNAME" && echo OK',
     ]
 
-    client_results = ssm_send_and_wait(
-        ssm, client_ids, create_list_upload_cmds, "Infinia S3 smoke test", timeout_sec=args.timeout_sec
+    fanout = ssm_send_and_wait(
+        ssm, client_ids, client_cmds, "Infinia S3 smoke test", timeout_sec=args.timeout_sec
     )
 
-    had_fail = False
-    for iid, res in client_results.items():
+    failed = False
+    for iid, res in fanout.items():
         print(f"---- Client {iid} ({res['Status']}) ----")
-        # Show stdout for visibility but keep credentials masked (we never echo them above)
+        # StdOut shows "LIST: ..." and "OK" if success; secrets are never printed
         print(res["StdOut"])
         if res["Status"] != "Success" or "OK" not in res["StdOut"]:
             eprint(f"❌ S3 smoke test failed on client {iid}")
             eprint(res["StdErr"])
-            had_fail = True
+            failed = True
 
-    if had_fail:
+    if failed:
         return 6
 
     print("✅ S3 smoke test passed on all clients")